Merge branch 'main' into fix/chore-fix

api/.env.example

@@ -42,6 +42,11 @@ REDIS_SENTINEL_USERNAME=
 REDIS_SENTINEL_PASSWORD=
 REDIS_SENTINEL_SOCKET_TIMEOUT=0.1
 
+# redis Cluster configuration.
+REDIS_USE_CLUSTERS=false
+REDIS_CLUSTERS=
+REDIS_CLUSTERS_PASSWORD=
+
 # PostgreSQL database configuration
 DB_USERNAME=postgres
 DB_PASSWORD=difyai123456
@@ -234,6 +239,10 @@ ANALYTICDB_ACCOUNT=testaccount
 ANALYTICDB_PASSWORD=testpassword
 ANALYTICDB_NAMESPACE=dify
 ANALYTICDB_NAMESPACE_PASSWORD=difypassword
+ANALYTICDB_HOST=gp-test.aliyuncs.com
+ANALYTICDB_PORT=5432
+ANALYTICDB_MIN_CONNECTION=1
+ANALYTICDB_MAX_CONNECTION=5
 
 # OpenSearch configuration
 OPENSEARCH_HOST=127.0.0.1

api/README.md

@@ -18,12 +18,17 @@
    ```
 
 2. Copy `.env.example` to `.env`
+
+   ```cli
+   cp .env.example .env 
+   ```
 3. Generate a `SECRET_KEY` in the `.env` file.
 
+   bash for Linux
    ```bash for Linux
    sed -i "/^SECRET_KEY=/c\SECRET_KEY=$(openssl rand -base64 42)" .env
    ```
-
+   bash for Mac
    ```bash for Mac
    secret_key=$(openssl rand -base64 42)
    sed -i '' "/^SECRET_KEY=/c\\
@@ -41,14 +46,6 @@
    poetry install
    ```
 
-   In case of contributors missing to update dependencies for `pyproject.toml`, you can perform the following shell instead.
-
-   ```bash
-   poetry shell                                               # activate current environment
-   poetry add $(cat requirements.txt)           # install dependencies of production and update pyproject.toml
-   poetry add $(cat requirements-dev.txt) --group dev    # install dependencies of development and update pyproject.toml
-   ```
-
 6. Run migrate
 
    Before the first launch, migrate the database to the latest version.

api/commands.py

@@ -590,7 +590,7 @@ def upgrade_db():
             click.echo(click.style("Database migration successful!", fg="green"))
 
         except Exception as e:
-            logging.exception(f"Database migration failed: {e}")
+            logging.exception("Failed to execute database migration")
         finally:
             lock.release()
     else:
@@ -634,7 +634,7 @@ where sites.id is null limit 1000"""
                 except Exception as e:
                     failed_app_ids.append(app_id)
                     click.echo(click.style("Failed to fix missing site for app {}".format(app_id), fg="red"))
-                    logging.exception(f"Fix app related site missing issue failed, error: {e}")
+                    logging.exception(f"Failed to fix app related site missing issue, app_id: {app_id}")
                     continue
 
             if not processed_count:

api/configs/feature/__init__.py

@@ -674,6 +674,11 @@ class DataSetConfig(BaseSettings):
         default=False,
     )
 
+    PLAN_SANDBOX_CLEAN_MESSAGE_DAY_SETTING: PositiveInt = Field(
+        description="Interval in days for message cleanup operations - plan: sandbox",
+        default=30,
+    )
+
 
 class WorkspaceConfig(BaseSettings):
     """

api/configs/middleware/cache/redis_config.py

@@ -68,3 +68,18 @@ class RedisConfig(BaseSettings):
         description="Socket timeout in seconds for Redis Sentinel connections",
         default=0.1,
     )
+
+    REDIS_USE_CLUSTERS: bool = Field(
+        description="Enable Redis Clusters mode for high availability",
+        default=False,
+    )
+
+    REDIS_CLUSTERS: Optional[str] = Field(
+        description="Comma-separated list of Redis Clusters nodes (host:port)",
+        default=None,
+    )
+
+    REDIS_CLUSTERS_PASSWORD: Optional[str] = Field(
+        description="Password for Redis Clusters authentication (if required)",
+        default=None,
+    )

api/configs/middleware/vdb/analyticdb_config.py

@@ -1,6 +1,6 @@
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, PositiveInt
 
 
 class AnalyticdbConfig(BaseModel):
@@ -40,3 +40,11 @@ class AnalyticdbConfig(BaseModel):
         description="The password for accessing the specified namespace within the AnalyticDB instance"
         " (if namespace feature is enabled).",
     )
+    ANALYTICDB_HOST: Optional[str] = Field(
+        default=None, description="The host of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_PORT: PositiveInt = Field(
+        default=5432, description="The port of the AnalyticDB instance you want to connect to."
+    )
+    ANALYTICDB_MIN_CONNECTION: PositiveInt = Field(default=1, description="Min connection of the AnalyticDB database.")
+    ANALYTICDB_MAX_CONNECTION: PositiveInt = Field(default=5, description="Max connection of the AnalyticDB database.")

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="0.11.1",
+        default="0.11.2",
     )
 
     COMMIT_SHA: str = Field(

api/controllers/console/app/app.py

@@ -9,6 +9,7 @@ from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     account_initialization_required,
     cloud_edition_billing_resource_check,
+    enterprise_license_required,
     setup_required,
 )
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -29,6 +30,7 @@ class AppListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @enterprise_license_required
     def get(self):
         """Get app list"""
 
@@ -188,6 +190,7 @@ class AppApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @enterprise_license_required
     @get_app_model
     @marshal_with(app_detail_fields_with_site)
     def get(self, app_model):

api/controllers/console/app/audio.py

@@ -70,7 +70,7 @@ class ChatMessageAudioApi(Resource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception(f"internal server error, {str(e)}.")
+            logging.exception("Failed to handle post request to ChatMessageAudioApi")
             raise InternalServerError()
 
 
@@ -128,7 +128,7 @@ class ChatMessageTextApi(Resource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception(f"internal server error, {str(e)}.")
+            logging.exception("Failed to handle post request to ChatMessageTextApi")
             raise InternalServerError()
 
 
@@ -170,7 +170,7 @@ class TextModesApi(Resource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception(f"internal server error, {str(e)}.")
+            logging.exception("Failed to handle get request to TextModesApi")
             raise InternalServerError()
 
 

api/controllers/console/auth/forgot_password.py

@@ -14,7 +14,7 @@ from controllers.console.auth.error import (
     InvalidTokenError,
     PasswordMismatchError,
 )
-from controllers.console.error import EmailSendIpLimitError, NotAllowedRegister
+from controllers.console.error import AccountNotFound, EmailSendIpLimitError
 from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
 from extensions.ext_database import db
@@ -51,7 +51,7 @@ class ForgotPasswordSendEmailApi(Resource):
                 token = AccountService.send_reset_password_email(email=args["email"], language=language)
                 return {"result": "fail", "data": token, "code": "account_not_found"}
             else:
-                raise NotAllowedRegister()
+                raise AccountNotFound()
         else:
             token = AccountService.send_reset_password_email(account=account, email=args["email"], language=language)
 

api/controllers/console/auth/login.py

@@ -16,9 +16,9 @@ from controllers.console.auth.error import (
 )
 from controllers.console.error import (
     AccountBannedError,
+    AccountNotFound,
     EmailSendIpLimitError,
     NotAllowedCreateWorkspace,
-    NotAllowedRegister,
 )
 from controllers.console.wraps import setup_required
 from events.tenant_event import tenant_was_created
@@ -76,7 +76,7 @@ class LoginApi(Resource):
                 token = AccountService.send_reset_password_email(email=args["email"], language=language)
                 return {"result": "fail", "data": token, "code": "account_not_found"}
             else:
-                raise NotAllowedRegister()
+                raise AccountNotFound()
         # SELF_HOSTED only have one workspace
         tenants = TenantService.get_join_tenants(account)
         if len(tenants) == 0:
@@ -119,7 +119,7 @@ class ResetPasswordSendEmailApi(Resource):
             if FeatureService.get_system_features().is_allow_register:
                 token = AccountService.send_reset_password_email(email=args["email"], language=language)
             else:
-                raise NotAllowedRegister()
+                raise AccountNotFound()
         else:
             token = AccountService.send_reset_password_email(account=account, language=language)
 
@@ -148,7 +148,7 @@ class EmailCodeLoginSendEmailApi(Resource):
             if FeatureService.get_system_features().is_allow_register:
                 token = AccountService.send_email_code_login_email(email=args["email"], language=language)
             else:
-                raise NotAllowedRegister()
+                raise AccountNotFound()
         else:
             token = AccountService.send_email_code_login_email(account=account, language=language)
 

api/controllers/console/datasets/datasets.py

@@ -10,7 +10,7 @@ from controllers.console import api
 from controllers.console.apikey import api_key_fields, api_key_list
 from controllers.console.app.error import ProviderNotInitializeError
 from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.indexing_runner import IndexingRunner
 from core.model_runtime.entities.model_entities import ModelType
@@ -44,6 +44,7 @@ class DatasetListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @enterprise_license_required
     def get(self):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)

api/controllers/console/datasets/datasets_document.py

@@ -959,7 +959,7 @@ class DocumentRetryApi(DocumentResource):
                     raise DocumentAlreadyFinishedError()
                 retry_documents.append(document)
             except Exception as e:
-                logging.exception(f"Document {document_id} retry failed: {str(e)}")
+                logging.exception(f"Failed to retry document, document id: {document_id}")
                 continue
         # retry document
         DocumentService.retry_document(dataset_id, retry_documents)

api/controllers/console/error.py

@@ -52,8 +52,8 @@ class AccountBannedError(BaseHTTPException):
     code = 400
 
 
-class NotAllowedRegister(BaseHTTPException):
-    error_code = "unauthorized"
+class AccountNotFound(BaseHTTPException):
+    error_code = "account_not_found"
     description = "Account not found."
     code = 400
 
@@ -86,3 +86,9 @@ class NoFileUploadedError(BaseHTTPException):
     error_code = "no_file_uploaded"
     description = "Please upload your file."
     code = 400
+
+
+class UnauthorizedAndForceLogout(BaseHTTPException):
+    error_code = "unauthorized_and_force_logout"
+    description = "Unauthorized and force logout."
+    code = 401

api/controllers/console/remote_files.py

@@ -45,7 +45,7 @@ class RemoteFileUploadApi(Resource):
 
         resp = ssrf_proxy.head(url=url)
         if resp.status_code != httpx.codes.OK:
-            resp = ssrf_proxy.get(url=url, timeout=3)
+            resp = ssrf_proxy.get(url=url, timeout=3, follow_redirects=True)
         resp.raise_for_status()
 
         file_info = helpers.guess_file_info_from_response(resp)

api/controllers/console/workspace/account.py

@@ -14,7 +14,7 @@ from controllers.console.workspace.error import (
     InvalidInvitationCodeError,
     RepeatPasswordNotMatchError,
 )
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
 from extensions.ext_database import db
 from fields.member_fields import account_fields
 from libs.helper import TimestampField, timezone
@@ -79,6 +79,7 @@ class AccountProfileApi(Resource):
     @login_required
     @account_initialization_required
     @marshal_with(account_fields)
+    @enterprise_license_required
     def get(self):
         return current_user
 

api/controllers/console/workspace/members.py

@@ -1,3 +1,5 @@
+from urllib import parse
+
 from flask_login import current_user
 from flask_restful import Resource, abort, marshal_with, reqparse
 
@@ -57,11 +59,12 @@ class MemberInviteEmailApi(Resource):
                 token = RegisterService.invite_new_member(
                     inviter.current_tenant, invitee_email, interface_language, role=invitee_role, inviter=inviter
                 )
+                encoded_invitee_email = parse.quote(invitee_email)
                 invitation_results.append(
                     {
                         "status": "success",
                         "email": invitee_email,
-                        "url": f"{console_web_url}/activate?email={invitee_email}&token={token}",
+                        "url": f"{console_web_url}/activate?email={encoded_invitee_email}&token={token}",
                     }
                 )
             except AccountAlreadyInTenantError:

api/controllers/console/workspace/models.py

@@ -72,7 +72,10 @@ class DefaultModelApi(Resource):
                     model=model_setting["model"],
                 )
             except Exception as ex:
-                logging.exception(f"{model_setting['model_type']} save error: {ex}")
+                logging.exception(
+                    f"Failed to update default model, model type: {model_setting['model_type']},"
+                    f" model:{model_setting.get('model')}"
+                )
                 raise ex
 
         return {"result": "success"}
@@ -156,7 +159,10 @@ class ModelProviderModelApi(Resource):
                         credentials=args["credentials"],
                     )
                 except CredentialsValidateFailedError as ex:
-                    logging.exception(f"save model credentials error: {ex}")
+                    logging.exception(
+                        f"Failed to save model credentials, tenant_id: {tenant_id},"
+                        f" model: {args.get('model')}, model_type: {args.get('model_type')}"
+                    )
                     raise ValueError(str(ex))
 
         return {"result": "success"}, 200

api/controllers/console/workspace/tool_providers.py

@@ -7,7 +7,7 @@ from werkzeug.exceptions import Forbidden
 
 from configs import dify_config
 from controllers.console import api
-from controllers.console.wraps import account_initialization_required, setup_required
+from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
 from libs.helper import alphanumeric, uuid_value
 from libs.login import login_required
@@ -608,6 +608,7 @@ class ToolLabelsApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
+    @enterprise_license_required
     def get(self):
         return jsonable_encoder(ToolLabelsService.list_tool_labels())
 

api/controllers/console/wraps.py

@@ -9,10 +9,10 @@ from configs import dify_config
 from controllers.console.workspace.error import AccountNotInitializedError
 from extensions.ext_database import db
 from models.model import DifySetup
-from services.feature_service import FeatureService
+from services.feature_service import FeatureService, LicenseStatus
 from services.operation_service import OperationService
 
-from .error import NotInitValidateError, NotSetupError
+from .error import NotInitValidateError, NotSetupError, UnauthorizedAndForceLogout
 
 
 def account_initialization_required(view):
@@ -147,3 +147,15 @@ def setup_required(view):
         return view(*args, **kwargs)
 
     return decorated
+
+
+def enterprise_license_required(view):
+    @wraps(view)
+    def decorated(*args, **kwargs):
+        settings = FeatureService.get_system_features()
+        if settings.license.status in [LicenseStatus.INACTIVE, LicenseStatus.EXPIRED, LicenseStatus.LOST]:
+            raise UnauthorizedAndForceLogout("Your license is invalid. Please contact your administrator.")
+
+        return view(*args, **kwargs)
+
+    return decorated

api/controllers/web/audio.py

@@ -59,7 +59,7 @@ class AudioApi(WebApiResource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception(f"internal server error: {str(e)}")
+            logging.exception("Failed to handle post request to AudioApi")
             raise InternalServerError()
 
 
@@ -117,7 +117,7 @@ class TextApi(WebApiResource):
         except ValueError as e:
             raise e
         except Exception as e:
-            logging.exception(f"internal server error: {str(e)}")
+            logging.exception("Failed to handle post request to TextApi")
             raise InternalServerError()
 
 

api/core/agent/base_agent_runner.py

@@ -106,16 +106,9 @@ class BaseAgentRunner(AppRunner):
         # check if model supports stream tool call
         llm_model = cast(LargeLanguageModel, model_instance.model_type_instance)
         model_schema = llm_model.get_model_schema(model_instance.model, model_instance.credentials)
-        if model_schema and ModelFeature.STREAM_TOOL_CALL in (model_schema.features or []):
-            self.stream_tool_call = True
-        else:
-            self.stream_tool_call = False
-
-        # check if model supports vision
-        if model_schema and ModelFeature.VISION in (model_schema.features or []):
-            self.files = application_generate_entity.files
-        else:
-            self.files = []
+        features = model_schema.features if model_schema and model_schema.features else []
+        self.stream_tool_call = ModelFeature.STREAM_TOOL_CALL in features
+        self.files = application_generate_entity.files if ModelFeature.VISION in features else []
         self.query = None
         self._current_thoughts: list[PromptMessage] = []
 
@@ -243,7 +236,7 @@ class BaseAgentRunner(AppRunner):
         update prompt message tool
         """
         # try to get tool runtime parameters
-        tool_runtime_parameters = tool.get_runtime_parameters() or []
+        tool_runtime_parameters = tool.get_runtime_parameters()
 
         for parameter in tool_runtime_parameters:
             if parameter.form != ToolParameter.ToolParameterForm.LLM:

api/core/app/app_config/features/file_upload/manager.py

@@ -16,9 +16,7 @@ class FileUploadConfigManager:
         file_upload_dict = config.get("file_upload")
         if file_upload_dict:
             if file_upload_dict.get("enabled"):
-                transform_methods = file_upload_dict.get("allowed_file_upload_methods") or file_upload_dict.get(
-                    "allowed_upload_methods", []
-                )
+                transform_methods = file_upload_dict.get("allowed_file_upload_methods", [])
                 data = {
                     "image_config": {
                         "number_limits": file_upload_dict["number_limits"],

api/core/app/apps/advanced_chat/app_generator.py

@@ -373,5 +373,5 @@ class AdvancedChatAppGenerator(MessageBasedAppGenerator):
             if e.args[0] == "I/O operation on closed file.":  # ignore this error
                 raise GenerateTaskStoppedError()
             else:
-                logger.exception(e)
+                logger.exception(f"Failed to process generate task pipeline, conversation_id: {conversation.id}")
                 raise e

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -242,7 +242,7 @@ class AdvancedChatAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCyc
                     start_listener_time = time.time()
                     yield MessageAudioStreamResponse(audio=audio_trunk.audio, task_id=task_id)
             except Exception as e:
-                logger.exception(e)
+                logger.exception(f"Failed to listen audio message, task_id: {task_id}")
                 break
         if tts_publisher:
             yield MessageAudioEndStreamResponse(audio="", task_id=task_id)

api/core/app/apps/base_app_generator.py

@@ -34,8 +34,8 @@ class BaseAppGenerator:
                 tenant_id=app_config.tenant_id,
                 config=FileUploadConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
-                    allowed_extensions=entity_dictionary[k].allowed_file_extensions,
-                    allowed_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
+                    allowed_file_extensions=entity_dictionary[k].allowed_file_extensions,
+                    allowed_file_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
                 ),
             )
             for k, v in user_inputs.items()
@@ -48,8 +48,8 @@ class BaseAppGenerator:
                 tenant_id=app_config.tenant_id,
                 config=FileUploadConfig(
                     allowed_file_types=entity_dictionary[k].allowed_file_types,
-                    allowed_extensions=entity_dictionary[k].allowed_file_extensions,
-                    allowed_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
+                    allowed_file_extensions=entity_dictionary[k].allowed_file_extensions,
+                    allowed_file_upload_methods=entity_dictionary[k].allowed_file_upload_methods,
                 ),
             )
             for k, v in user_inputs.items()
@@ -92,6 +92,9 @@ class BaseAppGenerator:
             )
 
         if variable_entity.type == VariableEntityType.NUMBER and isinstance(value, str):
+            # handle empty string case
+            if not value.strip():
+                return None
             # may raise ValueError if user_input_value is not a valid number
             try:
                 if "." in value:

api/core/app/apps/message_based_app_generator.py

@@ -80,7 +80,7 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             if e.args[0] == "I/O operation on closed file.":  # ignore this error
                 raise GenerateTaskStoppedError()
             else:
-                logger.exception(e)
+                logger.exception(f"Failed to handle response, conversation_id: {conversation.id}")
                 raise e
 
     def _get_conversation_by_user(

api/core/app/apps/workflow/app_generator.py

@@ -310,5 +310,7 @@ class WorkflowAppGenerator(BaseAppGenerator):
             if e.args[0] == "I/O operation on closed file.":  # ignore this error
                 raise GenerateTaskStoppedError()
             else:
-                logger.exception(e)
+                logger.exception(
+                    f"Fails to process generate task pipeline, task_id: {application_generate_entity.task_id}"
+                )
                 raise e

api/core/app/apps/workflow/generate_task_pipeline.py

@@ -216,7 +216,7 @@ class WorkflowAppGenerateTaskPipeline(BasedGenerateTaskPipeline, WorkflowCycleMa
                 else:
                     yield MessageAudioStreamResponse(audio=audio_trunk.audio, task_id=task_id)
             except Exception as e:
-                logger.exception(e)
+                logger.exception(f"Fails to get audio trunk, task_id: {task_id}")
                 break
         if tts_publisher:
             yield MessageAudioEndStreamResponse(audio="", task_id=task_id)

api/core/app/task_pipeline/message_cycle_manage.py

@@ -86,7 +86,7 @@ class MessageCycleManage:
                     conversation.name = name
                 except Exception as e:
                     if dify_config.DEBUG:
-                        logging.exception(f"generate conversation name failed: {e}")
+                        logging.exception(f"generate conversation name failed, conversation_id: {conversation_id}")
                     pass
 
                 db.session.merge(conversation)

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -381,7 +381,7 @@ class WorkflowCycleManage:
                 id=workflow_run.id,
                 workflow_id=workflow_run.workflow_id,
                 sequence_number=workflow_run.sequence_number,
-                inputs=workflow_run.inputs_dict or {},
+                inputs=workflow_run.inputs_dict,
                 created_at=int(workflow_run.created_at.timestamp()),
             ),
         )
@@ -428,7 +428,7 @@ class WorkflowCycleManage:
                 created_by=created_by,
                 created_at=int(workflow_run.created_at.timestamp()),
                 finished_at=int(workflow_run.finished_at.timestamp()),
-                files=self._fetch_files_from_node_outputs(workflow_run.outputs_dict or {}),
+                files=self._fetch_files_from_node_outputs(workflow_run.outputs_dict),
             ),
         )
 

api/core/file/models.py

@@ -28,8 +28,8 @@ class FileUploadConfig(BaseModel):
 
     image_config: Optional[ImageConfig] = None
     allowed_file_types: Sequence[FileType] = Field(default_factory=list)
-    allowed_extensions: Sequence[str] = Field(default_factory=list)
-    allowed_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
+    allowed_file_extensions: Sequence[str] = Field(default_factory=list)
+    allowed_file_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
     number_limits: int = 0
 
 

api/core/helper/moderation.py

@@ -55,7 +55,7 @@ def check_moderation(tenant_id: str, model_config: ModelConfigWithCredentialsEnt
                 if moderation_result is True:
                     return True
             except Exception as ex:
-                logger.exception(ex)
+                logger.exception(f"Fails to check moderation, provider_name: {provider_name}")
                 raise InvokeBadRequestError("Rate limit exceeded, please try again later.")
 
     return False

api/core/helper/module_import_helper.py

@@ -29,7 +29,7 @@ def import_module_from_source(*, module_name: str, py_file_path: AnyStr, use_laz
         spec.loader.exec_module(module)
         return module
     except Exception as e:
-        logging.exception(f"Failed to load module {module_name} from {py_file_path}: {str(e)}")
+        logging.exception(f"Failed to load module {module_name} from script file '{py_file_path}'")
         raise e
 
 

api/core/helper/ssrf_proxy.py

@@ -39,6 +39,7 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
         )
 
     retries = 0
+    stream = kwargs.pop("stream", False)
     while retries <= max_retries:
         try:
             if dify_config.SSRF_PROXY_ALL_URL:
@@ -52,6 +53,8 @@ def make_request(method, url, max_retries=SSRF_DEFAULT_MAX_RETRIES, **kwargs):
                     response = client.request(method=method, url=url, **kwargs)
 
             if response.status_code not in STATUS_FORCELIST:
+                if stream:
+                    return response.iter_bytes()
                 return response
             else:
                 logging.warning(f"Received status code {response.status_code} for URL {url} which is in the force list")

api/core/indexing_runner.py

@@ -29,6 +29,8 @@ from core.rag.splitter.fixed_text_splitter import (
     FixedRecursiveCharacterTextSplitter,
 )
 from core.rag.splitter.text_splitter import TextSplitter
+from core.tools.utils.text_processing_utils import remove_leading_symbols
+from core.tools.utils.web_reader_tool import get_image_upload_file_ids
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from extensions.ext_storage import storage
@@ -278,6 +280,19 @@ class IndexingRunner:
                 if len(preview_texts) < 5:
                     preview_texts.append(document.page_content)
 
+                # delete image files and related db records
+                image_upload_file_ids = get_image_upload_file_ids(document.page_content)
+                for upload_file_id in image_upload_file_ids:
+                    image_file = db.session.query(UploadFile).filter(UploadFile.id == upload_file_id).first()
+                    try:
+                        storage.delete(image_file.key)
+                    except Exception:
+                        logging.exception(
+                            "Delete image_files failed while indexing_estimate, \
+                                          image_upload_file_is: {}".format(upload_file_id)
+                        )
+                    db.session.delete(image_file)
+
         if doc_form and doc_form == "qa_model":
             if len(preview_texts) > 0:
                 # qa model document
@@ -500,11 +515,7 @@ class IndexingRunner:
                     document_node.metadata["doc_hash"] = hash
                     # delete Splitter character
                     page_content = document_node.page_content
-                    if page_content.startswith(".") or page_content.startswith("。"):
-                        page_content = page_content[1:]
-                    else:
-                        page_content = page_content
-                    document_node.page_content = page_content
+                    document_node.page_content = remove_leading_symbols(page_content)
 
                     if document_node.page_content:
                         split_documents.append(document_node)
@@ -554,7 +565,7 @@ class IndexingRunner:
                     qa_documents.append(qa_document)
                 format_documents.extend(qa_documents)
             except Exception as e:
-                logging.exception(e)
+                logging.exception("Failed to format qa document")
 
             all_qa_documents.extend(format_documents)
 

api/core/llm_generator/llm_generator.py

@@ -102,7 +102,7 @@ class LLMGenerator:
         except InvokeError:
             questions = []
         except Exception as e:
-            logging.exception(e)
+            logging.exception("Failed to generate suggested questions after answer")
             questions = []
 
         return questions
@@ -148,7 +148,7 @@ class LLMGenerator:
                 error = str(e)
                 error_step = "generate rule config"
             except Exception as e:
-                logging.exception(e)
+                logging.exception(f"Failed to generate rule config, model: {model_config.get('name')}")
                 rule_config["error"] = str(e)
 
             rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
@@ -234,7 +234,7 @@ class LLMGenerator:
                 error_step = "generate conversation opener"
 
         except Exception as e:
-            logging.exception(e)
+            logging.exception(f"Failed to generate rule config, model: {model_config.get('name')}")
             rule_config["error"] = str(e)
 
         rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
@@ -286,7 +286,9 @@ class LLMGenerator:
             error = str(e)
             return {"code": "", "language": code_language, "error": f"Failed to generate code. Error: {error}"}
         except Exception as e:
-            logging.exception(e)
+            logging.exception(
+                f"Failed to invoke LLM model, model: {model_config.get('name')}, language: {code_language}"
+            )
             return {"code": "", "language": code_language, "error": f"An unexpected error occurred: {str(e)}"}
 
     @classmethod

api/core/model_runtime/model_providers/gitee_ai/_assets/Gitee-AI-Logo.svg

@@ -1,3 +0,0 @@
-<svg width="40" height="40" viewBox="0 0 40 40" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M25.132 24.3947C25.497 25.7527 25.8984 27.1413 26.3334 28.5834C26.7302 29.8992 25.5459 30.4167 25.0752 29.1758C24.571 27.8466 24.0885 26.523 23.6347 25.1729C21.065 26.4654 18.5025 27.5424 15.5961 28.7541C16.7581 33.0256 17.8309 36.5984 19.4952 39.9935C19.4953 39.9936 19.4953 39.9937 19.4954 39.9938C19.6631 39.9979 19.8313 40 20 40C31.0457 40 40 31.0457 40 20C40 16.0335 38.8453 12.3366 36.8537 9.22729C31.6585 9.69534 27.0513 10.4562 22.8185 11.406C22.8882 12.252 22.9677 13.0739 23.0555 13.855C23.3824 16.7604 23.9112 19.5281 24.6137 22.3836C27.0581 21.2848 29.084 20.3225 30.6816 19.522C32.2154 18.7535 33.6943 18.7062 31.2018 20.6594C29.0388 22.1602 27.0644 23.3566 25.132 24.3947ZM36.1559 8.20846C33.0001 3.89184 28.1561 0.887462 22.5955 0.166882C22.4257 2.86234 22.4785 6.26344 22.681 9.50447C26.7473 8.88859 31.1721 8.46032 36.1559 8.20846ZM19.9369 9.73661e-05C19.7594 2.92694 19.8384 6.65663 20.19 9.91293C17.3748 10.4109 14.7225 11.0064 12.1592 11.7038C12.0486 10.4257 11.9927 9.25764 11.9927 8.24178C11.9927 7.5054 11.3957 6.90844 10.6593 6.90844C9.92296 6.90844 9.32601 7.5054 9.32601 8.24178C9.32601 9.47868 9.42873 10.898 9.61402 12.438C8.33567 12.8278 7.07397 13.2443 5.81918 13.688C5.12493 13.9336 4.76118 14.6954 5.0067 15.3896C5.25223 16.0839 6.01406 16.4476 6.7083 16.2021C7.7931 15.8185 8.88482 15.4388 9.98927 15.0659C10.5222 18.3344 11.3344 21.9428 12.2703 25.4156C12.4336 26.0218 12.6062 26.6262 12.7863 27.2263C9.34168 28.4135 5.82612 29.3782 2.61128 29.8879C0.949407 26.9716 0 23.5967 0 20C0 8.97534 8.92023 0.0341108 19.9369 9.73661e-05ZM4.19152 32.2527C7.45069 36.4516 12.3458 39.3173 17.9204 39.8932C16.5916 37.455 14.9338 33.717 13.5405 29.5901C10.4404 30.7762 7.25883 31.6027 4.19152 32.2527ZM22.9735 23.1135C22.1479 20.41 21.4462 17.5441 20.9225 14.277C20.746 13.5841 20.5918 12.8035 20.4593 11.9636C17.6508 12.6606 14.9992 13.4372 12.4356 14.2598C12.8479 17.4766 13.5448 21.1334 14.5118 24.7218C14.662 25.2792 14.8081 25.8248 14.9514 26.3594L14.9516 26.3603L14.9524 26.3634L14.9526 26.3639L14.973 26.4401C16.1833 25.9872 17.3746 25.5123 18.53 25.0259C20.1235 24.3552 21.6051 23.7165 22.9735 23.1135Z" fill="#141519"/>
-</svg>

api/core/model_runtime/model_providers/gitee_ai/_common.py

@@ -1,47 +0,0 @@
-from dashscope.common.error import (
-    AuthenticationError,
-    InvalidParameter,
-    RequestFailure,
-    ServiceUnavailableError,
-    UnsupportedHTTPMethod,
-    UnsupportedModel,
-)
-
-from core.model_runtime.errors.invoke import (
-    InvokeAuthorizationError,
-    InvokeBadRequestError,
-    InvokeConnectionError,
-    InvokeError,
-    InvokeRateLimitError,
-    InvokeServerUnavailableError,
-)
-
-
-class _CommonGiteeAI:
-    @property
-    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
-        """
-        Map model invoke error to unified error
-        The key is the error type thrown to the caller
-        The value is the error type thrown by the model,
-        which needs to be converted into a unified error type for the caller.
-
-        :return: Invoke error mapping
-        """
-        return {
-            InvokeConnectionError: [
-                RequestFailure,
-            ],
-            InvokeServerUnavailableError: [
-                ServiceUnavailableError,
-            ],
-            InvokeRateLimitError: [],
-            InvokeAuthorizationError: [
-                AuthenticationError,
-            ],
-            InvokeBadRequestError: [
-                InvalidParameter,
-                UnsupportedModel,
-                UnsupportedHTTPMethod,
-            ],
-        }

api/core/model_runtime/model_providers/gitee_ai/gitee_ai.py

@@ -1,36 +0,0 @@
-import logging
-
-import requests
-
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.__base.model_provider import ModelProvider
-
-logger = logging.getLogger(__name__)
-
-
-class GiteeAIProvider(ModelProvider):
-    def validate_provider_credentials(self, credentials: dict) -> None:
-        """
-        Validate provider credentials
-        if validate failed, raise exception
-
-        :param credentials: provider credentials, credentials form defined in `provider_credential_schema`.
-        """
-        try:
-            api_key = credentials.get("api_key")
-            if not api_key:
-                raise CredentialsValidateFailedError("Credentials validation failed: api_key not given")
-
-            # send a get request to validate the credentials
-            headers = {"Authorization": f"Bearer {api_key}"}
-            response = requests.get("https://ai.gitee.com/api/base/account/me", headers=headers, timeout=(10, 300))
-
-            if response.status_code != 200:
-                raise CredentialsValidateFailedError(
-                    f"Credentials validation failed with status code {response.status_code}"
-                )
-        except CredentialsValidateFailedError as ex:
-            raise ex
-        except Exception as ex:
-            logger.exception(f"{self.get_provider_schema().provider} credentials validate failed")
-            raise ex

api/core/model_runtime/model_providers/gitee_ai/gitee_ai.yaml

@@ -1,35 +0,0 @@
-provider: gitee_ai
-label:
-  en_US: Gitee AI
-  zh_Hans: Gitee AI
-description:
-  en_US: 快速体验大模型，领先探索 AI 开源世界
-  zh_Hans: 快速体验大模型，领先探索 AI 开源世界
-icon_small:
-  en_US: Gitee-AI-Logo.svg
-icon_large:
-  en_US: Gitee-AI-Logo-full.svg
-help:
-  title:
-    en_US: Get your token from Gitee AI
-    zh_Hans: 从 Gitee AI 获取 token
-  url:
-    en_US: https://ai.gitee.com/dashboard/settings/tokens
-supported_model_types:
-  - llm
-  - text-embedding
-  - rerank
-  - speech2text
-  - tts
-configurate_methods:
-  - predefined-model
-provider_credential_schema:
-  credential_form_schemas:
-    - variable: api_key
-      label:
-        en_US: API Key
-      type: secret-input
-      required: true
-      placeholder:
-        zh_Hans: 在此输入您的 API Key
-        en_US: Enter your API Key

api/core/model_runtime/model_providers/gitee_ai/llm/Qwen2-72B-Instruct.yaml

@@ -1,105 +0,0 @@
-model: Qwen2-72B-Instruct
-label:
-  zh_Hans: Qwen2-72B-Instruct
-  en_US: Qwen2-72B-Instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 6400
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/Qwen2-7B-Instruct.yaml

@@ -1,105 +0,0 @@
-model: Qwen2-7B-Instruct
-label:
-  zh_Hans: Qwen2-7B-Instruct
-  en_US: Qwen2-7B-Instruct
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 32768
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/Qwen2.5-72B-Instruct.yaml

@@ -1,95 +0,0 @@
-model: Qwen2.5-72B-Instruct
-label:
-  zh_Hans: Qwen2.5-72B-Instruct
-  en_US: Qwen2.5-72B-Instruct
-model_type: llm
-features:
-  - agent-thought
-  - tool-call
-  - stream-tool-call
-model_properties:
-  mode: chat
-  context_size: 32768
-parameter_rules:
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/Yi-1.5-34B-Chat.yaml

@@ -1,105 +0,0 @@
-model: Yi-1.5-34B-Chat
-label:
-  zh_Hans: Yi-1.5-34B-Chat
-  en_US: Yi-1.5-34B-Chat
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 4096
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/_position.yaml

@@ -1,8 +0,0 @@
-- Qwen2.5-72B-Instruct
-- Qwen2-7B-Instruct
-- Qwen2-72B-Instruct
-- Yi-1.5-34B-Chat
-- glm-4-9b-chat
-- deepseek-coder-33B-instruct-chat
-- deepseek-coder-33B-instruct-completions
-- codegeex4-all-9b

api/core/model_runtime/model_providers/gitee_ai/llm/codegeex4-all-9b.yaml

@@ -1,105 +0,0 @@
-model: codegeex4-all-9b
-label:
-  zh_Hans: codegeex4-all-9b
-  en_US: codegeex4-all-9b
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 40960
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/deepseek-coder-33B-instruct-chat.yaml

@@ -1,105 +0,0 @@
-model: deepseek-coder-33B-instruct-chat
-label:
-  zh_Hans: deepseek-coder-33B-instruct-chat
-  en_US: deepseek-coder-33B-instruct-chat
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 9000
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/deepseek-coder-33B-instruct-completions.yaml

@@ -1,91 +0,0 @@
-model: deepseek-coder-33B-instruct-completions
-label:
-  zh_Hans: deepseek-coder-33B-instruct-completions
-  en_US: deepseek-coder-33B-instruct-completions
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: completion
-  context_size: 9000
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/glm-4-9b-chat.yaml

@@ -1,105 +0,0 @@
-model: glm-4-9b-chat
-label:
-  zh_Hans: glm-4-9b-chat
-  en_US: glm-4-9b-chat
-model_type: llm
-features:
-  - agent-thought
-model_properties:
-  mode: chat
-  context_size: 32768
-parameter_rules:
-  - name: stream
-    use_template: boolean
-    label:
-      en_US: "Stream"
-      zh_Hans: "流式"
-    type: boolean
-    default: true
-    required: true
-    help:
-      en_US: "Whether to return the results in batches through streaming. If set to true, the generated text will be pushed to the user in real time during the generation process."
-      zh_Hans: "是否通过流式分批返回结果。如果设置为 true，生成过程中实时地向用户推送每一部分生成的文本。"
-
-  - name: max_tokens
-    use_template: max_tokens
-    label:
-      en_US: "Max Tokens"
-      zh_Hans: "最大Token数"
-    type: int
-    default: 512
-    min: 1
-    required: true
-    help:
-      en_US: "The maximum number of tokens that can be generated by the model varies depending on the model."
-      zh_Hans: "模型可生成的最大 token 个数，不同模型上限不同。"
-
-  - name: temperature
-    use_template: temperature
-    label:
-      en_US: "Temperature"
-      zh_Hans: "采样温度"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The randomness of the sampling temperature control output. The temperature value is within the range of [0.0, 1.0]. The higher the value, the more random and creative the output; the lower the value, the more stable it is. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样温度控制输出的随机性。温度值在 [0.0, 1.0] 范围内，值越高，输出越随机和创造性；值越低，输出越稳定。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_p
-    use_template: top_p
-    label:
-      en_US: "Top P"
-      zh_Hans: "Top P"
-    type: float
-    default: 0.7
-    min: 0.0
-    max: 1.0
-    precision: 1
-    required: true
-    help:
-      en_US: "The value range of the sampling method is [0.0, 1.0]. The top_p value determines that the model selects tokens from the top p% of candidate words with the highest probability; when top_p is 0, this parameter is invalid. It is recommended to adjust either top_p or temperature parameters according to your needs to avoid adjusting both at the same time."
-      zh_Hans: "采样方法的取值范围为 [0.0,1.0]。top_p 值确定模型从概率最高的前p%的候选词中选取 tokens；当 top_p 为 0 时，此参数无效。建议根据需求调整 top_p 或 temperature 参数，避免同时调整两者。"
-
-  - name: top_k
-    use_template: top_k
-    label:
-      en_US: "Top K"
-      zh_Hans: "Top K"
-    type: int
-    default: 50
-    min: 0
-    max: 100
-    required: true
-    help:
-      en_US: "The value range is [0,100], which limits the model to only select from the top k words with the highest probability when choosing the next word at each step. The larger the value, the more diverse text generation will be."
-      zh_Hans: "取值范围为 [0,100]，限制模型在每一步选择下一个词时，只从概率最高的前 k 个词中选取。数值越大，文本生成越多样。"
-
-  - name: frequency_penalty
-    use_template: frequency_penalty
-    label:
-      en_US: "Frequency Penalty"
-      zh_Hans: "频率惩罚"
-    type: float
-    default: 0
-    min: -1.0
-    max: 1.0
-    precision: 1
-    required: false
-    help:
-      en_US: "Used to adjust the frequency of repeated content in automatically generated text. Positive numbers reduce repetition, while negative numbers increase repetition. After setting this parameter, if a word has already appeared in the text, the model will decrease the probability of choosing that word for subsequent generation."
-      zh_Hans: "用于调整自动生成文本中重复内容的频率。正数减少重复，负数增加重复。设置此参数后，如果一个词在文本中已经出现过，模型在后续生成中选择该词的概率会降低。"
-
-  - name: user
-    use_template: text
-    label:
-      en_US: "User"
-      zh_Hans: "用户"
-    type: string
-    required: false
-    help:
-      en_US: "Used to track and differentiate conversation requests from different users."
-      zh_Hans: "用于追踪和区分不同用户的对话请求。"

api/core/model_runtime/model_providers/gitee_ai/llm/llm.py

@@ -1,51 +0,0 @@
-from collections.abc import Generator
-from typing import Optional, Union
-
-from core.model_runtime.entities.llm_entities import LLMMode, LLMResult
-from core.model_runtime.entities.message_entities import (
-    PromptMessage,
-    PromptMessageTool,
-)
-from core.model_runtime.entities.model_entities import ModelFeature
-from core.model_runtime.model_providers.openai_api_compatible.llm.llm import OAIAPICompatLargeLanguageModel
-
-
-class GiteeAILargeLanguageModel(OAIAPICompatLargeLanguageModel):
-    MODEL_TO_IDENTITY: dict[str, str] = {
-        "Yi-1.5-34B-Chat": "Yi-34B-Chat",
-        "deepseek-coder-33B-instruct-completions": "deepseek-coder-33B-instruct",
-        "deepseek-coder-33B-instruct-chat": "deepseek-coder-33B-instruct",
-    }
-
-    def _invoke(
-        self,
-        model: str,
-        credentials: dict,
-        prompt_messages: list[PromptMessage],
-        model_parameters: dict,
-        tools: Optional[list[PromptMessageTool]] = None,
-        stop: Optional[list[str]] = None,
-        stream: bool = True,
-        user: Optional[str] = None,
-    ) -> Union[LLMResult, Generator]:
-        self._add_custom_parameters(credentials, model, model_parameters)
-        return super()._invoke(model, credentials, prompt_messages, model_parameters, tools, stop, stream, user)
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        self._add_custom_parameters(credentials, model, None)
-        super().validate_credentials(model, credentials)
-
-    def _add_custom_parameters(self, credentials: dict, model: str, model_parameters: dict) -> None:
-        if model is None:
-            model = "bge-large-zh-v1.5"
-
-        model_identity = GiteeAILargeLanguageModel.MODEL_TO_IDENTITY.get(model, model)
-        credentials["endpoint_url"] = f"https://ai.gitee.com/api/serverless/{model_identity}/"
-        if model.endswith("completions"):
-            credentials["mode"] = LLMMode.COMPLETION.value
-        else:
-            credentials["mode"] = LLMMode.CHAT.value
-
-        schema = self.get_model_schema(model, credentials)
-        if ModelFeature.TOOL_CALL in schema.features or ModelFeature.MULTI_TOOL_CALL in schema.features:
-            credentials["function_calling_type"] = "tool_call"

api/core/model_runtime/model_providers/gitee_ai/rerank/_position.yaml

@@ -1 +0,0 @@
-- bge-reranker-v2-m3

api/core/model_runtime/model_providers/gitee_ai/rerank/bge-reranker-v2-m3.yaml

@@ -1,4 +0,0 @@
-model: bge-reranker-v2-m3
-model_type: rerank
-model_properties:
-  context_size: 1024

api/core/model_runtime/model_providers/gitee_ai/rerank/rerank.py

@@ -1,128 +0,0 @@
-from typing import Optional
-
-import httpx
-
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelPropertyKey, ModelType
-from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult
-from core.model_runtime.errors.invoke import (
-    InvokeAuthorizationError,
-    InvokeBadRequestError,
-    InvokeConnectionError,
-    InvokeError,
-    InvokeRateLimitError,
-    InvokeServerUnavailableError,
-)
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.__base.rerank_model import RerankModel
-
-
-class GiteeAIRerankModel(RerankModel):
-    """
-    Model class for rerank model.
-    """
-
-    def _invoke(
-        self,
-        model: str,
-        credentials: dict,
-        query: str,
-        docs: list[str],
-        score_threshold: Optional[float] = None,
-        top_n: Optional[int] = None,
-        user: Optional[str] = None,
-    ) -> RerankResult:
-        """
-        Invoke rerank model
-
-        :param model: model name
-        :param credentials: model credentials
-        :param query: search query
-        :param docs: docs for reranking
-        :param score_threshold: score threshold
-        :param top_n: top n documents to return
-        :param user: unique user id
-        :return: rerank result
-        """
-        if len(docs) == 0:
-            return RerankResult(model=model, docs=[])
-
-        base_url = credentials.get("base_url", "https://ai.gitee.com/api/serverless")
-        base_url = base_url.removesuffix("/")
-
-        try:
-            body = {"model": model, "query": query, "documents": docs}
-            if top_n is not None:
-                body["top_n"] = top_n
-            response = httpx.post(
-                f"{base_url}/{model}/rerank",
-                json=body,
-                headers={"Authorization": f"Bearer {credentials.get('api_key')}"},
-            )
-
-            response.raise_for_status()
-            results = response.json()
-
-            rerank_documents = []
-            for result in results["results"]:
-                rerank_document = RerankDocument(
-                    index=result["index"],
-                    text=result["document"]["text"],
-                    score=result["relevance_score"],
-                )
-                if score_threshold is None or result["relevance_score"] >= score_threshold:
-                    rerank_documents.append(rerank_document)
-            return RerankResult(model=model, docs=rerank_documents)
-        except httpx.HTTPStatusError as e:
-            raise InvokeServerUnavailableError(str(e))
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        """
-        Validate model credentials
-
-        :param model: model name
-        :param credentials: model credentials
-        :return:
-        """
-        try:
-            self._invoke(
-                model=model,
-                credentials=credentials,
-                query="What is the capital of the United States?",
-                docs=[
-                    "Carson City is the capital city of the American state of Nevada. At the 2010 United States "
-                    "Census, Carson City had a population of 55,274.",
-                    "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that "
-                    "are a political division controlled by the United States. Its capital is Saipan.",
-                ],
-                score_threshold=0.01,
-            )
-        except Exception as ex:
-            raise CredentialsValidateFailedError(str(ex))
-
-    @property
-    def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]:
-        """
-        Map model invoke error to unified error
-        """
-        return {
-            InvokeConnectionError: [httpx.ConnectError],
-            InvokeServerUnavailableError: [httpx.RemoteProtocolError],
-            InvokeRateLimitError: [],
-            InvokeAuthorizationError: [httpx.HTTPStatusError],
-            InvokeBadRequestError: [httpx.RequestError],
-        }
-
-    def get_customizable_model_schema(self, model: str, credentials: dict) -> AIModelEntity:
-        """
-        generate custom model entities from credentials
-        """
-        entity = AIModelEntity(
-            model=model,
-            label=I18nObject(en_US=model),
-            model_type=ModelType.RERANK,
-            fetch_from=FetchFrom.CUSTOMIZABLE_MODEL,
-            model_properties={ModelPropertyKey.CONTEXT_SIZE: int(credentials.get("context_size"))},
-        )
-
-        return entity

api/core/model_runtime/model_providers/gitee_ai/speech2text/_position.yaml

@@ -1,2 +0,0 @@
-- whisper-base
-- whisper-large

api/core/model_runtime/model_providers/gitee_ai/speech2text/speech2text.py

@@ -1,53 +0,0 @@
-import os
-from typing import IO, Optional
-
-import requests
-
-from core.model_runtime.errors.invoke import InvokeBadRequestError
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.__base.speech2text_model import Speech2TextModel
-from core.model_runtime.model_providers.gitee_ai._common import _CommonGiteeAI
-
-
-class GiteeAISpeech2TextModel(_CommonGiteeAI, Speech2TextModel):
-    """
-    Model class for OpenAI Compatible Speech to text model.
-    """
-
-    def _invoke(self, model: str, credentials: dict, file: IO[bytes], user: Optional[str] = None) -> str:
-        """
-        Invoke speech2text model
-
-        :param model: model name
-        :param credentials: model credentials
-        :param file: audio file
-        :param user: unique user id
-        :return: text for given audio file
-        """
-        # doc: https://ai.gitee.com/docs/openapi/serverless#tag/serverless/POST/{service}/speech-to-text
-
-        endpoint_url = f"https://ai.gitee.com/api/serverless/{model}/speech-to-text"
-        files = [("file", file)]
-        _, file_ext = os.path.splitext(file.name)
-        headers = {"Content-Type": f"audio/{file_ext}", "Authorization": f"Bearer {credentials.get('api_key')}"}
-        response = requests.post(endpoint_url, headers=headers, files=files)
-        if response.status_code != 200:
-            raise InvokeBadRequestError(response.text)
-        response_data = response.json()
-        return response_data["text"]
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        """
-        Validate model credentials
-
-        :param model: model name
-        :param credentials: model credentials
-        :return:
-        """
-        try:
-            audio_file_path = self._get_demo_file_path()
-
-            with open(audio_file_path, "rb") as audio_file:
-                self._invoke(model, credentials, audio_file)
-        except Exception as ex:
-            raise CredentialsValidateFailedError(str(ex))

api/core/model_runtime/model_providers/gitee_ai/speech2text/whisper-base.yaml

@@ -1,5 +0,0 @@
-model: whisper-base
-model_type: speech2text
-model_properties:
-  file_upload_limit: 1
-  supported_file_extensions: flac,mp3,mp4,mpeg,mpga,m4a,ogg,wav,webm

api/core/model_runtime/model_providers/gitee_ai/speech2text/whisper-large.yaml

@@ -1,5 +0,0 @@
-model: whisper-large
-model_type: speech2text
-model_properties:
-  file_upload_limit: 1
-  supported_file_extensions: flac,mp3,mp4,mpeg,mpga,m4a,ogg,wav,webm

api/core/model_runtime/model_providers/gitee_ai/text_embedding/_position.yaml

@@ -1,3 +0,0 @@
-- bge-large-zh-v1.5
-- bge-small-zh-v1.5
-- bge-m3

api/core/model_runtime/model_providers/gitee_ai/text_embedding/bge-large-zh-v1.5.yaml

@@ -1,8 +0,0 @@
-model: bge-large-zh-v1.5
-label:
-  zh_Hans: bge-large-zh-v1.5
-  en_US: bge-large-zh-v1.5
-model_type: text-embedding
-model_properties:
-  context_size: 200000
-  max_chunks: 20

api/core/model_runtime/model_providers/gitee_ai/text_embedding/bge-m3.yaml

@@ -1,8 +0,0 @@
-model: bge-m3
-label:
-  zh_Hans: bge-m3
-  en_US: bge-m3
-model_type: text-embedding
-model_properties:
-  context_size: 200000
-  max_chunks: 20

api/core/model_runtime/model_providers/gitee_ai/text_embedding/bge-small-zh-v1.5.yaml

@@ -1,8 +0,0 @@
-model: bge-small-zh-v1.5
-label:
-  zh_Hans: bge-small-zh-v1.5
-  en_US: bge-small-zh-v1.5
-model_type: text-embedding
-model_properties:
-  context_size: 200000
-  max_chunks: 20

api/core/model_runtime/model_providers/gitee_ai/text_embedding/text_embedding.py

@@ -1,31 +0,0 @@
-from typing import Optional
-
-from core.entities.embedding_type import EmbeddingInputType
-from core.model_runtime.entities.text_embedding_entities import TextEmbeddingResult
-from core.model_runtime.model_providers.openai_api_compatible.text_embedding.text_embedding import (
-    OAICompatEmbeddingModel,
-)
-
-
-class GiteeAIEmbeddingModel(OAICompatEmbeddingModel):
-    def _invoke(
-        self,
-        model: str,
-        credentials: dict,
-        texts: list[str],
-        user: Optional[str] = None,
-        input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT,
-    ) -> TextEmbeddingResult:
-        self._add_custom_parameters(credentials, model)
-        return super()._invoke(model, credentials, texts, user, input_type)
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        self._add_custom_parameters(credentials, None)
-        super().validate_credentials(model, credentials)
-
-    @staticmethod
-    def _add_custom_parameters(credentials: dict, model: str) -> None:
-        if model is None:
-            model = "bge-m3"
-
-        credentials["endpoint_url"] = f"https://ai.gitee.com/api/serverless/{model}/v1/"

api/core/model_runtime/model_providers/gitee_ai/tts/ChatTTS.yaml

@@ -1,11 +0,0 @@
-model: ChatTTS
-model_type: tts
-model_properties:
-  default_voice: 'default'
-  voices:
-    - mode: 'default'
-      name: 'Default'
-      language: [ 'zh-Hans', 'en-US', 'de-DE', 'fr-FR', 'es-ES', 'it-IT', 'th-TH', 'id-ID' ]
-  word_limit: 3500
-  audio_type: 'mp3'
-  max_workers: 5

api/core/model_runtime/model_providers/gitee_ai/tts/FunAudioLLM-CosyVoice-300M.yaml

@@ -1,11 +0,0 @@
-model: FunAudioLLM-CosyVoice-300M
-model_type: tts
-model_properties:
-  default_voice: 'default'
-  voices:
-    - mode: 'default'
-      name: 'Default'
-      language: [ 'zh-Hans', 'en-US', 'de-DE', 'fr-FR', 'es-ES', 'it-IT', 'th-TH', 'id-ID' ]
-  word_limit: 3500
-  audio_type: 'mp3'
-  max_workers: 5

api/core/model_runtime/model_providers/gitee_ai/tts/_position.yaml

@@ -1,4 +0,0 @@
-- speecht5_tts
-- ChatTTS
-- fish-speech-1.2-sft
-- FunAudioLLM-CosyVoice-300M

api/core/model_runtime/model_providers/gitee_ai/tts/fish-speech-1.2-sft.yaml

@@ -1,11 +0,0 @@
-model: fish-speech-1.2-sft
-model_type: tts
-model_properties:
-  default_voice: 'default'
-  voices:
-    - mode: 'default'
-      name: 'Default'
-      language: [ 'zh-Hans', 'en-US', 'de-DE', 'fr-FR', 'es-ES', 'it-IT', 'th-TH', 'id-ID' ]
-  word_limit: 3500
-  audio_type: 'mp3'
-  max_workers: 5

api/core/model_runtime/model_providers/gitee_ai/tts/speecht5_tts.yaml

@@ -1,11 +0,0 @@
-model: speecht5_tts
-model_type: tts
-model_properties:
-  default_voice: 'default'
-  voices:
-    - mode: 'default'
-      name: 'Default'
-      language: [ 'zh-Hans', 'en-US', 'de-DE', 'fr-FR', 'es-ES', 'it-IT', 'th-TH', 'id-ID' ]
-  word_limit: 3500
-  audio_type: 'mp3'
-  max_workers: 5

api/core/model_runtime/model_providers/gitee_ai/tts/tts.py

@@ -1,79 +0,0 @@
-from typing import Optional
-
-import requests
-
-from core.model_runtime.errors.invoke import InvokeBadRequestError
-from core.model_runtime.errors.validate import CredentialsValidateFailedError
-from core.model_runtime.model_providers.__base.tts_model import TTSModel
-from core.model_runtime.model_providers.gitee_ai._common import _CommonGiteeAI
-
-
-class GiteeAIText2SpeechModel(_CommonGiteeAI, TTSModel):
-    """
-    Model class for OpenAI Speech to text model.
-    """
-
-    def _invoke(
-        self, model: str, tenant_id: str, credentials: dict, content_text: str, voice: str, user: Optional[str] = None
-    ) -> any:
-        """
-        _invoke text2speech model
-
-        :param model: model name
-        :param tenant_id: user tenant id
-        :param credentials: model credentials
-        :param content_text: text content to be translated
-        :param voice: model timbre
-        :param user: unique user id
-        :return: text translated to audio file
-        """
-        return self._tts_invoke_streaming(model=model, credentials=credentials, content_text=content_text, voice=voice)
-
-    def validate_credentials(self, model: str, credentials: dict) -> None:
-        """
-        validate credentials text2speech model
-
-        :param model: model name
-        :param credentials: model credentials
-        :return: text translated to audio file
-        """
-        try:
-            self._tts_invoke_streaming(
-                model=model,
-                credentials=credentials,
-                content_text="Hello Dify!",
-                voice=self._get_model_default_voice(model, credentials),
-            )
-        except Exception as ex:
-            raise CredentialsValidateFailedError(str(ex))
-
-    def _tts_invoke_streaming(self, model: str, credentials: dict, content_text: str, voice: str) -> any:
-        """
-        _tts_invoke_streaming text2speech model
-        :param model: model name
-        :param credentials: model credentials
-        :param content_text: text content to be translated
-        :param voice: model timbre
-        :return: text translated to audio file
-        """
-        try:
-            # doc: https://ai.gitee.com/docs/openapi/serverless#tag/serverless/POST/{service}/text-to-speech
-            endpoint_url = "https://ai.gitee.com/api/serverless/" + model + "/text-to-speech"
-
-            headers = {"Content-Type": "application/json"}
-            api_key = credentials.get("api_key")
-            if api_key:
-                headers["Authorization"] = f"Bearer {api_key}"
-
-            payload = {"inputs": content_text}
-            response = requests.post(endpoint_url, headers=headers, json=payload)
-
-            if response.status_code != 200:
-                raise InvokeBadRequestError(response.text)
-
-            data = response.content
-
-            for i in range(0, len(data), 1024):
-                yield data[i : i + 1024]
-        except Exception as ex:
-            raise InvokeBadRequestError(str(ex))

api/core/moderation/keywords/keywords.py

@@ -1,3 +1,6 @@
+from collections.abc import Sequence
+from typing import Any
+
 from core.moderation.base import Moderation, ModerationAction, ModerationInputsResult, ModerationOutputsResult
 
 
@@ -62,5 +65,5 @@ class KeywordsModeration(Moderation):
     def _is_violated(self, inputs: dict, keywords_list: list) -> bool:
         return any(self._check_keywords_in_value(keywords_list, value) for value in inputs.values())
 
-    def _check_keywords_in_value(self, keywords_list, value) -> bool:
-        return any(keyword.lower() in value.lower() for keyword in keywords_list)
+    def _check_keywords_in_value(self, keywords_list: Sequence[str], value: Any) -> bool:
+        return any(keyword.lower() in str(value).lower() for keyword in keywords_list)

api/core/moderation/output_moderation.py

@@ -126,6 +126,6 @@ class OutputModeration(BaseModel):
             result: ModerationOutputsResult = moderation_factory.moderation_for_outputs(moderation_buffer)
             return result
         except Exception as e:
-            logger.exception("Moderation Output error: %s", e)
+            logger.exception(f"Moderation Output error, app_id: {app_id}")
 
         return None

api/core/ops/langsmith_trace/entities/langsmith_trace_entity.py

@@ -49,6 +49,7 @@ class LangSmithRunModel(LangSmithTokenUsage, LangSmithMultiModel):
     reference_example_id: Optional[str] = Field(None, description="Reference example ID associated with the run")
     input_attachments: Optional[dict[str, Any]] = Field(None, description="Input attachments of the run")
     output_attachments: Optional[dict[str, Any]] = Field(None, description="Output attachments of the run")
+    dotted_order: Optional[str] = Field(None, description="Dotted order of the run")
 
     @field_validator("inputs", "outputs")
     @classmethod

api/core/ops/langsmith_trace/langsmith_trace.py

@@ -25,7 +25,7 @@ from core.ops.langsmith_trace.entities.langsmith_trace_entity import (
     LangSmithRunType,
     LangSmithRunUpdateModel,
 )
-from core.ops.utils import filter_none_values
+from core.ops.utils import filter_none_values, generate_dotted_order
 from extensions.ext_database import db
 from models.model import EndUser, MessageFile
 from models.workflow import WorkflowNodeExecution
@@ -62,6 +62,16 @@ class LangSmithDataTrace(BaseTraceInstance):
             self.generate_name_trace(trace_info)
 
     def workflow_trace(self, trace_info: WorkflowTraceInfo):
+        trace_id = trace_info.message_id or trace_info.workflow_app_log_id or trace_info.workflow_run_id
+        message_dotted_order = (
+            generate_dotted_order(trace_info.message_id, trace_info.start_time) if trace_info.message_id else None
+        )
+        workflow_dotted_order = generate_dotted_order(
+            trace_info.workflow_app_log_id or trace_info.workflow_run_id,
+            trace_info.workflow_data.created_at,
+            message_dotted_order,
+        )
+
         if trace_info.message_id:
             message_run = LangSmithRunModel(
                 id=trace_info.message_id,
@@ -76,6 +86,8 @@ class LangSmithDataTrace(BaseTraceInstance):
                 },
                 tags=["message", "workflow"],
                 error=trace_info.error,
+                trace_id=trace_id,
+                dotted_order=message_dotted_order,
             )
             self.add_run(message_run)
 
@@ -95,6 +107,8 @@ class LangSmithDataTrace(BaseTraceInstance):
             error=trace_info.error,
             tags=["workflow"],
             parent_run_id=trace_info.message_id or None,
+            trace_id=trace_id,
+            dotted_order=workflow_dotted_order,
         )
 
         self.add_run(langsmith_run)
@@ -177,6 +191,7 @@ class LangSmithDataTrace(BaseTraceInstance):
             else:
                 run_type = LangSmithRunType.tool
 
+            node_dotted_order = generate_dotted_order(node_execution_id, created_at, workflow_dotted_order)
             langsmith_run = LangSmithRunModel(
                 total_tokens=node_total_tokens,
                 name=node_type,
@@ -191,6 +206,9 @@ class LangSmithDataTrace(BaseTraceInstance):
                 },
                 parent_run_id=trace_info.workflow_app_log_id or trace_info.workflow_run_id,
                 tags=["node_execution"],
+                id=node_execution_id,
+                trace_id=trace_id,
+                dotted_order=node_dotted_order,
             )
 
             self.add_run(langsmith_run)

api/core/ops/ops_trace_manager.py

@@ -711,7 +711,7 @@ class TraceQueueManager:
                 trace_task.app_id = self.app_id
                 trace_manager_queue.put(trace_task)
         except Exception as e:
-            logging.exception(f"Error adding trace task: {e}")
+            logging.exception(f"Error adding trace task, trace_type {trace_task.trace_type}")
         finally:
             self.start_timer()
 
@@ -730,7 +730,7 @@ class TraceQueueManager:
             if tasks:
                 self.send_to_celery(tasks)
         except Exception as e:
-            logging.exception(f"Error processing trace tasks: {e}")
+            logging.exception("Error processing trace tasks")
 
     def start_timer(self):
         global trace_manager_timer

api/core/ops/utils.py

@@ -1,5 +1,6 @@
 from contextlib import contextmanager
 from datetime import datetime
+from typing import Optional, Union
 
 from extensions.ext_database import db
 from models.model import Message
@@ -43,3 +44,19 @@ def replace_text_with_content(data):
         return [replace_text_with_content(item) for item in data]
     else:
         return data
+
+
+def generate_dotted_order(
+    run_id: str, start_time: Union[str, datetime], parent_dotted_order: Optional[str] = None
+) -> str:
+    """
+    generate dotted_order for langsmith
+    """
+    start_time = datetime.fromisoformat(start_time) if isinstance(start_time, str) else start_time
+    timestamp = start_time.strftime("%Y%m%dT%H%M%S%f")[:-3] + "Z"
+    current_segment = f"{timestamp}{run_id}"
+
+    if parent_dotted_order is None:
+        return current_segment
+
+    return f"{parent_dotted_order}.{current_segment}"

api/core/rag/datasource/vdb/analyticdb/analyticdb_vector.py

@@ -1,310 +1,62 @@
 import json
 from typing import Any
 
-from pydantic import BaseModel
-
-_import_err_msg = (
-    "`alibabacloud_gpdb20160503` and `alibabacloud_tea_openapi` packages not found, "
-    "please run `pip install alibabacloud_gpdb20160503 alibabacloud_tea_openapi`"
-)
-
 from configs import dify_config
+from core.rag.datasource.vdb.analyticdb.analyticdb_vector_openapi import (
+    AnalyticdbVectorOpenAPI,
+    AnalyticdbVectorOpenAPIConfig,
+)
+from core.rag.datasource.vdb.analyticdb.analyticdb_vector_sql import AnalyticdbVectorBySql, AnalyticdbVectorBySqlConfig
 from core.rag.datasource.vdb.vector_base import BaseVector
 from core.rag.datasource.vdb.vector_factory import AbstractVectorFactory
 from core.rag.datasource.vdb.vector_type import VectorType
 from core.rag.embedding.embedding_base import Embeddings
 from core.rag.models.document import Document
-from extensions.ext_redis import redis_client
 from models.dataset import Dataset
 
 
-class AnalyticdbConfig(BaseModel):
-    access_key_id: str
-    access_key_secret: str
-    region_id: str
-    instance_id: str
-    account: str
-    account_password: str
-    namespace: str = ("dify",)
-    namespace_password: str = (None,)
-    metrics: str = ("cosine",)
-    read_timeout: int = 60000
-
-    def to_analyticdb_client_params(self):
-        return {
-            "access_key_id": self.access_key_id,
-            "access_key_secret": self.access_key_secret,
-            "region_id": self.region_id,
-            "read_timeout": self.read_timeout,
-        }
-
-
 class AnalyticdbVector(BaseVector):
-    def __init__(self, collection_name: str, config: AnalyticdbConfig):
-        self._collection_name = collection_name.lower()
-        try:
-            from alibabacloud_gpdb20160503.client import Client
-            from alibabacloud_tea_openapi import models as open_api_models
-        except:
-            raise ImportError(_import_err_msg)
-        self.config = config
-        self._client_config = open_api_models.Config(user_agent="dify", **config.to_analyticdb_client_params())
-        self._client = Client(self._client_config)
-        self._initialize()
-
-    def _initialize(self) -> None:
-        cache_key = f"vector_indexing_{self.config.instance_id}"
-        lock_name = f"{cache_key}_lock"
-        with redis_client.lock(lock_name, timeout=20):
-            collection_exist_cache_key = f"vector_indexing_{self.config.instance_id}"
-            if redis_client.get(collection_exist_cache_key):
-                return
-            self._initialize_vector_database()
-            self._create_namespace_if_not_exists()
-            redis_client.set(collection_exist_cache_key, 1, ex=3600)
-
-    def _initialize_vector_database(self) -> None:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        request = gpdb_20160503_models.InitVectorDatabaseRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            manager_account=self.config.account,
-            manager_account_password=self.config.account_password,
-        )
-        self._client.init_vector_database(request)
-
-    def _create_namespace_if_not_exists(self) -> None:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-        from Tea.exceptions import TeaException
-
-        try:
-            request = gpdb_20160503_models.DescribeNamespaceRequest(
-                dbinstance_id=self.config.instance_id,
-                region_id=self.config.region_id,
-                namespace=self.config.namespace,
-                manager_account=self.config.account,
-                manager_account_password=self.config.account_password,
-            )
-            self._client.describe_namespace(request)
-        except TeaException as e:
-            if e.statusCode == 404:
-                request = gpdb_20160503_models.CreateNamespaceRequest(
-                    dbinstance_id=self.config.instance_id,
-                    region_id=self.config.region_id,
-                    manager_account=self.config.account,
-                    manager_account_password=self.config.account_password,
-                    namespace=self.config.namespace,
-                    namespace_password=self.config.namespace_password,
-                )
-                self._client.create_namespace(request)
-            else:
-                raise ValueError(f"failed to create namespace {self.config.namespace}: {e}")
-
-    def _create_collection_if_not_exists(self, embedding_dimension: int):
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-        from Tea.exceptions import TeaException
-
-        cache_key = f"vector_indexing_{self._collection_name}"
-        lock_name = f"{cache_key}_lock"
-        with redis_client.lock(lock_name, timeout=20):
-            collection_exist_cache_key = f"vector_indexing_{self._collection_name}"
-            if redis_client.get(collection_exist_cache_key):
-                return
-            try:
-                request = gpdb_20160503_models.DescribeCollectionRequest(
-                    dbinstance_id=self.config.instance_id,
-                    region_id=self.config.region_id,
-                    namespace=self.config.namespace,
-                    namespace_password=self.config.namespace_password,
-                    collection=self._collection_name,
-                )
-                self._client.describe_collection(request)
-            except TeaException as e:
-                if e.statusCode == 404:
-                    metadata = '{"ref_doc_id":"text","page_content":"text","metadata_":"jsonb"}'
-                    full_text_retrieval_fields = "page_content"
-                    request = gpdb_20160503_models.CreateCollectionRequest(
-                        dbinstance_id=self.config.instance_id,
-                        region_id=self.config.region_id,
-                        manager_account=self.config.account,
-                        manager_account_password=self.config.account_password,
-                        namespace=self.config.namespace,
-                        collection=self._collection_name,
-                        dimension=embedding_dimension,
-                        metrics=self.config.metrics,
-                        metadata=metadata,
-                        full_text_retrieval_fields=full_text_retrieval_fields,
-                    )
-                    self._client.create_collection(request)
-                else:
-                    raise ValueError(f"failed to create collection {self._collection_name}: {e}")
-            redis_client.set(collection_exist_cache_key, 1, ex=3600)
+    def __init__(
+        self, collection_name: str, api_config: AnalyticdbVectorOpenAPIConfig, sql_config: AnalyticdbVectorBySqlConfig
+    ):
+        super().__init__(collection_name)
+        if api_config is not None:
+            self.analyticdb_vector = AnalyticdbVectorOpenAPI(collection_name, api_config)
+        else:
+            self.analyticdb_vector = AnalyticdbVectorBySql(collection_name, sql_config)
 
     def get_type(self) -> str:
         return VectorType.ANALYTICDB
 
     def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
         dimension = len(embeddings[0])
-        self._create_collection_if_not_exists(dimension)
-        self.add_texts(texts, embeddings)
+        self.analyticdb_vector._create_collection_if_not_exists(dimension)
+        self.analyticdb_vector.add_texts(texts, embeddings)
 
-    def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        rows: list[gpdb_20160503_models.UpsertCollectionDataRequestRows] = []
-        for doc, embedding in zip(documents, embeddings, strict=True):
-            metadata = {
-                "ref_doc_id": doc.metadata["doc_id"],
-                "page_content": doc.page_content,
-                "metadata_": json.dumps(doc.metadata),
-            }
-            rows.append(
-                gpdb_20160503_models.UpsertCollectionDataRequestRows(
-                    vector=embedding,
-                    metadata=metadata,
-                )
-            )
-        request = gpdb_20160503_models.UpsertCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            rows=rows,
-        )
-        self._client.upsert_collection_data(request)
+    def add_texts(self, texts: list[Document], embeddings: list[list[float]], **kwargs):
+        self.analyticdb_vector.add_texts(texts, embeddings)
 
     def text_exists(self, id: str) -> bool:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        request = gpdb_20160503_models.QueryCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            metrics=self.config.metrics,
-            include_values=True,
-            vector=None,
-            content=None,
-            top_k=1,
-            filter=f"ref_doc_id='{id}'",
-        )
-        response = self._client.query_collection_data(request)
-        return len(response.body.matches.match) > 0
+        return self.analyticdb_vector.text_exists(id)
 
     def delete_by_ids(self, ids: list[str]) -> None:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        ids_str = ",".join(f"'{id}'" for id in ids)
-        ids_str = f"({ids_str})"
-        request = gpdb_20160503_models.DeleteCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            collection_data=None,
-            collection_data_filter=f"ref_doc_id IN {ids_str}",
-        )
-        self._client.delete_collection_data(request)
+        self.analyticdb_vector.delete_by_ids(ids)
 
     def delete_by_metadata_field(self, key: str, value: str) -> None:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        request = gpdb_20160503_models.DeleteCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            collection_data=None,
-            collection_data_filter=f"metadata_ ->> '{key}' = '{value}'",
-        )
-        self._client.delete_collection_data(request)
+        self.analyticdb_vector.delete_by_metadata_field(key, value)
 
     def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        score_threshold = kwargs.get("score_threshold") or 0.0
-        request = gpdb_20160503_models.QueryCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            include_values=kwargs.pop("include_values", True),
-            metrics=self.config.metrics,
-            vector=query_vector,
-            content=None,
-            top_k=kwargs.get("top_k", 4),
-            filter=None,
-        )
-        response = self._client.query_collection_data(request)
-        documents = []
-        for match in response.body.matches.match:
-            if match.score > score_threshold:
-                metadata = json.loads(match.metadata.get("metadata_"))
-                metadata["score"] = match.score
-                doc = Document(
-                    page_content=match.metadata.get("page_content"),
-                    metadata=metadata,
-                )
-                documents.append(doc)
-        documents = sorted(documents, key=lambda x: x.metadata["score"], reverse=True)
-        return documents
+        return self.analyticdb_vector.search_by_vector(query_vector)
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
-        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-        score_threshold = float(kwargs.get("score_threshold") or 0.0)
-        request = gpdb_20160503_models.QueryCollectionDataRequest(
-            dbinstance_id=self.config.instance_id,
-            region_id=self.config.region_id,
-            namespace=self.config.namespace,
-            namespace_password=self.config.namespace_password,
-            collection=self._collection_name,
-            include_values=kwargs.pop("include_values", True),
-            metrics=self.config.metrics,
-            vector=None,
-            content=query,
-            top_k=kwargs.get("top_k", 4),
-            filter=None,
-        )
-        response = self._client.query_collection_data(request)
-        documents = []
-        for match in response.body.matches.match:
-            if match.score > score_threshold:
-                metadata = json.loads(match.metadata.get("metadata_"))
-                metadata["score"] = match.score
-                doc = Document(
-                    page_content=match.metadata.get("page_content"),
-                    vector=match.metadata.get("vector"),
-                    metadata=metadata,
-                )
-                documents.append(doc)
-        documents = sorted(documents, key=lambda x: x.metadata["score"], reverse=True)
-        return documents
+        return self.analyticdb_vector.search_by_full_text(query, **kwargs)
 
     def delete(self) -> None:
-        try:
-            from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
-
-            request = gpdb_20160503_models.DeleteCollectionRequest(
-                collection=self._collection_name,
-                dbinstance_id=self.config.instance_id,
-                namespace=self.config.namespace,
-                namespace_password=self.config.namespace_password,
-                region_id=self.config.region_id,
-            )
-            self._client.delete_collection(request)
-        except Exception as e:
-            raise e
+        self.analyticdb_vector.delete()
 
 
 class AnalyticdbVectorFactory(AbstractVectorFactory):
-    def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings):
+    def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> AnalyticdbVector:
         if dataset.index_struct_dict:
             class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"]
             collection_name = class_prefix.lower()
@@ -313,26 +65,9 @@ class AnalyticdbVectorFactory(AbstractVectorFactory):
             collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower()
             dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.ANALYTICDB, collection_name))
 
-        # handle optional params
-        if dify_config.ANALYTICDB_KEY_ID is None:
-            raise ValueError("ANALYTICDB_KEY_ID should not be None")
-        if dify_config.ANALYTICDB_KEY_SECRET is None:
-            raise ValueError("ANALYTICDB_KEY_SECRET should not be None")
-        if dify_config.ANALYTICDB_REGION_ID is None:
-            raise ValueError("ANALYTICDB_REGION_ID should not be None")
-        if dify_config.ANALYTICDB_INSTANCE_ID is None:
-            raise ValueError("ANALYTICDB_INSTANCE_ID should not be None")
-        if dify_config.ANALYTICDB_ACCOUNT is None:
-            raise ValueError("ANALYTICDB_ACCOUNT should not be None")
-        if dify_config.ANALYTICDB_PASSWORD is None:
-            raise ValueError("ANALYTICDB_PASSWORD should not be None")
-        if dify_config.ANALYTICDB_NAMESPACE is None:
-            raise ValueError("ANALYTICDB_NAMESPACE should not be None")
-        if dify_config.ANALYTICDB_NAMESPACE_PASSWORD is None:
-            raise ValueError("ANALYTICDB_NAMESPACE_PASSWORD should not be None")
-        return AnalyticdbVector(
-            collection_name,
-            AnalyticdbConfig(
+        if dify_config.ANALYTICDB_HOST is None:
+            # implemented through OpenAPI
+            apiConfig = AnalyticdbVectorOpenAPIConfig(
                 access_key_id=dify_config.ANALYTICDB_KEY_ID,
                 access_key_secret=dify_config.ANALYTICDB_KEY_SECRET,
                 region_id=dify_config.ANALYTICDB_REGION_ID,
@@ -341,5 +76,22 @@ class AnalyticdbVectorFactory(AbstractVectorFactory):
                 account_password=dify_config.ANALYTICDB_PASSWORD,
                 namespace=dify_config.ANALYTICDB_NAMESPACE,
                 namespace_password=dify_config.ANALYTICDB_NAMESPACE_PASSWORD,
-            ),
+            )
+            sqlConfig = None
+        else:
+            # implemented through sql
+            sqlConfig = AnalyticdbVectorBySqlConfig(
+                host=dify_config.ANALYTICDB_HOST,
+                port=dify_config.ANALYTICDB_PORT,
+                account=dify_config.ANALYTICDB_ACCOUNT,
+                account_password=dify_config.ANALYTICDB_PASSWORD,
+                min_connection=dify_config.ANALYTICDB_MIN_CONNECTION,
+                max_connection=dify_config.ANALYTICDB_MAX_CONNECTION,
+                namespace=dify_config.ANALYTICDB_NAMESPACE,
+            )
+            apiConfig = None
+        return AnalyticdbVector(
+            collection_name,
+            apiConfig,
+            sqlConfig,
         )

api/core/rag/datasource/vdb/analyticdb/analyticdb_vector_openapi.py

@@ -0,0 +1,309 @@
+import json
+from typing import Any
+
+from pydantic import BaseModel, model_validator
+
+_import_err_msg = (
+    "`alibabacloud_gpdb20160503` and `alibabacloud_tea_openapi` packages not found, "
+    "please run `pip install alibabacloud_gpdb20160503 alibabacloud_tea_openapi`"
+)
+
+from core.rag.models.document import Document
+from extensions.ext_redis import redis_client
+
+
+class AnalyticdbVectorOpenAPIConfig(BaseModel):
+    access_key_id: str
+    access_key_secret: str
+    region_id: str
+    instance_id: str
+    account: str
+    account_password: str
+    namespace: str = "dify"
+    namespace_password: str = (None,)
+    metrics: str = "cosine"
+    read_timeout: int = 60000
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_config(cls, values: dict) -> dict:
+        if not values["access_key_id"]:
+            raise ValueError("config ANALYTICDB_KEY_ID is required")
+        if not values["access_key_secret"]:
+            raise ValueError("config ANALYTICDB_KEY_SECRET is required")
+        if not values["region_id"]:
+            raise ValueError("config ANALYTICDB_REGION_ID is required")
+        if not values["instance_id"]:
+            raise ValueError("config ANALYTICDB_INSTANCE_ID is required")
+        if not values["account"]:
+            raise ValueError("config ANALYTICDB_ACCOUNT is required")
+        if not values["account_password"]:
+            raise ValueError("config ANALYTICDB_PASSWORD is required")
+        if not values["namespace_password"]:
+            raise ValueError("config ANALYTICDB_NAMESPACE_PASSWORD is required")
+        return values
+
+    def to_analyticdb_client_params(self):
+        return {
+            "access_key_id": self.access_key_id,
+            "access_key_secret": self.access_key_secret,
+            "region_id": self.region_id,
+            "read_timeout": self.read_timeout,
+        }
+
+
+class AnalyticdbVectorOpenAPI:
+    def __init__(self, collection_name: str, config: AnalyticdbVectorOpenAPIConfig):
+        try:
+            from alibabacloud_gpdb20160503.client import Client
+            from alibabacloud_tea_openapi import models as open_api_models
+        except:
+            raise ImportError(_import_err_msg)
+        self._collection_name = collection_name.lower()
+        self.config = config
+        self._client_config = open_api_models.Config(user_agent="dify", **config.to_analyticdb_client_params())
+        self._client = Client(self._client_config)
+        self._initialize()
+
+    def _initialize(self) -> None:
+        cache_key = f"vector_initialize_{self.config.instance_id}"
+        lock_name = f"{cache_key}_lock"
+        with redis_client.lock(lock_name, timeout=20):
+            database_exist_cache_key = f"vector_initialize_{self.config.instance_id}"
+            if redis_client.get(database_exist_cache_key):
+                return
+            self._initialize_vector_database()
+            self._create_namespace_if_not_exists()
+            redis_client.set(database_exist_cache_key, 1, ex=3600)
+
+    def _initialize_vector_database(self) -> None:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        request = gpdb_20160503_models.InitVectorDatabaseRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            manager_account=self.config.account,
+            manager_account_password=self.config.account_password,
+        )
+        self._client.init_vector_database(request)
+
+    def _create_namespace_if_not_exists(self) -> None:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+        from Tea.exceptions import TeaException
+
+        try:
+            request = gpdb_20160503_models.DescribeNamespaceRequest(
+                dbinstance_id=self.config.instance_id,
+                region_id=self.config.region_id,
+                namespace=self.config.namespace,
+                manager_account=self.config.account,
+                manager_account_password=self.config.account_password,
+            )
+            self._client.describe_namespace(request)
+        except TeaException as e:
+            if e.statusCode == 404:
+                request = gpdb_20160503_models.CreateNamespaceRequest(
+                    dbinstance_id=self.config.instance_id,
+                    region_id=self.config.region_id,
+                    manager_account=self.config.account,
+                    manager_account_password=self.config.account_password,
+                    namespace=self.config.namespace,
+                    namespace_password=self.config.namespace_password,
+                )
+                self._client.create_namespace(request)
+            else:
+                raise ValueError(f"failed to create namespace {self.config.namespace}: {e}")
+
+    def _create_collection_if_not_exists(self, embedding_dimension: int):
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+        from Tea.exceptions import TeaException
+
+        cache_key = f"vector_indexing_{self._collection_name}"
+        lock_name = f"{cache_key}_lock"
+        with redis_client.lock(lock_name, timeout=20):
+            collection_exist_cache_key = f"vector_indexing_{self._collection_name}"
+            if redis_client.get(collection_exist_cache_key):
+                return
+            try:
+                request = gpdb_20160503_models.DescribeCollectionRequest(
+                    dbinstance_id=self.config.instance_id,
+                    region_id=self.config.region_id,
+                    namespace=self.config.namespace,
+                    namespace_password=self.config.namespace_password,
+                    collection=self._collection_name,
+                )
+                self._client.describe_collection(request)
+            except TeaException as e:
+                if e.statusCode == 404:
+                    metadata = '{"ref_doc_id":"text","page_content":"text","metadata_":"jsonb"}'
+                    full_text_retrieval_fields = "page_content"
+                    request = gpdb_20160503_models.CreateCollectionRequest(
+                        dbinstance_id=self.config.instance_id,
+                        region_id=self.config.region_id,
+                        manager_account=self.config.account,
+                        manager_account_password=self.config.account_password,
+                        namespace=self.config.namespace,
+                        collection=self._collection_name,
+                        dimension=embedding_dimension,
+                        metrics=self.config.metrics,
+                        metadata=metadata,
+                        full_text_retrieval_fields=full_text_retrieval_fields,
+                    )
+                    self._client.create_collection(request)
+                else:
+                    raise ValueError(f"failed to create collection {self._collection_name}: {e}")
+            redis_client.set(collection_exist_cache_key, 1, ex=3600)
+
+    def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        rows: list[gpdb_20160503_models.UpsertCollectionDataRequestRows] = []
+        for doc, embedding in zip(documents, embeddings, strict=True):
+            metadata = {
+                "ref_doc_id": doc.metadata["doc_id"],
+                "page_content": doc.page_content,
+                "metadata_": json.dumps(doc.metadata),
+            }
+            rows.append(
+                gpdb_20160503_models.UpsertCollectionDataRequestRows(
+                    vector=embedding,
+                    metadata=metadata,
+                )
+            )
+        request = gpdb_20160503_models.UpsertCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            rows=rows,
+        )
+        self._client.upsert_collection_data(request)
+
+    def text_exists(self, id: str) -> bool:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        request = gpdb_20160503_models.QueryCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            metrics=self.config.metrics,
+            include_values=True,
+            vector=None,
+            content=None,
+            top_k=1,
+            filter=f"ref_doc_id='{id}'",
+        )
+        response = self._client.query_collection_data(request)
+        return len(response.body.matches.match) > 0
+
+    def delete_by_ids(self, ids: list[str]) -> None:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        ids_str = ",".join(f"'{id}'" for id in ids)
+        ids_str = f"({ids_str})"
+        request = gpdb_20160503_models.DeleteCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            collection_data=None,
+            collection_data_filter=f"ref_doc_id IN {ids_str}",
+        )
+        self._client.delete_collection_data(request)
+
+    def delete_by_metadata_field(self, key: str, value: str) -> None:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        request = gpdb_20160503_models.DeleteCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            collection_data=None,
+            collection_data_filter=f"metadata_ ->> '{key}' = '{value}'",
+        )
+        self._client.delete_collection_data(request)
+
+    def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        score_threshold = kwargs.get("score_threshold") or 0.0
+        request = gpdb_20160503_models.QueryCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            include_values=kwargs.pop("include_values", True),
+            metrics=self.config.metrics,
+            vector=query_vector,
+            content=None,
+            top_k=kwargs.get("top_k", 4),
+            filter=None,
+        )
+        response = self._client.query_collection_data(request)
+        documents = []
+        for match in response.body.matches.match:
+            if match.score > score_threshold:
+                metadata = json.loads(match.metadata.get("metadata_"))
+                metadata["score"] = match.score
+                doc = Document(
+                    page_content=match.metadata.get("page_content"),
+                    vector=match.values.value,
+                    metadata=metadata,
+                )
+                documents.append(doc)
+        documents = sorted(documents, key=lambda x: x.metadata["score"], reverse=True)
+        return documents
+
+    def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
+        from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+        score_threshold = float(kwargs.get("score_threshold") or 0.0)
+        request = gpdb_20160503_models.QueryCollectionDataRequest(
+            dbinstance_id=self.config.instance_id,
+            region_id=self.config.region_id,
+            namespace=self.config.namespace,
+            namespace_password=self.config.namespace_password,
+            collection=self._collection_name,
+            include_values=kwargs.pop("include_values", True),
+            metrics=self.config.metrics,
+            vector=None,
+            content=query,
+            top_k=kwargs.get("top_k", 4),
+            filter=None,
+        )
+        response = self._client.query_collection_data(request)
+        documents = []
+        for match in response.body.matches.match:
+            if match.score > score_threshold:
+                metadata = json.loads(match.metadata.get("metadata_"))
+                metadata["score"] = match.score
+                doc = Document(
+                    page_content=match.metadata.get("page_content"),
+                    vector=match.values.value,
+                    metadata=metadata,
+                )
+                documents.append(doc)
+        documents = sorted(documents, key=lambda x: x.metadata["score"], reverse=True)
+        return documents
+
+    def delete(self) -> None:
+        try:
+            from alibabacloud_gpdb20160503 import models as gpdb_20160503_models
+
+            request = gpdb_20160503_models.DeleteCollectionRequest(
+                collection=self._collection_name,
+                dbinstance_id=self.config.instance_id,
+                namespace=self.config.namespace,
+                namespace_password=self.config.namespace_password,
+                region_id=self.config.region_id,
+            )
+            self._client.delete_collection(request)
+        except Exception as e:
+            raise e

api/core/rag/datasource/vdb/analyticdb/analyticdb_vector_sql.py

@@ -0,0 +1,245 @@
+import json
+import uuid
+from contextlib import contextmanager
+from typing import Any
+
+import psycopg2.extras
+import psycopg2.pool
+from pydantic import BaseModel, model_validator
+
+from core.rag.models.document import Document
+from extensions.ext_redis import redis_client
+
+
+class AnalyticdbVectorBySqlConfig(BaseModel):
+    host: str
+    port: int
+    account: str
+    account_password: str
+    min_connection: int
+    max_connection: int
+    namespace: str = "dify"
+    metrics: str = "cosine"
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_config(cls, values: dict) -> dict:
+        if not values["host"]:
+            raise ValueError("config ANALYTICDB_HOST is required")
+        if not values["port"]:
+            raise ValueError("config ANALYTICDB_PORT is required")
+        if not values["account"]:
+            raise ValueError("config ANALYTICDB_ACCOUNT is required")
+        if not values["account_password"]:
+            raise ValueError("config ANALYTICDB_PASSWORD is required")
+        if not values["min_connection"]:
+            raise ValueError("config ANALYTICDB_MIN_CONNECTION is required")
+        if not values["max_connection"]:
+            raise ValueError("config ANALYTICDB_MAX_CONNECTION is required")
+        if values["min_connection"] > values["max_connection"]:
+            raise ValueError("config ANALYTICDB_MIN_CONNECTION should less than ANALYTICDB_MAX_CONNECTION")
+        return values
+
+
+class AnalyticdbVectorBySql:
+    def __init__(self, collection_name: str, config: AnalyticdbVectorBySqlConfig):
+        self._collection_name = collection_name.lower()
+        self.databaseName = "knowledgebase"
+        self.config = config
+        self.table_name = f"{self.config.namespace}.{self._collection_name}"
+        self.pool = None
+        self._initialize()
+        if not self.pool:
+            self.pool = self._create_connection_pool()
+
+    def _initialize(self) -> None:
+        cache_key = f"vector_initialize_{self.config.host}"
+        lock_name = f"{cache_key}_lock"
+        with redis_client.lock(lock_name, timeout=20):
+            database_exist_cache_key = f"vector_initialize_{self.config.host}"
+            if redis_client.get(database_exist_cache_key):
+                return
+            self._initialize_vector_database()
+            redis_client.set(database_exist_cache_key, 1, ex=3600)
+
+    def _create_connection_pool(self):
+        return psycopg2.pool.SimpleConnectionPool(
+            self.config.min_connection,
+            self.config.max_connection,
+            host=self.config.host,
+            port=self.config.port,
+            user=self.config.account,
+            password=self.config.account_password,
+            database=self.databaseName,
+        )
+
+    @contextmanager
+    def _get_cursor(self):
+        conn = self.pool.getconn()
+        cur = conn.cursor()
+        try:
+            yield cur
+        finally:
+            cur.close()
+            conn.commit()
+            self.pool.putconn(conn)
+
+    def _initialize_vector_database(self) -> None:
+        conn = psycopg2.connect(
+            host=self.config.host,
+            port=self.config.port,
+            user=self.config.account,
+            password=self.config.account_password,
+            database="postgres",
+        )
+        conn.autocommit = True
+        cur = conn.cursor()
+        try:
+            cur.execute(f"CREATE DATABASE {self.databaseName}")
+        except Exception as e:
+            if "already exists" in str(e):
+                return
+            raise e
+        finally:
+            cur.close()
+            conn.close()
+        self.pool = self._create_connection_pool()
+        with self._get_cursor() as cur:
+            try:
+                cur.execute("CREATE TEXT SEARCH CONFIGURATION zh_cn (PARSER = zhparser)")
+                cur.execute("ALTER TEXT SEARCH CONFIGURATION zh_cn ADD MAPPING FOR n,v,a,i,e,l,x WITH simple")
+            except Exception as e:
+                if "already exists" not in str(e):
+                    raise e
+            cur.execute(
+                "CREATE OR REPLACE FUNCTION "
+                "public.to_tsquery_from_text(txt text, lang regconfig DEFAULT 'english'::regconfig) "
+                "RETURNS tsquery LANGUAGE sql IMMUTABLE STRICT AS $function$ "
+                "SELECT to_tsquery(lang, COALESCE(string_agg(split_part(word, ':', 1), ' | '), '')) "
+                "FROM (SELECT unnest(string_to_array(to_tsvector(lang, txt)::text, ' ')) AS word) "
+                "AS words_only;$function$"
+            )
+            cur.execute(f"CREATE SCHEMA IF NOT EXISTS {self.config.namespace}")
+
+    def _create_collection_if_not_exists(self, embedding_dimension: int):
+        cache_key = f"vector_indexing_{self._collection_name}"
+        lock_name = f"{cache_key}_lock"
+        with redis_client.lock(lock_name, timeout=20):
+            collection_exist_cache_key = f"vector_indexing_{self._collection_name}"
+            if redis_client.get(collection_exist_cache_key):
+                return
+            with self._get_cursor() as cur:
+                cur.execute(
+                    f"CREATE TABLE IF NOT EXISTS {self.table_name}("
+                    f"id text PRIMARY KEY,"
+                    f"vector real[], ref_doc_id text, page_content text, metadata_ jsonb, "
+                    f"to_tsvector TSVECTOR"
+                    f") WITH (fillfactor=70) DISTRIBUTED BY (id);"
+                )
+                if embedding_dimension is not None:
+                    index_name = f"{self._collection_name}_embedding_idx"
+                    cur.execute(f"ALTER TABLE {self.table_name} ALTER COLUMN vector SET STORAGE PLAIN")
+                    cur.execute(
+                        f"CREATE INDEX {index_name} ON {self.table_name} USING ann(vector) "
+                        f"WITH(dim='{embedding_dimension}', distancemeasure='{self.config.metrics}', "
+                        f"pq_enable=0, external_storage=0)"
+                    )
+                    cur.execute(f"CREATE INDEX ON {self.table_name} USING gin(to_tsvector)")
+            redis_client.set(collection_exist_cache_key, 1, ex=3600)
+
+    def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):
+        values = []
+        id_prefix = str(uuid.uuid4()) + "_"
+        sql = f"""
+                INSERT INTO {self.table_name} 
+                (id, ref_doc_id, vector, page_content, metadata_, to_tsvector) 
+                VALUES (%s, %s, %s, %s, %s, to_tsvector('zh_cn',  %s));
+            """
+        for i, doc in enumerate(documents):
+            values.append(
+                (
+                    id_prefix + str(i),
+                    doc.metadata.get("doc_id", str(uuid.uuid4())),
+                    embeddings[i],
+                    doc.page_content,
+                    json.dumps(doc.metadata),
+                    doc.page_content,
+                )
+            )
+        with self._get_cursor() as cur:
+            psycopg2.extras.execute_batch(cur, sql, values)
+
+    def text_exists(self, id: str) -> bool:
+        with self._get_cursor() as cur:
+            cur.execute(f"SELECT id FROM {self.table_name} WHERE ref_doc_id = %s", (id,))
+            return cur.fetchone() is not None
+
+    def delete_by_ids(self, ids: list[str]) -> None:
+        with self._get_cursor() as cur:
+            try:
+                cur.execute(f"DELETE FROM {self.table_name} WHERE ref_doc_id IN %s", (tuple(ids),))
+            except Exception as e:
+                if "does not exist" not in str(e):
+                    raise e
+
+    def delete_by_metadata_field(self, key: str, value: str) -> None:
+        with self._get_cursor() as cur:
+            try:
+                cur.execute(f"DELETE FROM {self.table_name} WHERE metadata_->>%s = %s", (key, value))
+            except Exception as e:
+                if "does not exist" not in str(e):
+                    raise e
+
+    def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]:
+        top_k = kwargs.get("top_k", 4)
+        score_threshold = float(kwargs.get("score_threshold") or 0.0)
+        with self._get_cursor() as cur:
+            query_vector_str = json.dumps(query_vector)
+            query_vector_str = "{" + query_vector_str[1:-1] + "}"
+            cur.execute(
+                f"SELECT t.id AS id, t.vector AS vector, (1.0 - t.score) AS score, "
+                f"t.page_content as page_content, t.metadata_ AS metadata_ "
+                f"FROM (SELECT id, vector, page_content, metadata_, vector <=> %s AS score "
+                f"FROM {self.table_name} ORDER BY score LIMIT {top_k} ) t",
+                (query_vector_str,),
+            )
+            documents = []
+            for record in cur:
+                id, vector, score, page_content, metadata = record
+                if score > score_threshold:
+                    metadata["score"] = score
+                    doc = Document(
+                        page_content=page_content,
+                        vector=vector,
+                        metadata=metadata,
+                    )
+                    documents.append(doc)
+        return documents
+
+    def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
+        top_k = kwargs.get("top_k", 4)
+        with self._get_cursor() as cur:
+            cur.execute(
+                f"""SELECT id, vector, page_content, metadata_, 
+                ts_rank(to_tsvector, to_tsquery_from_text(%s, 'zh_cn'), 32) AS score
+                FROM {self.table_name}
+                WHERE to_tsvector@@to_tsquery_from_text(%s, 'zh_cn')
+                ORDER BY score DESC
+                LIMIT {top_k}""",
+                (f"'{query}'", f"'{query}'"),
+            )
+            documents = []
+            for record in cur:
+                id, vector, page_content, metadata, score = record
+                metadata["score"] = score
+                doc = Document(
+                    page_content=page_content,
+                    vector=vector,
+                    metadata=metadata,
+                )
+                documents.append(doc)
+        return documents
+
+    def delete(self) -> None:
+        with self._get_cursor() as cur:
+            cur.execute(f"DROP TABLE IF EXISTS {self.table_name}")

api/core/rag/datasource/vdb/couchbase/couchbase_vector.py

@@ -242,7 +242,7 @@ class CouchbaseVector(BaseVector):
         try:
             self._cluster.query(query, named_parameters={"doc_ids": ids}).execute()
         except Exception as e:
-            logger.exception(e)
+            logger.exception(f"Failed to delete documents, ids: {ids}")
 
     def delete_by_document_id(self, document_id: str):
         query = f"""

api/core/rag/datasource/vdb/lindorm/lindorm_vector.py

@@ -79,7 +79,7 @@ class LindormVectorStore(BaseVector):
                 existing_docs = self._client.mget(index=self._collection_name, body={"ids": batch_ids}, _source=False)
                 return {doc["_id"] for doc in existing_docs["docs"] if doc["found"]}
             except Exception as e:
-                logger.exception(f"Error fetching batch {batch_ids}: {e}")
+                logger.exception(f"Error fetching batch {batch_ids}")
                 return set()
 
         @retry(stop=stop_after_attempt(3), wait=wait_fixed(60))
@@ -96,7 +96,7 @@ class LindormVectorStore(BaseVector):
                 )
                 return {doc["_id"] for doc in existing_docs["docs"] if doc["found"]}
             except Exception as e:
-                logger.exception(f"Error fetching batch {batch_ids}: {e}")
+                logger.exception(f"Error fetching batch ids: {batch_ids}")
                 return set()
 
         if ids is None:
@@ -177,7 +177,7 @@ class LindormVectorStore(BaseVector):
             else:
                 logger.warning(f"Index '{self._collection_name}' does not exist. No deletion performed.")
         except Exception as e:
-            logger.exception(f"Error occurred while deleting the index: {e}")
+            logger.exception(f"Error occurred while deleting the index: {self._collection_name}")
             raise e
 
     def text_exists(self, id: str) -> bool:
@@ -201,7 +201,7 @@ class LindormVectorStore(BaseVector):
         try:
             response = self._client.search(index=self._collection_name, body=query)
         except Exception as e:
-            logger.exception(f"Error executing search: {e}")
+            logger.exception(f"Error executing vector search, query: {query}")
             raise
 
         docs_and_scores = []

api/core/rag/datasource/vdb/myscale/myscale_vector.py

@@ -142,7 +142,7 @@ class MyScaleVector(BaseVector):
                 for r in self._client.query(sql).named_results()
             ]
         except Exception as e:
-            logging.exception(f"\033[91m\033[1m{type(e)}\033[0m \033[95m{str(e)}\033[0m")
+            logging.exception(f"\033[91m\033[1m{type(e)}\033[0m \033[95m{str(e)}\033[0m")  # noqa:TRY401
             return []
 
     def delete(self) -> None:

api/core/rag/datasource/vdb/opensearch/opensearch_vector.py

@@ -158,7 +158,7 @@ class OpenSearchVector(BaseVector):
         try:
             response = self._client.search(index=self._collection_name.lower(), body=query)
         except Exception as e:
-            logger.exception(f"Error executing search: {e}")
+            logger.exception(f"Error executing vector search, query: {query}")
             raise
 
         docs = []

api/core/rag/embedding/cached_embedding.py

@@ -69,7 +69,7 @@ class CacheEmbedding(Embeddings):
                         except IntegrityError:
                             db.session.rollback()
                         except Exception as e:
-                            logging.exception("Failed transform embedding: %s", e)
+                            logging.exception("Failed transform embedding")
                 cache_embeddings = []
                 try:
                     for i, embedding in zip(embedding_queue_indices, embedding_queue_embeddings):
@@ -89,7 +89,7 @@ class CacheEmbedding(Embeddings):
                     db.session.rollback()
             except Exception as ex:
                 db.session.rollback()
-                logger.exception("Failed to embed documents: %s", ex)
+                logger.exception("Failed to embed documents: %s")
                 raise ex
 
         return text_embeddings
@@ -112,7 +112,7 @@ class CacheEmbedding(Embeddings):
             embedding_results = (embedding_results / np.linalg.norm(embedding_results)).tolist()
         except Exception as ex:
             if dify_config.DEBUG:
-                logging.exception(f"Failed to embed query text: {ex}")
+                logging.exception(f"Failed to embed query text '{text[:10]}...({len(text)} chars)'")
             raise ex
 
         try:
@@ -126,7 +126,7 @@ class CacheEmbedding(Embeddings):
             redis_client.setex(embedding_cache_key, 600, encoded_str)
         except Exception as ex:
             if dify_config.DEBUG:
-                logging.exception("Failed to add embedding to redis %s", ex)
+                logging.exception(f"Failed to add embedding to redis for the text '{text[:10]}...({len(text)} chars)'")
             raise ex
 
         return embedding_results

api/core/rag/extractor/word_extractor.py

@@ -229,7 +229,7 @@ class WordExtractor(BaseExtractor):
                                 for i in url_pattern.findall(x.text):
                                     hyperlinks_url = str(i)
                     except Exception as e:
-                        logger.exception(e)
+                        logger.exception("Failed to parse HYPERLINK xml")
 
         def parse_paragraph(paragraph):
             paragraph_content = []

api/core/rag/index_processor/processor/paragraph_index_processor.py

@@ -11,6 +11,7 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.extract_processor import ExtractProcessor
 from core.rag.index_processor.index_processor_base import BaseIndexProcessor
 from core.rag.models.document import Document
+from core.tools.utils.text_processing_utils import remove_leading_symbols
 from libs import helper
 from models.dataset import Dataset
 
@@ -43,11 +44,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
                     document_node.metadata["doc_id"] = doc_id
                     document_node.metadata["doc_hash"] = hash
                     # delete Splitter character
-                    page_content = document_node.page_content
-                    if page_content.startswith(".") or page_content.startswith("。"):
-                        page_content = page_content[1:].strip()
-                    else:
-                        page_content = page_content
+                    page_content = remove_leading_symbols(document_node.page_content).strip()
                     if len(page_content) > 0:
                         document_node.page_content = page_content
                         split_documents.append(document_node)

api/core/rag/index_processor/processor/qa_index_processor.py

@@ -18,6 +18,7 @@ from core.rag.extractor.entity.extract_setting import ExtractSetting
 from core.rag.extractor.extract_processor import ExtractProcessor
 from core.rag.index_processor.index_processor_base import BaseIndexProcessor
 from core.rag.models.document import Document
+from core.tools.utils.text_processing_utils import remove_leading_symbols
 from libs import helper
 from models.dataset import Dataset
 
@@ -53,11 +54,7 @@ class QAIndexProcessor(BaseIndexProcessor):
                     document_node.metadata["doc_hash"] = hash
                     # delete Splitter character
                     page_content = document_node.page_content
-                    if page_content.startswith(".") or page_content.startswith("。"):
-                        page_content = page_content[1:]
-                    else:
-                        page_content = page_content
-                    document_node.page_content = page_content
+                    document_node.page_content = remove_leading_symbols(page_content)
                     split_documents.append(document_node)
             all_documents.extend(split_documents)
         for i in range(0, len(all_documents), 10):
@@ -159,7 +156,7 @@ class QAIndexProcessor(BaseIndexProcessor):
                     qa_documents.append(qa_document)
                 format_documents.extend(qa_documents)
             except Exception as e:
-                logging.exception(e)
+                logging.exception("Failed to format qa document")
 
             all_qa_documents.extend(format_documents)
 

api/core/rag/rerank/weight_rerank.py

@@ -36,23 +36,21 @@ class WeightRerankRunner(BaseRerankRunner):
 
         :return:
         """
-        docs = []
-        doc_id = []
         unique_documents = []
+        doc_id = set()
         for document in documents:
-            if document.metadata["doc_id"] not in doc_id:
-                doc_id.append(document.metadata["doc_id"])
-                docs.append(document.page_content)
+            doc_id = document.metadata.get("doc_id")
+            if doc_id not in doc_id:
+                doc_id.add(doc_id)
                 unique_documents.append(document)
 
         documents = unique_documents
 
-        rerank_documents = []
         query_scores = self._calculate_keyword_score(query, documents)
-
         query_vector_scores = self._calculate_cosine(self.tenant_id, query, documents, self.weights.vector_setting)
+
+        rerank_documents = []
         for document, query_score, query_vector_score in zip(documents, query_scores, query_vector_scores):
-            # format document
             score = (
                 self.weights.vector_setting.vector_weight * query_vector_score
                 + self.weights.keyword_setting.keyword_weight * query_score
@@ -61,7 +59,8 @@ class WeightRerankRunner(BaseRerankRunner):
                 continue
             document.metadata["score"] = score
             rerank_documents.append(document)
-        rerank_documents = sorted(rerank_documents, key=lambda x: x.metadata["score"], reverse=True)
+
+        rerank_documents.sort(key=lambda x: x.metadata["score"], reverse=True)
         return rerank_documents[:top_n] if top_n else rerank_documents
 
     def _calculate_keyword_score(self, query: str, documents: list[Document]) -> list[float]:

api/core/tools/custom_tool/tool.py

@@ -6,6 +6,7 @@ from urllib.parse import urlencode
 
 import httpx
 
+from core.file.file_manager import download
 from core.helper import ssrf_proxy
 from core.tools.__base.tool import Tool
 from core.tools.__base.tool_runtime import ToolRuntime
@@ -145,6 +146,7 @@ class ApiTool(Tool):
         path_params = {}
         body = {}
         cookies = {}
+        files = []
 
         # check parameters
         for parameter in self.api_bundle.openapi.get("parameters", []):
@@ -173,8 +175,12 @@ class ApiTool(Tool):
                     properties = body_schema.get("properties", {})
                     for name, property in properties.items():
                         if name in parameters:
-                            # convert type
-                            body[name] = self._convert_body_property_type(property, parameters[name])
+                            if property.get("format") == "binary":
+                                f = parameters[name]
+                                files.append((name, (f.filename, download(f), f.mime_type)))
+                            else:
+                                # convert type
+                                body[name] = self._convert_body_property_type(property, parameters[name])
                         elif name in required:
                             raise ToolParameterValidationError(
                                 f"Missing required parameter {name} in operation {self.api_bundle.operation_id}"
@@ -189,7 +195,7 @@ class ApiTool(Tool):
         for name, value in path_params.items():
             url = url.replace(f"{{{name}}}", f"{value}")
 
-        # parse http body data if needed, for GET/HEAD/OPTIONS/TRACE, the body is ignored
+        # parse http body data if needed
         if "Content-Type" in headers:
             if headers["Content-Type"] == "application/json":
                 body = json.dumps(body)
@@ -205,6 +211,7 @@ class ApiTool(Tool):
                 headers=headers,
                 cookies=cookies,
                 data=body,
+                files=files,
                 timeout=API_TOOL_DEFAULT_TIMEOUT,
                 follow_redirects=True,
             )

api/core/tools/tool_engine.py

@@ -58,7 +58,7 @@ class ToolEngine:
             # check if this tool has only one parameter
             parameters = [
                 parameter
-                for parameter in tool.get_runtime_parameters() or []
+                for parameter in tool.get_runtime_parameters()
                 if parameter.form == ToolParameter.ToolParameterForm.LLM
             ]
             if parameters and len(parameters) == 1:

api/core/tools/tool_file_manager.py

@@ -98,7 +98,7 @@ class ToolFileManager:
             response.raise_for_status()
             blob = response.content
         except Exception as e:
-            logger.exception(f"Failed to download file from {file_url}: {e}")
+            logger.exception(f"Failed to download file from {file_url}")
             raise
 
         mimetype = guess_type(file_url)[0] or "octet/stream"

api/core/tools/tool_manager.py

@@ -526,7 +526,7 @@ class ToolManager:
                     yield provider
 
                 except Exception as e:
-                    logger.exception(f"load builtin provider {provider} error: {e}")
+                    logger.exception(f"load builtin provider {provider}")
                     continue
         # set builtin providers loaded
         cls._builtin_providers_loaded = True

api/core/tools/utils/configuration.py

@@ -145,7 +145,7 @@ class ToolParameterConfigurationManager:
         # get tool parameters
         tool_parameters = self.tool_runtime.entity.parameters or []
         # get tool runtime parameters
-        runtime_parameters = self.tool_runtime.get_runtime_parameters() or []
+        runtime_parameters = self.tool_runtime.get_runtime_parameters()
         # override parameters
         current_parameters = tool_parameters.copy()
         for runtime_parameter in runtime_parameters:

api/core/tools/utils/message_transformer.py

@@ -49,7 +49,7 @@ class ToolFileMessageTransformer:
                         meta=message.meta.copy() if message.meta is not None else {},
                     )
                 except Exception as e:
-                    logger.exception(e)
+                    logger.exception(f"Failed to download image from {url}")
                     yield ToolInvokeMessage(
                         type=ToolInvokeMessage.MessageType.TEXT,
                         message=ToolInvokeMessage.TextMessage(

api/core/tools/utils/parser.py

@@ -160,6 +160,9 @@ class ApiBasedToolSchemaParser:
     def _get_tool_parameter_type(parameter: dict) -> ToolParameter.ToolParameterType:
         parameter = parameter or {}
         typ = None
+        if parameter.get("format") == "binary":
+            return ToolParameter.ToolParameterType.FILE
+
         if "type" in parameter:
             typ = parameter["type"]
         elif "schema" in parameter and "type" in parameter["schema"]:

api/core/tools/utils/text_processing_utils.py

@@ -0,0 +1,16 @@
+import re
+
+
+def remove_leading_symbols(text: str) -> str:
+    """
+    Remove leading punctuation or symbols from the given text.
+
+    Args:
+        text (str): The input text to process.
+
+    Returns:
+        str: The text with leading punctuation or symbols removed.
+    """
+    # Match Unicode ranges for punctuation and symbols
+    pattern = r"^[\u2000-\u206F\u2E00-\u2E7F\u3000-\u303F!\"#$%&'()*+,\-./:;<=>?@\[\]^_`{|}~]+"
+    return re.sub(pattern, "", text)

api/core/tools/workflow_as_tool/tool.py

@@ -199,7 +199,7 @@ class WorkflowTool(Tool):
 
                             files.append(file_dict)
                     except Exception as e:
-                        logger.exception(e)
+                        logger.exception(f"Failed to transform file {file}")
             else:
                 parameters_result[parameter.name] = tool_parameters.get(parameter.name)