From 41b5596441837a1f37b4077ec8ec6babdff9329a Mon Sep 17 00:00:00 2001 From: jyong <718720800@qq.com> Date: Thu, 4 Sep 2025 17:11:13 +0800 Subject: [PATCH] fix file name --- api/commands.py | 262 +++++++++++------------- api/extensions/ext_commands.py | 4 + api/services/plugin/plugin_migration.py | 14 +- 3 files changed, 129 insertions(+), 151 deletions(-) diff --git a/api/commands.py b/api/commands.py index d47cd6371a..1119866f99 100644 --- a/api/commands.py +++ b/api/commands.py @@ -42,6 +42,7 @@ from services.account_service import AccountService, RegisterService, TenantServ from services.clear_free_plan_tenant_expired_logs import ClearFreePlanTenantExpiredLogs from services.plugin.data_migration import PluginDataMigration from services.plugin.plugin_migration import PluginMigration +from services.plugin.plugin_service import PluginService from tasks.remove_app_and_related_data_task import delete_draft_variables_batch logger = logging.getLogger(__name__) @@ -1426,154 +1427,135 @@ def transform_datasource_credentials(): # deal notion credentials deal_notion_count = 0 notion_credentials = db.session.query(DataSourceOauthBinding).filter_by(provider="notion").all() - notion_credentials_tenant_mapping: dict[str, list[DataSourceOauthBinding]] = {} - for credential in notion_credentials: - tenant_id = credential.tenant_id - if tenant_id not in notion_credentials_tenant_mapping: - notion_credentials_tenant_mapping[tenant_id] = [] - notion_credentials_tenant_mapping[tenant_id].append(credential) - for tenant_id, credentials in notion_credentials_tenant_mapping.items(): - # check notion plugin is installed - installed_plugins = installer_manager.list_plugins(tenant_id) - installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] - if notion_plugin_id not in installed_plugins_ids: - if notion_plugin_unique_identifier: - # install notion plugin - installer_manager.install_from_identifiers( - tenant_id, - [notion_plugin_unique_identifier], - PluginInstallationSource.Marketplace, - metas=[ - { - "plugin_unique_identifier": notion_plugin_unique_identifier, - } - ], + if notion_credentials: + notion_credentials_tenant_mapping: dict[str, list[DataSourceOauthBinding]] = {} + for credential in notion_credentials: + tenant_id = credential.tenant_id + if tenant_id not in notion_credentials_tenant_mapping: + notion_credentials_tenant_mapping[tenant_id] = [] + notion_credentials_tenant_mapping[tenant_id].append(credential) + for tenant_id, credentials in notion_credentials_tenant_mapping.items(): + # check notion plugin is installed + installed_plugins = installer_manager.list_plugins(tenant_id) + installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] + if notion_plugin_id not in installed_plugins_ids: + if notion_plugin_unique_identifier: + # install notion plugin + PluginService.install_from_marketplace_pkg(tenant_id, [notion_plugin_unique_identifier]) + auth_count = 0 + for credential in credentials: + auth_count += 1 + # get credential oauth params + access_token = credential.access_token + # notion info + notion_info = credential.source_info + workspace_id = notion_info.get("workspace_id") + workspace_name = notion_info.get("workspace_name") + workspace_icon = notion_info.get("workspace_icon") + new_credentials = { + "integration_secret": encrypter.encrypt_token(tenant_id, access_token), + "workspace_id": workspace_id, + "workspace_name": workspace_name, + "workspace_icon": workspace_icon, + } + datasource_provider = DatasourceProvider( + provider="notion_datasource", + tenant_id=tenant_id, + plugin_id=notion_plugin_id, + auth_type=oauth_credential_type.value, + encrypted_credentials=new_credentials, + name=f"Auth {auth_count}", + avatar_url=workspace_icon or "default", + is_default=False, ) - auth_count = 0 - for credential in credentials: - auth_count += 1 - # get credential oauth params - access_token = credential.access_token - # notion info - notion_info = credential.source_info - workspace_id = notion_info.get("workspace_id") - workspace_name = notion_info.get("workspace_name") - workspace_icon = notion_info.get("workspace_icon") - new_credentials = { - "integration_secret": encrypter.encrypt_token(tenant_id, access_token), - "workspace_id": workspace_id, - "workspace_name": workspace_name, - "workspace_icon": workspace_icon, - } - datasource_provider = DatasourceProvider( - provider="notion", - tenant_id=tenant_id, - plugin_id=notion_plugin_id, - auth_type=oauth_credential_type.value, - encrypted_credentials=new_credentials, - name=f"Auth {auth_count}", - avatar_url=workspace_icon or "default", - is_default=False, - ) - db.session.add(datasource_provider) - deal_notion_count += 1 - db.session.commit() + db.session.add(datasource_provider) + deal_notion_count += 1 + db.session.commit() # deal firecrawl credentials deal_firecrawl_count = 0 firecrawl_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="firecrawl").all() - firecrawl_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {} - for credential in firecrawl_credentials: - tenant_id = credential.tenant_id - if tenant_id not in firecrawl_credentials_tenant_mapping: - firecrawl_credentials_tenant_mapping[tenant_id] = [] - firecrawl_credentials_tenant_mapping[tenant_id].append(credential) - for tenant_id, credentials in firecrawl_credentials_tenant_mapping.items(): - # check firecrawl plugin is installed - installed_plugins = installer_manager.list_plugins(tenant_id) - installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] - if firecrawl_plugin_id not in installed_plugins_ids: - if firecrawl_plugin_unique_identifier: - # install firecrawl plugin - installer_manager.install_from_identifiers( - tenant_id, - [firecrawl_plugin_unique_identifier], - PluginInstallationSource.Marketplace, - metas=[ - { - "plugin_unique_identifier": firecrawl_plugin_unique_identifier, - } - ], + if firecrawl_credentials: + firecrawl_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {} + for credential in firecrawl_credentials: + tenant_id = credential.tenant_id + if tenant_id not in firecrawl_credentials_tenant_mapping: + firecrawl_credentials_tenant_mapping[tenant_id] = [] + firecrawl_credentials_tenant_mapping[tenant_id].append(credential) + for tenant_id, credentials in firecrawl_credentials_tenant_mapping.items(): + # check firecrawl plugin is installed + installed_plugins = installer_manager.list_plugins(tenant_id) + installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] + if firecrawl_plugin_id not in installed_plugins_ids: + if firecrawl_plugin_unique_identifier: + # install firecrawl plugin + PluginService.install_from_marketplace_pkg(tenant_id, [firecrawl_plugin_unique_identifier]) + + auth_count = 0 + for credential in credentials: + auth_count += 1 + # get credential api key + credentials_json =json.loads(credential.credentials) + api_key = credentials_json.get("config", {}).get("api_key") + base_url = credentials_json.get("config", {}).get("base_url") + new_credentials = { + "firecrawl_api_key": api_key, + "base_url": base_url, + } + datasource_provider = DatasourceProvider( + provider="firecrawl", + tenant_id=tenant_id, + plugin_id=firecrawl_plugin_id, + auth_type=api_key_credential_type.value, + encrypted_credentials=new_credentials, + name=f"Auth {auth_count}", + avatar_url="default", + is_default=False, ) - auth_count = 0 - for credential in credentials: - auth_count += 1 - # get credential api key - api_key = credential.credentials.get("config", {}).get("api_key") - base_url = credential.credentials.get("config", {}).get("base_url") - new_credentials = { - "firecrawl_api_key": api_key, - "base_url": base_url, - } - datasource_provider = DatasourceProvider( - provider="firecrawl", - tenant_id=tenant_id, - plugin_id=firecrawl_plugin_id, - auth_type=api_key_credential_type.value, - encrypted_credentials=new_credentials, - name=f"Auth {auth_count}", - avatar_url="default", - is_default=False, - ) - db.session.add(datasource_provider) - deal_firecrawl_count += 1 - db.session.commit() + db.session.add(datasource_provider) + deal_firecrawl_count += 1 + db.session.commit() # deal jina credentials deal_jina_count = 0 - jina_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="jina").all() - jina_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {} - for credential in jina_credentials: - tenant_id = credential.tenant_id - if tenant_id not in jina_credentials_tenant_mapping: - jina_credentials_tenant_mapping[tenant_id] = [] - jina_credentials_tenant_mapping[tenant_id].append(credential) - for tenant_id, credentials in jina_credentials_tenant_mapping.items(): - # check jina plugin is installed - installed_plugins = installer_manager.list_plugins(tenant_id) - installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] - if jina_plugin_id not in installed_plugins_ids: - if jina_plugin_unique_identifier: - # install jina plugin - installer_manager.install_from_identifiers( - tenant_id, - [jina_plugin_unique_identifier], - PluginInstallationSource.Marketplace, - metas=[ - { - "plugin_unique_identifier": jina_plugin_unique_identifier, - } - ], + jina_credentials = db.session.query(DataSourceApiKeyAuthBinding).filter_by(provider="jinareader").all() + if jina_credentials: + jina_credentials_tenant_mapping: dict[str, list[DataSourceApiKeyAuthBinding]] = {} + for credential in jina_credentials: + tenant_id = credential.tenant_id + if tenant_id not in jina_credentials_tenant_mapping: + jina_credentials_tenant_mapping[tenant_id] = [] + jina_credentials_tenant_mapping[tenant_id].append(credential) + for tenant_id, credentials in jina_credentials_tenant_mapping.items(): + # check jina plugin is installed + installed_plugins = installer_manager.list_plugins(tenant_id) + installed_plugins_ids = [plugin.plugin_id for plugin in installed_plugins] + if jina_plugin_id not in installed_plugins_ids: + if jina_plugin_unique_identifier: + # install jina plugin + print(jina_plugin_unique_identifier) + PluginService.install_from_marketplace_pkg(tenant_id, [jina_plugin_unique_identifier]) + + auth_count = 0 + for credential in credentials: + auth_count += 1 + # get credential api key + credentials_json = json.loads(credential.credentials) + api_key = credentials_json.get("config", {}).get("api_key") + new_credentials = { + "integration_secret": api_key, + } + datasource_provider = DatasourceProvider( + provider="jina", + tenant_id=tenant_id, + plugin_id=jina_plugin_id, + auth_type=api_key_credential_type.value, + encrypted_credentials=new_credentials, + name=f"Auth {auth_count}", + avatar_url="default", + is_default=False, ) - auth_count = 0 - for credential in credentials: - auth_count += 1 - # get credential api key - api_key = credential.credentials.get("config", {}).get("api_key") - new_credentials = { - "integration_secret": api_key, - } - datasource_provider = DatasourceProvider( - provider="jina", - tenant_id=tenant_id, - plugin_id=jina_plugin_id, - auth_type=api_key_credential_type.value, - encrypted_credentials=new_credentials, - name=f"Auth {auth_count}", - avatar_url="default", - is_default=False, - ) - db.session.add(datasource_provider) - deal_jina_count += 1 - db.session.commit() + db.session.add(datasource_provider) + deal_jina_count += 1 + db.session.commit() except Exception as e: click.echo(click.style(f"Error parsing client params: {str(e)}", fg="red")) return diff --git a/api/extensions/ext_commands.py b/api/extensions/ext_commands.py index a7c27f7b74..c716e6b64e 100644 --- a/api/extensions/ext_commands.py +++ b/api/extensions/ext_commands.py @@ -13,6 +13,7 @@ def init_app(app: DifyApp): extract_unique_plugins, fix_app_site_missing, install_plugins, + install_rag_pipeline_plugins, migrate_data_for_plugin, old_metadata_migration, remove_orphaned_files_on_storage, @@ -21,6 +22,7 @@ def init_app(app: DifyApp): reset_password, setup_datasource_oauth_client, setup_system_tool_oauth_client, + transform_datasource_credentials, upgrade_db, vdb_migrate, ) @@ -46,6 +48,8 @@ def init_app(app: DifyApp): setup_system_tool_oauth_client, cleanup_orphaned_draft_variables, setup_datasource_oauth_client, + transform_datasource_credentials, + install_rag_pipeline_plugins, ] for cmd in cmds_to_register: app.cli.add_command(cmd) diff --git a/api/services/plugin/plugin_migration.py b/api/services/plugin/plugin_migration.py index 66a75b0049..73021b607f 100644 --- a/api/services/plugin/plugin_migration.py +++ b/api/services/plugin/plugin_migration.py @@ -26,6 +26,7 @@ from models.model import App, AppMode, AppModelConfig from models.provider_ids import ModelProviderID, ToolProviderID from models.tools import BuiltinToolProvider from models.workflow import Workflow +from services.plugin.plugin_service import PluginService logger = logging.getLogger(__name__) @@ -457,17 +458,8 @@ class PluginMigration: for plugin_id in batch_plugin_ids if plugin_id not in installed_plugins_ids and plugin_id in plugin_ids ] - manager.install_from_identifiers( - tenant_id, - batch_plugin_identifiers, - PluginInstallationSource.Marketplace, - metas=[ - { - "plugin_unique_identifier": identifier, - } - for identifier in batch_plugin_identifiers - ], - ) + PluginService.install_from_marketplace_pkg(tenant_id, batch_plugin_identifiers) + total_success_tenant += 1 except Exception: logger.exception("Failed to install plugins for tenant %s", tenant_id)