diff --git a/api/commands.py b/api/commands.py index 9a023b1c48..73d2150de2 100644 --- a/api/commands.py +++ b/api/commands.py @@ -1,5 +1,6 @@ import base64 import json +import logging import secrets import click @@ -12,11 +13,12 @@ from extensions.ext_database import db from libs.helper import email as email_validate from libs.password import hash_password, password_pattern, valid_password from libs.rsa import generate_key_pair -from models.account import Tenant +from models.account import Tenant, TenantAccountJoin from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment from models.dataset import Document as DatasetDocument from models.model import Account, App, AppMode, AppModelConfig, AppAnnotationSetting, Conversation, MessageAnnotation from models.provider import Provider, ProviderModel +from services.workflow.workflow_converter import WorkflowConverter @click.command('reset-password', help='Reset the account password.') @@ -422,9 +424,77 @@ and am.agent_mode like '{"enabled": true%' ORDER BY a.created_at DESC LIMIT 1000 click.echo(click.style('Congratulations! Converted {} agent apps.'.format(len(proceeded_app_ids)), fg='green')) +@click.command('convert-to-workflow-chatbot-apps', help='Convert Basic Export Assistant to Chatbot Workflow App.') +def convert_to_workflow_chatbot_apps(): + """ + Convert Basic Export Assistant to Chatbot Workflow App. + """ + click.echo(click.style('Start convert to workflow chatbot apps.', fg='green')) + + proceeded_app_ids = [] + workflow_converter = WorkflowConverter() + + while True: + # fetch first 1000 apps + sql_query = """SELECT a.id FROM apps a +LEFT JOIN app_model_configs am ON a.app_model_config_id=am.id +WHERE a.mode = 'chat' AND am.prompt_type='advanced' ORDER BY a.created_at DESC LIMIT 1000""" + + with db.engine.begin() as conn: + rs = conn.execute(db.text(sql_query)) + + apps = [] + for i in rs: + app_id = str(i.id) + print(app_id) + if app_id not in proceeded_app_ids: + proceeded_app_ids.append(app_id) + app = db.session.query(App).filter(App.id == app_id).first() + apps.append(app) + + if len(apps) == 0: + break + + for app in apps: + click.echo('Converting app: {}'.format(app.id)) + + try: + # get workspace of app + tenant = db.session.query(Tenant).filter(Tenant.id == app.tenant_id).first() + if not tenant: + click.echo(click.style('Tenant not found: {}'.format(app.tenant_id), fg='red')) + continue + + # get workspace owner + tenant_account_join = db.session.query(TenantAccountJoin).filter( + TenantAccountJoin.tenant_id == tenant.id, + TenantAccountJoin.role == 'owner' + ).first() + + if not tenant_account_join: + click.echo(click.style('Tenant owner not found: {}'.format(tenant.id), fg='red')) + continue + + # convert to workflow + workflow_converter.convert_to_workflow( + app_model=app, + account_id=tenant_account_join.account_id + ) + + click.echo(click.style('Converted app: {}'.format(app.id), fg='green')) + except Exception as e: + logging.exception('Convert app error: {}'.format(app.id)) + click.echo( + click.style('Convert app error: {} {}'.format(e.__class__.__name__, + str(e)), fg='red')) + + click.echo(click.style('Congratulations! Converted {} workflow chatbot apps.'.format(len(proceeded_app_ids)), fg='green')) + + def register_commands(app): app.cli.add_command(reset_password) app.cli.add_command(reset_email) app.cli.add_command(reset_encrypt_key_pair) app.cli.add_command(vdb_migrate) app.cli.add_command(convert_to_agent_apps) + app.cli.add_command(convert_to_workflow_chatbot_apps) diff --git a/api/core/application_manager.py b/api/core/application_manager.py index 77bb81b0da..ea0c85427d 100644 --- a/api/core/application_manager.py +++ b/api/core/application_manager.py @@ -235,12 +235,15 @@ class ApplicationManager: logger.exception(e) raise e - def convert_from_app_model_config_dict(self, tenant_id: str, app_model_config_dict: dict) \ + def convert_from_app_model_config_dict(self, tenant_id: str, + app_model_config_dict: dict, + skip_check: bool = False) \ -> AppOrchestrationConfigEntity: """ Convert app model config dict to entity. :param tenant_id: tenant ID :param app_model_config_dict: app model config dict + :param skip_check: skip check :raises ProviderTokenNotInitError: provider token not init error :return: app orchestration config entity """ @@ -268,24 +271,28 @@ class ApplicationManager: ) if model_credentials is None: - raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.") + if not skip_check: + raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.") + else: + model_credentials = {} - # check model - provider_model = provider_model_bundle.configuration.get_provider_model( - model=copy_app_model_config_dict['model']['name'], - model_type=ModelType.LLM - ) + if not skip_check: + # check model + provider_model = provider_model_bundle.configuration.get_provider_model( + model=copy_app_model_config_dict['model']['name'], + model_type=ModelType.LLM + ) - if provider_model is None: - model_name = copy_app_model_config_dict['model']['name'] - raise ValueError(f"Model {model_name} not exist.") + if provider_model is None: + model_name = copy_app_model_config_dict['model']['name'] + raise ValueError(f"Model {model_name} not exist.") - if provider_model.status == ModelStatus.NO_CONFIGURE: - raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.") - elif provider_model.status == ModelStatus.NO_PERMISSION: - raise ModelCurrentlyNotSupportError(f"Dify Hosted OpenAI {model_name} currently not support.") - elif provider_model.status == ModelStatus.QUOTA_EXCEEDED: - raise QuotaExceededError(f"Model provider {provider_name} quota exceeded.") + if provider_model.status == ModelStatus.NO_CONFIGURE: + raise ProviderTokenNotInitError(f"Model {model_name} credentials is not initialized.") + elif provider_model.status == ModelStatus.NO_PERMISSION: + raise ModelCurrentlyNotSupportError(f"Dify Hosted OpenAI {model_name} currently not support.") + elif provider_model.status == ModelStatus.QUOTA_EXCEEDED: + raise QuotaExceededError(f"Model provider {provider_name} quota exceeded.") # model config completion_params = copy_app_model_config_dict['model'].get('completion_params') @@ -309,7 +316,7 @@ class ApplicationManager: model_credentials ) - if not model_schema: + if not skip_check and not model_schema: raise ValueError(f"Model {model_name} not exist.") properties['model_config'] = ModelConfigEntity( diff --git a/api/core/entities/application_entities.py b/api/core/entities/application_entities.py index 667940f184..f5ea4d1eb0 100644 --- a/api/core/entities/application_entities.py +++ b/api/core/entities/application_entities.py @@ -15,7 +15,7 @@ class ModelConfigEntity(BaseModel): """ provider: str model: str - model_schema: AIModelEntity + model_schema: Optional[AIModelEntity] = None mode: str provider_model_bundle: ProviderModelBundle credentials: dict[str, Any] = {} diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index 31df58a583..1d3cbe2e0e 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -13,12 +13,11 @@ from core.entities.application_entities import ( ) from core.helper import encrypter from core.model_runtime.entities.llm_entities import LLMMode -from core.model_runtime.utils import helper +from core.model_runtime.utils.encoders import jsonable_encoder from core.prompt.simple_prompt_transform import SimplePromptTransform from core.workflow.entities.NodeEntities import NodeType from core.workflow.nodes.end.entities import EndNodeOutputType from extensions.ext_database import db -from models.account import Account from models.api_based_extension import APIBasedExtension, APIBasedExtensionPoint from models.model import App, AppMode, ChatbotAppEngine from models.workflow import Workflow, WorkflowType @@ -29,7 +28,7 @@ class WorkflowConverter: App Convert to Workflow Mode """ - def convert_to_workflow(self, app_model: App, account: Account) -> Workflow: + def convert_to_workflow(self, app_model: App, account_id: str) -> Workflow: """ Convert to workflow mode @@ -40,7 +39,7 @@ class WorkflowConverter: - completion app (for migration) :param app_model: App instance - :param account: Account instance + :param account_id: Account ID :return: workflow instance """ # get new app mode @@ -53,7 +52,8 @@ class WorkflowConverter: application_manager = ApplicationManager() app_orchestration_config_entity = application_manager.convert_from_app_model_config_dict( tenant_id=app_model.tenant_id, - app_model_config_dict=app_model_config.to_dict() + app_model_config_dict=app_model_config.to_dict(), + skip_check=True ) # init workflow graph @@ -122,7 +122,7 @@ class WorkflowConverter: type=WorkflowType.from_app_mode(new_app_mode).value, version='draft', graph=json.dumps(graph), - created_by=account.id + created_by=account_id ) db.session.add(workflow) @@ -130,6 +130,7 @@ class WorkflowConverter: # create new app model config record new_app_model_config = app_model_config.copy() + new_app_model_config.id = None new_app_model_config.external_data_tools = '' new_app_model_config.model = '' new_app_model_config.user_input_form = '' @@ -147,6 +148,9 @@ class WorkflowConverter: db.session.add(new_app_model_config) db.session.commit() + app_model.app_model_config_id = new_app_model_config.id + db.session.commit() + return workflow def _convert_to_start_node(self, variables: list[VariableEntity]) -> dict: @@ -161,7 +165,7 @@ class WorkflowConverter: "data": { "title": "START", "type": NodeType.START.value, - "variables": [helper.dump_model(v) for v in variables] + "variables": [jsonable_encoder(v) for v in variables] } } @@ -369,7 +373,10 @@ class WorkflowConverter: ] else: advanced_chat_prompt_template = prompt_template.advanced_chat_prompt_template - prompts = [helper.dump_model(m) for m in advanced_chat_prompt_template.messages] \ + prompts = [{ + "role": m.role.value, + "text": m.text + } for m in advanced_chat_prompt_template.messages] \ if advanced_chat_prompt_template else [] # Completion Model else: diff --git a/api/services/workflow_service.py b/api/services/workflow_service.py index 0cb398225d..bd88f3cbe2 100644 --- a/api/services/workflow_service.py +++ b/api/services/workflow_service.py @@ -79,6 +79,6 @@ class WorkflowService: # convert to workflow mode workflow_converter = WorkflowConverter() - workflow = workflow_converter.convert_to_workflow(app_model=app_model, account=account) + workflow = workflow_converter.convert_to_workflow(app_model=app_model, account_id=account.id) return workflow diff --git a/api/tests/unit_tests/services/workflow/test_workflow_converter.py b/api/tests/unit_tests/services/workflow/test_workflow_converter.py index ee9e5eb2fa..d4edc73410 100644 --- a/api/tests/unit_tests/services/workflow/test_workflow_converter.py +++ b/api/tests/unit_tests/services/workflow/test_workflow_converter.py @@ -41,6 +41,8 @@ def test__convert_to_start_node(default_variables): result = WorkflowConverter()._convert_to_start_node(default_variables) # assert + assert isinstance(result["data"]["variables"][0]["type"], str) + assert result["data"]["variables"][0]["type"] == "text-input" assert result["data"]["variables"][0]["variable"] == "text-input" assert result["data"]["variables"][1]["variable"] == "paragraph" assert result["data"]["variables"][2]["variable"] == "select"