diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 458fa5098f..c7f3a598ca 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -43,7 +43,7 @@ class ChatMessageAudioApi(Resource): try: response = AudioService.transcript_asr( - tenant_id=app_model.tenant_id, + app_model=app_model, file=file, end_user=None, ) @@ -83,9 +83,9 @@ class ChatMessageTextApi(Resource): def post(self, app_model): try: response = AudioService.transcript_tts( - tenant_id=app_model.tenant_id, + app_model=app_model, text=request.form['text'], - voice=request.form['voice'] if request.form['voice'] else app_model.app_model_config.text_to_speech_dict.get('voice'), + voice=request.form.get('voice'), streaming=False ) diff --git a/api/controllers/console/explore/audio.py b/api/controllers/console/explore/audio.py index dc546ce0dd..34ce1ec1ee 100644 --- a/api/controllers/console/explore/audio.py +++ b/api/controllers/console/explore/audio.py @@ -32,16 +32,12 @@ from services.errors.audio import ( class ChatAudioApi(InstalledAppResource): def post(self, installed_app): app_model = installed_app.app - app_model_config: AppModelConfig = app_model.app_model_config - - if not app_model_config.speech_to_text_dict['enabled']: - raise AppUnavailableError() file = request.files['file'] try: response = AudioService.transcript_asr( - tenant_id=app_model.tenant_id, + app_model=app_model, file=file, end_user=None ) @@ -76,16 +72,12 @@ class ChatAudioApi(InstalledAppResource): class ChatTextApi(InstalledAppResource): def post(self, installed_app): app_model = installed_app.app - app_model_config: AppModelConfig = app_model.app_model_config - - if not app_model_config.text_to_speech_dict['enabled']: - raise AppUnavailableError() try: response = AudioService.transcript_tts( - tenant_id=app_model.tenant_id, + app_model=app_model, text=request.form['text'], - voice=request.form['voice'] if request.form['voice'] else app_model.app_model_config.text_to_speech_dict.get('voice'), + voice=request.form.get('voice'), streaming=False ) return {'data': response.data.decode('latin1')} diff --git a/api/controllers/console/explore/parameter.py b/api/controllers/console/explore/parameter.py index c4afb0b923..0239742a4a 100644 --- a/api/controllers/console/explore/parameter.py +++ b/api/controllers/console/explore/parameter.py @@ -4,9 +4,10 @@ from flask import current_app from flask_restful import fields, marshal_with from controllers.console import api +from controllers.console.app.error import AppUnavailableError from controllers.console.explore.wraps import InstalledAppResource from extensions.ext_database import db -from models.model import AppModelConfig, InstalledApp +from models.model import AppModelConfig, InstalledApp, AppMode from models.tools import ApiToolProvider @@ -45,30 +46,55 @@ class AppParameterApi(InstalledAppResource): def get(self, installed_app: InstalledApp): """Retrieve app parameters.""" app_model = installed_app.app - app_model_config = app_model.app_model_config + + if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]: + workflow = app_model.workflow + if workflow is None: + raise AppUnavailableError() + + features_dict = workflow.features_dict + user_input_form = workflow.user_input_form + else: + app_model_config = app_model.app_model_config + features_dict = app_model_config.to_dict() + + user_input_form = features_dict.get('user_input_form', []) return { - 'opening_statement': app_model_config.opening_statement, - 'suggested_questions': app_model_config.suggested_questions_list, - 'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict, - 'speech_to_text': app_model_config.speech_to_text_dict, - 'text_to_speech': app_model_config.text_to_speech_dict, - 'retriever_resource': app_model_config.retriever_resource_dict, - 'annotation_reply': app_model_config.annotation_reply_dict, - 'more_like_this': app_model_config.more_like_this_dict, - 'user_input_form': app_model_config.user_input_form_list, - 'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict, - 'file_upload': app_model_config.file_upload_dict, + 'opening_statement': features_dict.get('opening_statement'), + 'suggested_questions': features_dict.get('suggested_questions', []), + 'suggested_questions_after_answer': features_dict.get('suggested_questions_after_answer', + {"enabled": False}), + 'speech_to_text': features_dict.get('speech_to_text', {"enabled": False}), + 'text_to_speech': features_dict.get('text_to_speech', {"enabled": False}), + 'retriever_resource': features_dict.get('retriever_resource', {"enabled": False}), + 'annotation_reply': features_dict.get('annotation_reply', {"enabled": False}), + 'more_like_this': features_dict.get('more_like_this', {"enabled": False}), + 'user_input_form': user_input_form, + 'sensitive_word_avoidance': features_dict.get('sensitive_word_avoidance', + {"enabled": False, "type": "", "configs": []}), + 'file_upload': features_dict.get('file_upload', {"image": { + "enabled": False, + "number_limits": 3, + "detail": "high", + "transfer_methods": ["remote_url", "local_file"] + }}), 'system_parameters': { 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT') } } + class ExploreAppMetaApi(InstalledAppResource): def get(self, installed_app: InstalledApp): """Get app meta""" app_model_config: AppModelConfig = installed_app.app.app_model_config + if not app_model_config: + return { + 'tool_icons': {} + } + agent_config = app_model_config.agent_mode_dict or {} meta = { 'tool_icons': {} @@ -77,7 +103,7 @@ class ExploreAppMetaApi(InstalledAppResource): # get all tools tools = agent_config.get('tools', []) url_prefix = (current_app.config.get("CONSOLE_API_URL") - + "/console/api/workspaces/current/tool-provider/builtin/") + + "/console/api/workspaces/current/tool-provider/builtin/") for tool in tools: keys = list(tool.keys()) if len(keys) >= 4: @@ -94,12 +120,14 @@ class ExploreAppMetaApi(InstalledAppResource): ) meta['tool_icons'][tool_name] = json.loads(provider.icon) except: - meta['tool_icons'][tool_name] = { + meta['tool_icons'][tool_name] = { "background": "#252525", "content": "\ud83d\ude01" } return meta -api.add_resource(AppParameterApi, '/installed-apps//parameters', endpoint='installed_app_parameters') + +api.add_resource(AppParameterApi, '/installed-apps//parameters', + endpoint='installed_app_parameters') api.add_resource(ExploreAppMetaApi, '/installed-apps//meta', endpoint='installed_app_meta') diff --git a/api/controllers/service_api/app/app.py b/api/controllers/service_api/app/app.py index a3151fc4a2..76708716c2 100644 --- a/api/controllers/service_api/app/app.py +++ b/api/controllers/service_api/app/app.py @@ -4,9 +4,10 @@ from flask import current_app from flask_restful import fields, marshal_with, Resource from controllers.service_api import api +from controllers.service_api.app.error import AppUnavailableError from controllers.service_api.wraps import validate_app_token from extensions.ext_database import db -from models.model import App, AppModelConfig +from models.model import App, AppModelConfig, AppMode from models.tools import ApiToolProvider @@ -46,31 +47,55 @@ class AppParameterApi(Resource): @marshal_with(parameters_fields) def get(self, app_model: App): """Retrieve app parameters.""" - app_model_config = app_model.app_model_config + if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]: + workflow = app_model.workflow + if workflow is None: + raise AppUnavailableError() + + features_dict = workflow.features_dict + user_input_form = workflow.user_input_form + else: + app_model_config = app_model.app_model_config + features_dict = app_model_config.to_dict() + + user_input_form = features_dict.get('user_input_form', []) return { - 'opening_statement': app_model_config.opening_statement, - 'suggested_questions': app_model_config.suggested_questions_list, - 'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict, - 'speech_to_text': app_model_config.speech_to_text_dict, - 'text_to_speech': app_model_config.text_to_speech_dict, - 'retriever_resource': app_model_config.retriever_resource_dict, - 'annotation_reply': app_model_config.annotation_reply_dict, - 'more_like_this': app_model_config.more_like_this_dict, - 'user_input_form': app_model_config.user_input_form_list, - 'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict, - 'file_upload': app_model_config.file_upload_dict, + 'opening_statement': features_dict.get('opening_statement'), + 'suggested_questions': features_dict.get('suggested_questions', []), + 'suggested_questions_after_answer': features_dict.get('suggested_questions_after_answer', + {"enabled": False}), + 'speech_to_text': features_dict.get('speech_to_text', {"enabled": False}), + 'text_to_speech': features_dict.get('text_to_speech', {"enabled": False}), + 'retriever_resource': features_dict.get('retriever_resource', {"enabled": False}), + 'annotation_reply': features_dict.get('annotation_reply', {"enabled": False}), + 'more_like_this': features_dict.get('more_like_this', {"enabled": False}), + 'user_input_form': user_input_form, + 'sensitive_word_avoidance': features_dict.get('sensitive_word_avoidance', + {"enabled": False, "type": "", "configs": []}), + 'file_upload': features_dict.get('file_upload', {"image": { + "enabled": False, + "number_limits": 3, + "detail": "high", + "transfer_methods": ["remote_url", "local_file"] + }}), 'system_parameters': { 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT') } } + class AppMetaApi(Resource): @validate_app_token def get(self, app_model: App): """Get app meta""" app_model_config: AppModelConfig = app_model.app_model_config + if not app_model_config: + return { + 'tool_icons': {} + } + agent_config = app_model_config.agent_mode_dict or {} meta = { 'tool_icons': {} diff --git a/api/controllers/service_api/app/audio.py b/api/controllers/service_api/app/audio.py index f6cad501f0..57edab4090 100644 --- a/api/controllers/service_api/app/audio.py +++ b/api/controllers/service_api/app/audio.py @@ -33,18 +33,13 @@ from services.errors.audio import ( class AudioApi(Resource): @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.FORM)) def post(self, app_model: App, end_user: EndUser): - app_model_config: AppModelConfig = app_model.app_model_config - - if not app_model_config.speech_to_text_dict['enabled']: - raise AppUnavailableError() - file = request.files['file'] try: response = AudioService.transcript_asr( - tenant_id=app_model.tenant_id, + app_model=app_model, file=file, - end_user=end_user.get_id() + end_user=end_user ) return response @@ -79,15 +74,16 @@ class TextApi(Resource): def post(self, app_model: App, end_user: EndUser): parser = reqparse.RequestParser() parser.add_argument('text', type=str, required=True, nullable=False, location='json') + parser.add_argument('voice', type=str, location='json') parser.add_argument('streaming', type=bool, required=False, nullable=False, location='json') args = parser.parse_args() try: response = AudioService.transcript_tts( - tenant_id=app_model.tenant_id, + app_model=app_model, text=args['text'], - end_user=end_user.get_id(), - voice=app_model.app_model_config.text_to_speech_dict.get('voice'), + end_user=end_user, + voice=args.get('voice'), streaming=args['streaming'] ) diff --git a/api/controllers/web/app.py b/api/controllers/web/app.py index 25492b1143..07ce098298 100644 --- a/api/controllers/web/app.py +++ b/api/controllers/web/app.py @@ -4,9 +4,10 @@ from flask import current_app from flask_restful import fields, marshal_with from controllers.web import api +from controllers.web.error import AppUnavailableError from controllers.web.wraps import WebApiResource from extensions.ext_database import db -from models.model import App, AppModelConfig +from models.model import App, AppModelConfig, AppMode from models.tools import ApiToolProvider @@ -44,30 +45,52 @@ class AppParameterApi(WebApiResource): @marshal_with(parameters_fields) def get(self, app_model: App, end_user): """Retrieve app parameters.""" - app_model_config = app_model.app_model_config + if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]: + workflow = app_model.workflow + if workflow is None: + raise AppUnavailableError() + + features_dict = workflow.features_dict + user_input_form = workflow.user_input_form + else: + app_model_config = app_model.app_model_config + features_dict = app_model_config.to_dict() + + user_input_form = features_dict.get('user_input_form', []) return { - 'opening_statement': app_model_config.opening_statement, - 'suggested_questions': app_model_config.suggested_questions_list, - 'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict, - 'speech_to_text': app_model_config.speech_to_text_dict, - 'text_to_speech': app_model_config.text_to_speech_dict, - 'retriever_resource': app_model_config.retriever_resource_dict, - 'annotation_reply': app_model_config.annotation_reply_dict, - 'more_like_this': app_model_config.more_like_this_dict, - 'user_input_form': app_model_config.user_input_form_list, - 'sensitive_word_avoidance': app_model_config.sensitive_word_avoidance_dict, - 'file_upload': app_model_config.file_upload_dict, + 'opening_statement': features_dict.get('opening_statement'), + 'suggested_questions': features_dict.get('suggested_questions', []), + 'suggested_questions_after_answer': features_dict.get('suggested_questions_after_answer', + {"enabled": False}), + 'speech_to_text': features_dict.get('speech_to_text', {"enabled": False}), + 'text_to_speech': features_dict.get('text_to_speech', {"enabled": False}), + 'retriever_resource': features_dict.get('retriever_resource', {"enabled": False}), + 'annotation_reply': features_dict.get('annotation_reply', {"enabled": False}), + 'more_like_this': features_dict.get('more_like_this', {"enabled": False}), + 'user_input_form': user_input_form, + 'sensitive_word_avoidance': features_dict.get('sensitive_word_avoidance', + {"enabled": False, "type": "", "configs": []}), + 'file_upload': features_dict.get('file_upload', {"image": { + "enabled": False, + "number_limits": 3, + "detail": "high", + "transfer_methods": ["remote_url", "local_file"] + }}), 'system_parameters': { 'image_file_size_limit': current_app.config.get('UPLOAD_IMAGE_FILE_SIZE_LIMIT') } } + class AppMeta(WebApiResource): def get(self, app_model: App, end_user): """Get app meta""" app_model_config: AppModelConfig = app_model.app_model_config + if not app_model_config: + raise AppUnavailableError() + agent_config = app_model_config.agent_mode_dict or {} meta = { 'tool_icons': {} diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index 4e677ae288..8b8ab8f090 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -31,16 +31,11 @@ from services.errors.audio import ( class AudioApi(WebApiResource): def post(self, app_model: App, end_user): - app_model_config: AppModelConfig = app_model.app_model_config - - if not app_model_config.speech_to_text_dict['enabled']: - raise AppUnavailableError() - file = request.files['file'] try: response = AudioService.transcript_asr( - tenant_id=app_model.tenant_id, + app_model=app_model, file=file, end_user=end_user ) @@ -74,17 +69,12 @@ class AudioApi(WebApiResource): class TextApi(WebApiResource): def post(self, app_model: App, end_user): - app_model_config: AppModelConfig = app_model.app_model_config - - if not app_model_config.text_to_speech_dict['enabled']: - raise AppUnavailableError() - try: response = AudioService.transcript_tts( - tenant_id=app_model.tenant_id, + app_model=app_model, text=request.form['text'], end_user=end_user.external_user_id, - voice=request.form['voice'] if request.form['voice'] else app_model.app_model_config.text_to_speech_dict.get('voice'), + voice=request.form.get('voice'), streaming=False ) diff --git a/api/controllers/web/site.py b/api/controllers/web/site.py index d8e2d59707..bf3536d276 100644 --- a/api/controllers/web/site.py +++ b/api/controllers/web/site.py @@ -83,7 +83,3 @@ class AppSiteInfo: 'remove_webapp_brand': remove_webapp_brand, 'replace_webapp_logo': replace_webapp_logo, } - - if app.enable_site and site.prompt_public: - app_model_config = app.app_model_config - self.model_config = app_model_config diff --git a/api/core/file/message_file_parser.py b/api/core/file/message_file_parser.py index 1b7b8b87da..c132073578 100644 --- a/api/core/file/message_file_parser.py +++ b/api/core/file/message_file_parser.py @@ -96,16 +96,16 @@ class MessageFileParser: # return all file objs return new_files - def transform_message_files(self, files: list[MessageFile], app_model_config: Optional[AppModelConfig]) -> list[FileObj]: + def transform_message_files(self, files: list[MessageFile], file_upload_config: Optional[dict]) -> list[FileObj]: """ transform message files :param files: - :param app_model_config: + :param file_upload_config: :return: """ # transform files to file objs - type_file_objs = self._to_file_objs(files, app_model_config.file_upload_dict) + type_file_objs = self._to_file_objs(files, file_upload_config) # return all file objs return [file_obj for file_objs in type_file_objs.values() for file_obj in file_objs] diff --git a/api/core/memory/token_buffer_memory.py b/api/core/memory/token_buffer_memory.py index 4d44ac3818..f9200dcc71 100644 --- a/api/core/memory/token_buffer_memory.py +++ b/api/core/memory/token_buffer_memory.py @@ -10,7 +10,7 @@ from core.model_runtime.entities.message_entities import ( from core.model_runtime.entities.model_entities import ModelType from core.model_runtime.model_providers import model_provider_factory from extensions.ext_database import db -from models.model import Conversation, Message +from models.model import Conversation, Message, AppMode class TokenBufferMemory: @@ -44,7 +44,10 @@ class TokenBufferMemory: files = message.message_files if files: file_objs = message_file_parser.transform_message_files( - files, message.app_model_config + files, + message.app_model_config.file_upload_dict + if self.conversation.mode not in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value] + else message.workflow_run.workflow.features_dict.get('file_upload', {}) ) if not file_objs: diff --git a/api/models/model.py b/api/models/model.py index c6409c61ed..e514ea729b 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -82,9 +82,10 @@ class App(db.Model): @property def app_model_config(self) -> Optional['AppModelConfig']: - app_model_config = db.session.query(AppModelConfig).filter( - AppModelConfig.id == self.app_model_config_id).first() - return app_model_config + if self.app_model_config_id: + return db.session.query(AppModelConfig).filter(AppModelConfig.id == self.app_model_config_id).first() + + return None @property def workflow(self): diff --git a/api/models/workflow.py b/api/models/workflow.py index c38c1dd610..ff4e944e29 100644 --- a/api/models/workflow.py +++ b/api/models/workflow.py @@ -129,6 +129,22 @@ class Workflow(db.Model): def features_dict(self): return self.features if not self.features else json.loads(self.features) + def user_input_form(self): + # get start node from graph + if not self.graph: + return [] + + graph_dict = self.graph_dict + if 'nodes' not in graph_dict: + return [] + + start_node = next((node for node in graph_dict['nodes'] if node['type'] == 'start'), None) + if not start_node: + return [] + + # get user_input_form from start node + return start_node.get('variables', []) + class WorkflowRunTriggeredFrom(Enum): """ diff --git a/api/services/app_service.py b/api/services/app_service.py index 7dd5d770ea..e0a7835cb7 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -175,12 +175,17 @@ class AppService: if workflow: # init draft workflow workflow_service = WorkflowService() - workflow_service.sync_draft_workflow( + draft_workflow = workflow_service.sync_draft_workflow( app_model=app, graph=workflow.get('graph'), features=workflow.get('features'), account=account ) + workflow_service.publish_workflow( + app_model=app, + account=account, + draft_workflow=draft_workflow + ) if model_config_data: app_model_config = AppModelConfig() diff --git a/api/services/audio_service.py b/api/services/audio_service.py index a9fe65df6f..0123666644 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -5,6 +5,7 @@ from werkzeug.datastructures import FileStorage from core.model_manager import ModelManager from core.model_runtime.entities.model_entities import ModelType +from models.model import AppModelConfig, App, AppMode from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, @@ -20,7 +21,21 @@ ALLOWED_EXTENSIONS = ['mp3', 'mp4', 'mpeg', 'mpga', 'm4a', 'wav', 'webm', 'amr'] class AudioService: @classmethod - def transcript_asr(cls, tenant_id: str, file: FileStorage, end_user: Optional[str] = None): + def transcript_asr(cls, app_model: App, file: FileStorage, end_user: Optional[str] = None): + if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]: + workflow = app_model.workflow + if workflow is None: + raise ValueError("Speech to text is not enabled") + + features_dict = workflow.features_dict + if 'speech_to_text' not in features_dict or not features_dict['speech_to_text'].get('enabled'): + raise ValueError("Speech to text is not enabled") + else: + app_model_config: AppModelConfig = app_model.app_model_config + + if not app_model_config.speech_to_text_dict['enabled']: + raise ValueError("Speech to text is not enabled") + if file is None: raise NoAudioUploadedServiceError() @@ -37,7 +52,7 @@ class AudioService: model_manager = ModelManager() model_instance = model_manager.get_default_model_instance( - tenant_id=tenant_id, + tenant_id=app_model.tenant_id, model_type=ModelType.SPEECH2TEXT ) if model_instance is None: @@ -49,17 +64,41 @@ class AudioService: return {"text": model_instance.invoke_speech2text(file=buffer, user=end_user)} @classmethod - def transcript_tts(cls, tenant_id: str, text: str, voice: str, streaming: bool, end_user: Optional[str] = None): + def transcript_tts(cls, app_model: App, text: str, streaming: bool, end_user: Optional[str] = None): + if app_model.mode in [AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value]: + workflow = app_model.workflow + if workflow is None: + raise ValueError("TTS is not enabled") + + features_dict = workflow.features_dict + if 'text_to_speech' not in features_dict or not features_dict['text_to_speech'].get('enabled'): + raise ValueError("TTS is not enabled") + + voice = features_dict['text_to_speech'].get('voice') + else: + text_to_speech_dict = app_model.app_model_config.text_to_speech_dict + + if not text_to_speech_dict.get('enabled'): + raise ValueError("TTS is not enabled") + + voice = text_to_speech_dict.get('voice'), + model_manager = ModelManager() model_instance = model_manager.get_default_model_instance( - tenant_id=tenant_id, + tenant_id=app_model.tenant_id, model_type=ModelType.TTS ) if model_instance is None: raise ProviderNotSupportTextToSpeechServiceError() try: - return model_instance.invoke_tts(content_text=text.strip(), user=end_user, streaming=streaming, tenant_id=tenant_id, voice=voice) + return model_instance.invoke_tts( + content_text=text.strip(), + user=end_user, + streaming=streaming, + tenant_id=app_model.tenant_id, + voice=voice + ) except Exception as e: raise e