From 607f9eda354f1920645a8f8e4c91b8c3d3fbe03c Mon Sep 17 00:00:00 2001 From: nan jiang Date: Mon, 4 Mar 2024 13:32:17 +0800 Subject: [PATCH 01/13] Fix/app runner typo (#2661) --- api/core/app_runner/app_runner.py | 2 +- api/core/app_runner/basic_app_runner.py | 2 +- api/core/features/assistant_cot_runner.py | 2 +- api/core/features/assistant_fc_runner.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/core/app_runner/app_runner.py b/api/core/app_runner/app_runner.py index 2b8ddc5d4e..f9678b372f 100644 --- a/api/core/app_runner/app_runner.py +++ b/api/core/app_runner/app_runner.py @@ -84,7 +84,7 @@ class AppRunner: return rest_tokens - def recale_llm_max_tokens(self, model_config: ModelConfigEntity, + def recalc_llm_max_tokens(self, model_config: ModelConfigEntity, prompt_messages: list[PromptMessage]): # recalc max_tokens if sum(prompt_token + max_tokens) over model token limit model_type_instance = model_config.provider_model_bundle.model_type_instance diff --git a/api/core/app_runner/basic_app_runner.py b/api/core/app_runner/basic_app_runner.py index 99df249ddf..83f4f6929a 100644 --- a/api/core/app_runner/basic_app_runner.py +++ b/api/core/app_runner/basic_app_runner.py @@ -181,7 +181,7 @@ class BasicApplicationRunner(AppRunner): return # Re-calculate the max tokens if sum(prompt_token + max_tokens) over model token limit - self.recale_llm_max_tokens( + self.recalc_llm_max_tokens( model_config=app_orchestration_config.model_config, prompt_messages=prompt_messages ) diff --git a/api/core/features/assistant_cot_runner.py b/api/core/features/assistant_cot_runner.py index 809834c8cb..09ab27109b 100644 --- a/api/core/features/assistant_cot_runner.py +++ b/api/core/features/assistant_cot_runner.py @@ -131,7 +131,7 @@ class AssistantCotApplicationRunner(BaseAssistantApplicationRunner): ) # recale llm max tokens - self.recale_llm_max_tokens(self.model_config, prompt_messages) + self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Generator[LLMResultChunk, None, None] = model_instance.invoke_llm( prompt_messages=prompt_messages, diff --git a/api/core/features/assistant_fc_runner.py b/api/core/features/assistant_fc_runner.py index 7ad9d7bd2a..afb312341d 100644 --- a/api/core/features/assistant_fc_runner.py +++ b/api/core/features/assistant_fc_runner.py @@ -106,7 +106,7 @@ class AssistantFunctionCallApplicationRunner(BaseAssistantApplicationRunner): ) # recale llm max tokens - self.recale_llm_max_tokens(self.model_config, prompt_messages) + self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Union[Generator[LLMResultChunk, None, None], LLMResult] = model_instance.invoke_llm( prompt_messages=prompt_messages, From 76da66fb7e987cd5555585b2aca490aeb62f1fe5 Mon Sep 17 00:00:00 2001 From: takatost Date: Mon, 4 Mar 2024 14:06:54 +0800 Subject: [PATCH 02/13] fix: fix import from explore apps err when OpenAI not inited (#2671) --- api/controllers/console/app/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 59a7535144..21ce9cb6af 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -129,7 +129,7 @@ class AppListApi(Resource): "No Default System Reasoning Model available. Please configure " "in the Settings -> Model Provider.") else: - model_config_dict["model"]["provider"] = default_model_entity.provider + model_config_dict["model"]["provider"] = default_model_entity.provider.provider model_config_dict["model"]["name"] = default_model_entity.model model_configuration = AppModelConfigService.validate_configuration( From 83a6b0c6260163afeefc24df1dac59b0b2f7f30c Mon Sep 17 00:00:00 2001 From: Chenhe Gu Date: Mon, 4 Mar 2024 14:10:39 +0800 Subject: [PATCH 03/13] Doc/update license (#2666) --- LICENSE | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/LICENSE b/LICENSE index 8c9dd2961c..071ef42bda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,24 +1,26 @@ -# Dify Open Source License +# Open Source License -The Dify project is licensed under the Apache License 2.0, with the following additional conditions: +Dify is licensed under the Apache License 2.0, with the following additional conditions: -1. Dify is permitted to be used for commercialization, such as using Dify as a "backend-as-a-service" for your other applications, or delivering it to enterprises as an application development platform. However, when the following conditions are met, you must contact the producer to obtain a commercial license: +1. Dify may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. Should the conditions below be met, a commercial license must be obtained from the producer: -a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify.AI source code to operate a multi-tenant SaaS service that is similar to the Dify.AI service edition. -b. LOGO and copyright information: In the process of using Dify, you may not remove or modify the LOGO or copyright information in the Dify console. +a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. + - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations. + +b. LOGO and copyright information: In the process of using Dify's frontend components, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend components. Please contact business@dify.ai by email to inquire about licensing matters. -2. As a contributor, you should agree that your contributed code: +2. As a contributor, you should agree that: -a. The producer can adjust the open-source agreement to be more strict or relaxed. -b. Can be used for commercial purposes, such as Dify's cloud business. +a. The producer can adjust the open-source agreement to be more strict or relaxed as deemed necessary. +b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. -Apart from this, all other rights and restrictions follow the Apache License 2.0. If you need more detailed information, you can refer to the full version of Apache License 2.0. +Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. The interactive design of this product is protected by appearance patent. -© 2023 LangGenius, Inc. +© 2024 LangGenius, Inc. ---------- From 34387ec0f112eb54e5b906cae77f9062b9487fbb Mon Sep 17 00:00:00 2001 From: cola <45722758+xiangpingjiang@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:15:53 +0800 Subject: [PATCH 04/13] fix typo recale to recalc (#2670) --- api/core/features/assistant_cot_runner.py | 2 +- api/core/features/assistant_fc_runner.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/features/assistant_cot_runner.py b/api/core/features/assistant_cot_runner.py index 09ab27109b..8fcbff983d 100644 --- a/api/core/features/assistant_cot_runner.py +++ b/api/core/features/assistant_cot_runner.py @@ -130,7 +130,7 @@ class AssistantCotApplicationRunner(BaseAssistantApplicationRunner): input=query ) - # recale llm max tokens + # recalc llm max tokens self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Generator[LLMResultChunk, None, None] = model_instance.invoke_llm( diff --git a/api/core/features/assistant_fc_runner.py b/api/core/features/assistant_fc_runner.py index afb312341d..391e040c53 100644 --- a/api/core/features/assistant_fc_runner.py +++ b/api/core/features/assistant_fc_runner.py @@ -105,7 +105,7 @@ class AssistantFunctionCallApplicationRunner(BaseAssistantApplicationRunner): messages_ids=message_file_ids ) - # recale llm max tokens + # recalc llm max tokens self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Union[Generator[LLMResultChunk, None, None], LLMResult] = model_instance.invoke_llm( From 36686d7425e24828d6cdb4f7eb020801ad343b39 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:16:47 +0800 Subject: [PATCH 05/13] fix: test custom tool already exists without decrypting credentials (#2668) --- .../console/workspace/tool_providers.py | 2 + api/core/tools/tool/api_tool.py | 3 ++ api/services/tools_manage_service.py | 47 ++++++++++++++----- .../edit-custom-collection-modal/test-api.tsx | 1 + 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index c2c5286d51..817c75765a 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -259,6 +259,7 @@ class ToolApiProviderPreviousTestApi(Resource): parser = reqparse.RequestParser() parser.add_argument('tool_name', type=str, required=True, nullable=False, location='json') + parser.add_argument('provider_name', type=str, required=False, nullable=False, location='json') parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') parser.add_argument('parameters', type=dict, required=True, nullable=False, location='json') parser.add_argument('schema_type', type=str, required=True, nullable=False, location='json') @@ -268,6 +269,7 @@ class ToolApiProviderPreviousTestApi(Resource): return ToolManageService.test_api_tool_preview( current_user.current_tenant_id, + args['provider_name'] if args['provider_name'] else '', args['tool_name'], args['credentials'], args['parameters'], diff --git a/api/core/tools/tool/api_tool.py b/api/core/tools/tool/api_tool.py index d5a4bf20bd..31519734ed 100644 --- a/api/core/tools/tool/api_tool.py +++ b/api/core/tools/tool/api_tool.py @@ -1,6 +1,7 @@ import json from json import dumps from typing import Any, Union +from urllib.parse import urlencode import httpx import requests @@ -203,6 +204,8 @@ class ApiTool(Tool): if 'Content-Type' in headers: if headers['Content-Type'] == 'application/json': body = dumps(body) + elif headers['Content-Type'] == 'application/x-www-form-urlencoded': + body = urlencode(body) else: body = body diff --git a/api/services/tools_manage_service.py b/api/services/tools_manage_service.py index 7e305c3f7b..0e3d481640 100644 --- a/api/services/tools_manage_service.py +++ b/api/services/tools_manage_service.py @@ -498,12 +498,16 @@ class ToolManageService: @staticmethod def test_api_tool_preview( - tenant_id: str, tool_name: str, credentials: dict, parameters: dict, schema_type: str, schema: str + tenant_id: str, + provider_name: str, + tool_name: str, + credentials: dict, + parameters: dict, + schema_type: str, + schema: str ): """ test api tool before adding api tool provider - - 1. parse schema into tool bundle """ if schema_type not in [member.value for member in ApiProviderSchemaType]: raise ValueError(f'invalid schema type {schema_type}') @@ -518,15 +522,21 @@ class ToolManageService: if tool_bundle is None: raise ValueError(f'invalid tool name {tool_name}') - # create a fake db provider - db_provider = ApiToolProvider( - tenant_id='', user_id='', name='', icon='', - schema=schema, - description='', - schema_type_str=ApiProviderSchemaType.OPENAPI.value, - tools_str=serialize_base_model_array(tool_bundles), - credentials_str=json.dumps(credentials), - ) + db_provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( + ApiToolProvider.tenant_id == tenant_id, + ApiToolProvider.name == provider_name, + ).first() + + if not db_provider: + # create a fake db provider + db_provider = ApiToolProvider( + tenant_id='', user_id='', name='', icon='', + schema=schema, + description='', + schema_type_str=ApiProviderSchemaType.OPENAPI.value, + tools_str=serialize_base_model_array(tool_bundles), + credentials_str=json.dumps(credentials), + ) if 'auth_type' not in credentials: raise ValueError('auth_type is required') @@ -539,6 +549,19 @@ class ToolManageService: # load tools into provider entity provider_controller.load_bundled_tools(tool_bundles) + # decrypt credentials + if db_provider.id: + tool_configuration = ToolConfiguration( + tenant_id=tenant_id, + provider_controller=provider_controller + ) + decrypted_credentials = tool_configuration.decrypt_tool_credentials(credentials) + # check if the credential has changed, save the original credential + masked_credentials = tool_configuration.mask_tool_credentials(decrypted_credentials) + for name, value in credentials.items(): + if name in masked_credentials and value == masked_credentials[name]: + credentials[name] = decrypted_credentials[name] + try: provider_controller.validate_credentials_format(credentials) # get tool diff --git a/web/app/components/tools/edit-custom-collection-modal/test-api.tsx b/web/app/components/tools/edit-custom-collection-modal/test-api.tsx index 620ceb8f48..791ac5edbf 100644 --- a/web/app/components/tools/edit-custom-collection-modal/test-api.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/test-api.tsx @@ -42,6 +42,7 @@ const TestApi: FC = ({ delete credentials.api_key_value } const data = { + provider_name: customCollection.provider, tool_name: toolName, credentials, schema_type: customCollection.schema_type, From 72ddedfc5c6bef92536f47462e375465ed201705 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 4 Mar 2024 14:17:00 +0800 Subject: [PATCH 06/13] fix: setup default filters while add credentials (#2669) --- api/core/tools/provider/builtin/bing/bing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/core/tools/provider/builtin/bing/bing.py b/api/core/tools/provider/builtin/bing/bing.py index 76ce630f22..ff131b26cd 100644 --- a/api/core/tools/provider/builtin/bing/bing.py +++ b/api/core/tools/provider/builtin/bing/bing.py @@ -16,7 +16,8 @@ class BingProvider(BuiltinToolProviderController): user_id='', tool_parameters={ "query": "test", - "result_type": "link" + "result_type": "link", + "enable_webpages": True, }, ) except Exception as e: From 05ce7b9d5e170ae0673d4495aca02c2f73792858 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Mon, 4 Mar 2024 15:20:20 +0800 Subject: [PATCH 07/13] fix: deep copy customColletion (#2673) --- .../edit-custom-collection-modal/index.tsx | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/web/app/components/tools/edit-custom-collection-modal/index.tsx b/web/app/components/tools/edit-custom-collection-modal/index.tsx index 45e0a014ce..89cde8ca47 100644 --- a/web/app/components/tools/edit-custom-collection-modal/index.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/index.tsx @@ -2,10 +2,9 @@ import type { FC } from 'react' import React, { useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' -import produce from 'immer' import { useDebounce, useGetState } from 'ahooks' -import { clone } from 'lodash-es' import cn from 'classnames' +import produce from 'immer' import { LinkExternal02, Settings01 } from '../../base/icons/src/vender/line/general' import type { Credential, CustomCollectionBackend, CustomParamSchema, Emoji } from '../types' import { AuthHeaderPrefix, AuthType } from '../types' @@ -116,14 +115,16 @@ const EditCustomCollectionModal: FC = ({ const [isShowTestApi, setIsShowTestApi] = useState(false) const handleSave = () => { - const postData = clone(customCollection) - delete postData.tools + // const postData = clone(customCollection) + const postData = produce(customCollection, (draft) => { + delete draft.tools - if (postData.credentials.auth_type === AuthType.none) { - delete postData.credentials.api_key_header - delete postData.credentials.api_key_header_prefix - delete postData.credentials.api_key_value - } + if (draft.credentials.auth_type === AuthType.none) { + delete draft.credentials.api_key_header + delete draft.credentials.api_key_header_prefix + delete draft.credentials.api_key_value + } + }) if (isAdd) { onAdd?.(postData) From f322d9bddb8e4668377ac819a588542c9fce3b41 Mon Sep 17 00:00:00 2001 From: waltcow Date: Mon, 4 Mar 2024 16:35:50 +0800 Subject: [PATCH 08/13] Fix vdb merge error (#2650) --- .../datasource/vdb/qdrant/qdrant_vector.py | 37 ++++++++++++------- api/requirements.txt | 2 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 2432931228..6bd4b5c340 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -231,21 +231,30 @@ class QdrantVector(BaseVector): def delete(self): from qdrant_client.http import models - filter = models.Filter( - must=[ - models.FieldCondition( - key="group_id", - match=models.MatchValue(value=self._group_id), + from qdrant_client.http.exceptions import UnexpectedResponse + + try: + filter = models.Filter( + must=[ + models.FieldCondition( + key="group_id", + match=models.MatchValue(value=self._group_id), + ), + ], + ) + self._client.delete( + collection_name=self._collection_name, + points_selector=FilterSelector( + filter=filter ), - ], - ) - self._client.delete( - collection_name=self._collection_name, - points_selector=FilterSelector( - filter=filter - ), - ) - + ) + except UnexpectedResponse as e: + # Collection does not exist, so return + if e.status_code == 404: + return + # Some other error occurred, so re-raise the exception + else: + raise e def delete_by_ids(self, ids: list[str]) -> None: from qdrant_client.http import models diff --git a/api/requirements.txt b/api/requirements.txt index fbadcdbf04..ae5c77137a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -52,7 +52,7 @@ safetensors==0.3.2 zhipuai==1.0.7 werkzeug~=3.0.1 pymilvus==2.3.0 -qdrant-client==1.6.4 +qdrant-client==1.7.3 cohere~=4.44 pyyaml~=6.0.1 numpy~=1.25.2 From 3631e53ff027a182e7174520e2d33874a088c57e Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:22:06 +0800 Subject: [PATCH 09/13] Feat/add annotation migrate (#2675) Co-authored-by: jyong --- api/commands.py | 111 +++++++++++++++++- .../datasource/vdb/milvus/milvus_vector.py | 3 +- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/api/commands.py b/api/commands.py index 45977cbb9e..250039a365 100644 --- a/api/commands.py +++ b/api/commands.py @@ -15,7 +15,7 @@ from libs.rsa import generate_key_pair from models.account import Tenant from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment from models.dataset import Document as DatasetDocument -from models.model import Account +from models.model import Account, App, AppAnnotationSetting, MessageAnnotation from models.provider import Provider, ProviderModel @@ -125,7 +125,114 @@ def reset_encrypt_key_pair(): @click.command('vdb-migrate', help='migrate vector db.') -def vdb_migrate(): +@click.option('--scope', default='all', prompt=False, help='The scope of vector database to migrate, Default is All.') +def vdb_migrate(scope: str): + if scope in ['knowledge', 'all']: + migrate_knowledge_vector_database() + if scope in ['annotation', 'all']: + migrate_annotation_vector_database() + + +def migrate_annotation_vector_database(): + """ + Migrate annotation datas to target vector database . + """ + click.echo(click.style('Start migrate annotation data.', fg='green')) + create_count = 0 + skipped_count = 0 + total_count = 0 + page = 1 + while True: + try: + # get apps info + apps = db.session.query(App).filter( + App.status == 'normal' + ).order_by(App.created_at.desc()).paginate(page=page, per_page=50) + except NotFound: + break + + page += 1 + for app in apps: + total_count = total_count + 1 + click.echo(f'Processing the {total_count} app {app.id}. ' + + f'{create_count} created, {skipped_count} skipped.') + try: + click.echo('Create app annotation index: {}'.format(app.id)) + app_annotation_setting = db.session.query(AppAnnotationSetting).filter( + AppAnnotationSetting.app_id == app.id + ).first() + + if not app_annotation_setting: + skipped_count = skipped_count + 1 + click.echo('App annotation setting is disabled: {}'.format(app.id)) + continue + # get dataset_collection_binding info + dataset_collection_binding = db.session.query(DatasetCollectionBinding).filter( + DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id + ).first() + if not dataset_collection_binding: + click.echo('App annotation collection binding is not exist: {}'.format(app.id)) + continue + annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all() + dataset = Dataset( + id=app.id, + tenant_id=app.tenant_id, + indexing_technique='high_quality', + embedding_model_provider=dataset_collection_binding.provider_name, + embedding_model=dataset_collection_binding.model_name, + collection_binding_id=dataset_collection_binding.id + ) + documents = [] + if annotations: + for annotation in annotations: + document = Document( + page_content=annotation.question, + metadata={ + "annotation_id": annotation.id, + "app_id": app.id, + "doc_id": annotation.id + } + ) + documents.append(document) + + vector = Vector(dataset, attributes=['doc_id', 'annotation_id', 'app_id']) + click.echo(f"Start to migrate annotation, app_id: {app.id}.") + + try: + vector.delete() + click.echo( + click.style(f'Successfully delete vector index for app: {app.id}.', + fg='green')) + except Exception as e: + click.echo( + click.style(f'Failed to delete vector index for app {app.id}.', + fg='red')) + raise e + if documents: + try: + click.echo(click.style( + f'Start to created vector index with {len(documents)} annotations for app {app.id}.', + fg='green')) + vector.create(documents) + click.echo( + click.style(f'Successfully created vector index for app {app.id}.', fg='green')) + except Exception as e: + click.echo(click.style(f'Failed to created vector index for app {app.id}.', fg='red')) + raise e + click.echo(f'Successfully migrated app annotation {app.id}.') + create_count += 1 + except Exception as e: + click.echo( + click.style('Create app annotation index error: {} {}'.format(e.__class__.__name__, str(e)), + fg='red')) + continue + + click.echo( + click.style(f'Congratulations! Create {create_count} app annotation indexes, and skipped {skipped_count} apps.', + fg='green')) + + +def migrate_knowledge_vector_database(): """ Migrate vector database datas to target vector database . """ diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 0fc8ed5a26..203b7eff37 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -140,7 +140,8 @@ class MilvusVector(BaseVector): connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) from pymilvus import utility - utility.drop_collection(self._collection_name, None, using=alias) + if utility.has_collection(self._collection_name, using=alias): + utility.drop_collection(self._collection_name, None, using=alias) def text_exists(self, id: str) -> bool: From bbf5f42c8791fc97062c5847729fd9f5e6390d0a Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 4 Mar 2024 17:25:31 +0800 Subject: [PATCH 10/13] fix: CE edition limits upload file nums (#2677) --- web/app/components/datasets/create/file-uploader/index.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/file-uploader/index.tsx b/web/app/components/datasets/create/file-uploader/index.tsx index 21acf114ed..43cab0e949 100644 --- a/web/app/components/datasets/create/file-uploader/index.tsx +++ b/web/app/components/datasets/create/file-uploader/index.tsx @@ -13,6 +13,7 @@ import { fetchFileUploadConfig } from '@/service/common' import { fetchSupportFileTypes } from '@/service/datasets' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n/language' +import { IS_CE_EDITION } from '@/config' const FILES_NUMBER_LIMIT = 20 @@ -180,7 +181,7 @@ const FileUploader = ({ if (!files.length) return false - if (files.length + fileList.length > FILES_NUMBER_LIMIT) { + if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) { notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) }) return false } From 65cfd4360aace19637d1f469aaef04d745eef0a7 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Mon, 4 Mar 2024 17:25:42 +0800 Subject: [PATCH 11/13] fix: typo in wecom tool (#2674) --- api/core/tools/provider/builtin/wecom/wecom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/tools/provider/builtin/wecom/wecom.py b/api/core/tools/provider/builtin/wecom/wecom.py index b50d3ff68a..6380061b4f 100644 --- a/api/core/tools/provider/builtin/wecom/wecom.py +++ b/api/core/tools/provider/builtin/wecom/wecom.py @@ -2,7 +2,7 @@ from core.tools.provider.builtin.wecom.tools.wecom_group_bot import WecomReposit from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -class GaodeProvider(BuiltinToolProviderController): +class WecomProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: WecomRepositoriesTool() pass From 8523b34be73b4c743b462a0dd416d73a9773361d Mon Sep 17 00:00:00 2001 From: Joshua <138381132+joshua20231026@users.noreply.github.com> Date: Mon, 4 Mar 2024 17:31:01 +0800 Subject: [PATCH 12/13] add jina-reranker-v1-base-en (#2676) --- .../model_providers/jina/jina.yaml | 5 +- .../model_providers/jina/rerank/__init__.py | 0 .../jina/rerank/jina-reranker-v1-base-en.yaml | 4 + .../model_providers/jina/rerank/rerank.py | 105 ++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 api/core/model_runtime/model_providers/jina/rerank/__init__.py create mode 100644 api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml create mode 100644 api/core/model_runtime/model_providers/jina/rerank/rerank.py diff --git a/api/core/model_runtime/model_providers/jina/jina.yaml b/api/core/model_runtime/model_providers/jina/jina.yaml index ad90344d53..935546234b 100644 --- a/api/core/model_runtime/model_providers/jina/jina.yaml +++ b/api/core/model_runtime/model_providers/jina/jina.yaml @@ -2,7 +2,7 @@ provider: jina label: en_US: Jina description: - en_US: Embedding Model Supported + en_US: Embedding and Rerank Model Supported icon_small: en_US: icon_s_en.svg icon_large: @@ -13,9 +13,10 @@ help: en_US: Get your API key from Jina AI zh_Hans: 从 Jina 获取 API Key url: - en_US: https://jina.ai/embeddings/ + en_US: https://jina.ai/ supported_model_types: - text-embedding + - rerank configurate_methods: - predefined-model provider_credential_schema: diff --git a/api/core/model_runtime/model_providers/jina/rerank/__init__.py b/api/core/model_runtime/model_providers/jina/rerank/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml b/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml new file mode 100644 index 0000000000..bd3f31fbd1 --- /dev/null +++ b/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml @@ -0,0 +1,4 @@ +model: jina-reranker-v1-base-en +model_type: rerank +model_properties: + context_size: 8192 diff --git a/api/core/model_runtime/model_providers/jina/rerank/rerank.py b/api/core/model_runtime/model_providers/jina/rerank/rerank.py new file mode 100644 index 0000000000..f644ea6512 --- /dev/null +++ b/api/core/model_runtime/model_providers/jina/rerank/rerank.py @@ -0,0 +1,105 @@ +from typing import Optional + +import httpx + +from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.rerank_model import RerankModel + + +class JinaRerankModel(RerankModel): + """ + Model class for Jina rerank model. + """ + + def _invoke(self, model: str, credentials: dict, + query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + user: Optional[str] = None) -> RerankResult: + """ + Invoke rerank model + + :param model: model name + :param credentials: model credentials + :param query: search query + :param docs: docs for reranking + :param score_threshold: score threshold + :param top_n: top n documents to return + :param user: unique user id + :return: rerank result + """ + if len(docs) == 0: + return RerankResult(model=model, docs=[]) + + try: + response = httpx.post( + "https://api.jina.ai/v1/rerank", + json={ + "model": model, + "query": query, + "documents": docs, + "top_n": top_n + }, + headers={"Authorization": f"Bearer {credentials.get('api_key')}"} + ) + response.raise_for_status() + results = response.json() + + rerank_documents = [] + for result in results['results']: + rerank_document = RerankDocument( + index=result['index'], + text=result['document']['text'], + score=result['relevance_score'], + ) + if score_threshold is None or result['relevance_score'] >= score_threshold: + rerank_documents.append(rerank_document) + + return RerankResult(model=model, docs=rerank_documents) + except httpx.HTTPStatusError as e: + raise InvokeServerUnavailableError(str(e)) + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + + self._invoke( + model=model, + credentials=credentials, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + """ + return { + InvokeConnectionError: [httpx.ConnectError], + InvokeServerUnavailableError: [httpx.RemoteProtocolError], + InvokeRateLimitError: [], + InvokeAuthorizationError: [httpx.HTTPStatusError], + InvokeBadRequestError: [httpx.RequestError] + } From 3c1825187a833adf7508c3331dbe645d793ccb0b Mon Sep 17 00:00:00 2001 From: Joel Date: Mon, 4 Mar 2024 17:36:11 +0800 Subject: [PATCH 13/13] fix: auto generate prompt result not show (#2678) --- .../config/automatic/get-automatic-res.tsx | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx index c1aa603f0b..1e476785e7 100644 --- a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx +++ b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx @@ -20,7 +20,7 @@ import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' const noDataIcon = ( - + ) @@ -33,9 +33,9 @@ export type IGetAutomaticResProps = { const genIcon = ( - - - + + + ) @@ -74,14 +74,14 @@ const GetAutomaticRes: FC = ({ const [res, setRes] = React.useState(null) const renderLoading = ( -
+
{t('appDebug.automatic.loading')}
) const renderNoData = ( -
+
{noDataIcon}
{t('appDebug.automatic.noData')}
@@ -142,7 +142,7 @@ const GetAutomaticRes: FC = ({
{t('appDebug.automatic.description')}
{/* inputs */} -
+
{t('appDebug.automatic.intendedAudience')}
setAudiences(e.target.value)} /> @@ -167,8 +167,8 @@ const GetAutomaticRes: FC = ({
} {(!isLoading && res) && ( -
-
{t('appDebug.automatic.resTitle')}
+
+
{t('appDebug.automatic.resTitle')}
= ({
)} -
+