diff --git a/LICENSE b/LICENSE index 8c9dd2961c..071ef42bda 100644 --- a/LICENSE +++ b/LICENSE @@ -1,24 +1,26 @@ -# Dify Open Source License +# Open Source License -The Dify project is licensed under the Apache License 2.0, with the following additional conditions: +Dify is licensed under the Apache License 2.0, with the following additional conditions: -1. Dify is permitted to be used for commercialization, such as using Dify as a "backend-as-a-service" for your other applications, or delivering it to enterprises as an application development platform. However, when the following conditions are met, you must contact the producer to obtain a commercial license: +1. Dify may be utilized commercially, including as a backend service for other applications or as an application development platform for enterprises. Should the conditions below be met, a commercial license must be obtained from the producer: -a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify.AI source code to operate a multi-tenant SaaS service that is similar to the Dify.AI service edition. -b. LOGO and copyright information: In the process of using Dify, you may not remove or modify the LOGO or copyright information in the Dify console. +a. Multi-tenant SaaS service: Unless explicitly authorized by Dify in writing, you may not use the Dify source code to operate a multi-tenant environment. + - Tenant Definition: Within the context of Dify, one tenant corresponds to one workspace. The workspace provides a separated area for each tenant's data and configurations. + +b. LOGO and copyright information: In the process of using Dify's frontend components, you may not remove or modify the LOGO or copyright information in the Dify console or applications. This restriction is inapplicable to uses of Dify that do not involve its frontend components. Please contact business@dify.ai by email to inquire about licensing matters. -2. As a contributor, you should agree that your contributed code: +2. As a contributor, you should agree that: -a. The producer can adjust the open-source agreement to be more strict or relaxed. -b. Can be used for commercial purposes, such as Dify's cloud business. +a. The producer can adjust the open-source agreement to be more strict or relaxed as deemed necessary. +b. Your contributed code may be used for commercial purposes, including but not limited to its cloud business operations. -Apart from this, all other rights and restrictions follow the Apache License 2.0. If you need more detailed information, you can refer to the full version of Apache License 2.0. +Apart from the specific conditions mentioned above, all other rights and restrictions follow the Apache License 2.0. Detailed information about the Apache License 2.0 can be found at http://www.apache.org/licenses/LICENSE-2.0. The interactive design of this product is protected by appearance patent. -© 2023 LangGenius, Inc. +© 2024 LangGenius, Inc. ---------- diff --git a/api/commands.py b/api/commands.py index 45977cbb9e..250039a365 100644 --- a/api/commands.py +++ b/api/commands.py @@ -15,7 +15,7 @@ from libs.rsa import generate_key_pair from models.account import Tenant from models.dataset import Dataset, DatasetCollectionBinding, DocumentSegment from models.dataset import Document as DatasetDocument -from models.model import Account +from models.model import Account, App, AppAnnotationSetting, MessageAnnotation from models.provider import Provider, ProviderModel @@ -125,7 +125,114 @@ def reset_encrypt_key_pair(): @click.command('vdb-migrate', help='migrate vector db.') -def vdb_migrate(): +@click.option('--scope', default='all', prompt=False, help='The scope of vector database to migrate, Default is All.') +def vdb_migrate(scope: str): + if scope in ['knowledge', 'all']: + migrate_knowledge_vector_database() + if scope in ['annotation', 'all']: + migrate_annotation_vector_database() + + +def migrate_annotation_vector_database(): + """ + Migrate annotation datas to target vector database . + """ + click.echo(click.style('Start migrate annotation data.', fg='green')) + create_count = 0 + skipped_count = 0 + total_count = 0 + page = 1 + while True: + try: + # get apps info + apps = db.session.query(App).filter( + App.status == 'normal' + ).order_by(App.created_at.desc()).paginate(page=page, per_page=50) + except NotFound: + break + + page += 1 + for app in apps: + total_count = total_count + 1 + click.echo(f'Processing the {total_count} app {app.id}. ' + + f'{create_count} created, {skipped_count} skipped.') + try: + click.echo('Create app annotation index: {}'.format(app.id)) + app_annotation_setting = db.session.query(AppAnnotationSetting).filter( + AppAnnotationSetting.app_id == app.id + ).first() + + if not app_annotation_setting: + skipped_count = skipped_count + 1 + click.echo('App annotation setting is disabled: {}'.format(app.id)) + continue + # get dataset_collection_binding info + dataset_collection_binding = db.session.query(DatasetCollectionBinding).filter( + DatasetCollectionBinding.id == app_annotation_setting.collection_binding_id + ).first() + if not dataset_collection_binding: + click.echo('App annotation collection binding is not exist: {}'.format(app.id)) + continue + annotations = db.session.query(MessageAnnotation).filter(MessageAnnotation.app_id == app.id).all() + dataset = Dataset( + id=app.id, + tenant_id=app.tenant_id, + indexing_technique='high_quality', + embedding_model_provider=dataset_collection_binding.provider_name, + embedding_model=dataset_collection_binding.model_name, + collection_binding_id=dataset_collection_binding.id + ) + documents = [] + if annotations: + for annotation in annotations: + document = Document( + page_content=annotation.question, + metadata={ + "annotation_id": annotation.id, + "app_id": app.id, + "doc_id": annotation.id + } + ) + documents.append(document) + + vector = Vector(dataset, attributes=['doc_id', 'annotation_id', 'app_id']) + click.echo(f"Start to migrate annotation, app_id: {app.id}.") + + try: + vector.delete() + click.echo( + click.style(f'Successfully delete vector index for app: {app.id}.', + fg='green')) + except Exception as e: + click.echo( + click.style(f'Failed to delete vector index for app {app.id}.', + fg='red')) + raise e + if documents: + try: + click.echo(click.style( + f'Start to created vector index with {len(documents)} annotations for app {app.id}.', + fg='green')) + vector.create(documents) + click.echo( + click.style(f'Successfully created vector index for app {app.id}.', fg='green')) + except Exception as e: + click.echo(click.style(f'Failed to created vector index for app {app.id}.', fg='red')) + raise e + click.echo(f'Successfully migrated app annotation {app.id}.') + create_count += 1 + except Exception as e: + click.echo( + click.style('Create app annotation index error: {} {}'.format(e.__class__.__name__, str(e)), + fg='red')) + continue + + click.echo( + click.style(f'Congratulations! Create {create_count} app annotation indexes, and skipped {skipped_count} apps.', + fg='green')) + + +def migrate_knowledge_vector_database(): """ Migrate vector database datas to target vector database . """ diff --git a/api/controllers/console/app/app.py b/api/controllers/console/app/app.py index 59a7535144..21ce9cb6af 100644 --- a/api/controllers/console/app/app.py +++ b/api/controllers/console/app/app.py @@ -129,7 +129,7 @@ class AppListApi(Resource): "No Default System Reasoning Model available. Please configure " "in the Settings -> Model Provider.") else: - model_config_dict["model"]["provider"] = default_model_entity.provider + model_config_dict["model"]["provider"] = default_model_entity.provider.provider model_config_dict["model"]["name"] = default_model_entity.model model_configuration = AppModelConfigService.validate_configuration( diff --git a/api/controllers/console/workspace/tool_providers.py b/api/controllers/console/workspace/tool_providers.py index c2c5286d51..817c75765a 100644 --- a/api/controllers/console/workspace/tool_providers.py +++ b/api/controllers/console/workspace/tool_providers.py @@ -259,6 +259,7 @@ class ToolApiProviderPreviousTestApi(Resource): parser = reqparse.RequestParser() parser.add_argument('tool_name', type=str, required=True, nullable=False, location='json') + parser.add_argument('provider_name', type=str, required=False, nullable=False, location='json') parser.add_argument('credentials', type=dict, required=True, nullable=False, location='json') parser.add_argument('parameters', type=dict, required=True, nullable=False, location='json') parser.add_argument('schema_type', type=str, required=True, nullable=False, location='json') @@ -268,6 +269,7 @@ class ToolApiProviderPreviousTestApi(Resource): return ToolManageService.test_api_tool_preview( current_user.current_tenant_id, + args['provider_name'] if args['provider_name'] else '', args['tool_name'], args['credentials'], args['parameters'], diff --git a/api/core/app_runner/app_runner.py b/api/core/app_runner/app_runner.py index 2b8ddc5d4e..f9678b372f 100644 --- a/api/core/app_runner/app_runner.py +++ b/api/core/app_runner/app_runner.py @@ -84,7 +84,7 @@ class AppRunner: return rest_tokens - def recale_llm_max_tokens(self, model_config: ModelConfigEntity, + def recalc_llm_max_tokens(self, model_config: ModelConfigEntity, prompt_messages: list[PromptMessage]): # recalc max_tokens if sum(prompt_token + max_tokens) over model token limit model_type_instance = model_config.provider_model_bundle.model_type_instance diff --git a/api/core/app_runner/basic_app_runner.py b/api/core/app_runner/basic_app_runner.py index 99df249ddf..83f4f6929a 100644 --- a/api/core/app_runner/basic_app_runner.py +++ b/api/core/app_runner/basic_app_runner.py @@ -181,7 +181,7 @@ class BasicApplicationRunner(AppRunner): return # Re-calculate the max tokens if sum(prompt_token + max_tokens) over model token limit - self.recale_llm_max_tokens( + self.recalc_llm_max_tokens( model_config=app_orchestration_config.model_config, prompt_messages=prompt_messages ) diff --git a/api/core/features/assistant_cot_runner.py b/api/core/features/assistant_cot_runner.py index 809834c8cb..8fcbff983d 100644 --- a/api/core/features/assistant_cot_runner.py +++ b/api/core/features/assistant_cot_runner.py @@ -130,8 +130,8 @@ class AssistantCotApplicationRunner(BaseAssistantApplicationRunner): input=query ) - # recale llm max tokens - self.recale_llm_max_tokens(self.model_config, prompt_messages) + # recalc llm max tokens + self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Generator[LLMResultChunk, None, None] = model_instance.invoke_llm( prompt_messages=prompt_messages, diff --git a/api/core/features/assistant_fc_runner.py b/api/core/features/assistant_fc_runner.py index 7ad9d7bd2a..391e040c53 100644 --- a/api/core/features/assistant_fc_runner.py +++ b/api/core/features/assistant_fc_runner.py @@ -105,8 +105,8 @@ class AssistantFunctionCallApplicationRunner(BaseAssistantApplicationRunner): messages_ids=message_file_ids ) - # recale llm max tokens - self.recale_llm_max_tokens(self.model_config, prompt_messages) + # recalc llm max tokens + self.recalc_llm_max_tokens(self.model_config, prompt_messages) # invoke model chunks: Union[Generator[LLMResultChunk, None, None], LLMResult] = model_instance.invoke_llm( prompt_messages=prompt_messages, diff --git a/api/core/model_runtime/model_providers/jina/jina.yaml b/api/core/model_runtime/model_providers/jina/jina.yaml index ad90344d53..935546234b 100644 --- a/api/core/model_runtime/model_providers/jina/jina.yaml +++ b/api/core/model_runtime/model_providers/jina/jina.yaml @@ -2,7 +2,7 @@ provider: jina label: en_US: Jina description: - en_US: Embedding Model Supported + en_US: Embedding and Rerank Model Supported icon_small: en_US: icon_s_en.svg icon_large: @@ -13,9 +13,10 @@ help: en_US: Get your API key from Jina AI zh_Hans: 从 Jina 获取 API Key url: - en_US: https://jina.ai/embeddings/ + en_US: https://jina.ai/ supported_model_types: - text-embedding + - rerank configurate_methods: - predefined-model provider_credential_schema: diff --git a/api/core/model_runtime/model_providers/jina/rerank/__init__.py b/api/core/model_runtime/model_providers/jina/rerank/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml b/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml new file mode 100644 index 0000000000..bd3f31fbd1 --- /dev/null +++ b/api/core/model_runtime/model_providers/jina/rerank/jina-reranker-v1-base-en.yaml @@ -0,0 +1,4 @@ +model: jina-reranker-v1-base-en +model_type: rerank +model_properties: + context_size: 8192 diff --git a/api/core/model_runtime/model_providers/jina/rerank/rerank.py b/api/core/model_runtime/model_providers/jina/rerank/rerank.py new file mode 100644 index 0000000000..f644ea6512 --- /dev/null +++ b/api/core/model_runtime/model_providers/jina/rerank/rerank.py @@ -0,0 +1,105 @@ +from typing import Optional + +import httpx + +from core.model_runtime.entities.rerank_entities import RerankDocument, RerankResult +from core.model_runtime.errors.invoke import ( + InvokeAuthorizationError, + InvokeBadRequestError, + InvokeConnectionError, + InvokeError, + InvokeRateLimitError, + InvokeServerUnavailableError, +) +from core.model_runtime.errors.validate import CredentialsValidateFailedError +from core.model_runtime.model_providers.__base.rerank_model import RerankModel + + +class JinaRerankModel(RerankModel): + """ + Model class for Jina rerank model. + """ + + def _invoke(self, model: str, credentials: dict, + query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + user: Optional[str] = None) -> RerankResult: + """ + Invoke rerank model + + :param model: model name + :param credentials: model credentials + :param query: search query + :param docs: docs for reranking + :param score_threshold: score threshold + :param top_n: top n documents to return + :param user: unique user id + :return: rerank result + """ + if len(docs) == 0: + return RerankResult(model=model, docs=[]) + + try: + response = httpx.post( + "https://api.jina.ai/v1/rerank", + json={ + "model": model, + "query": query, + "documents": docs, + "top_n": top_n + }, + headers={"Authorization": f"Bearer {credentials.get('api_key')}"} + ) + response.raise_for_status() + results = response.json() + + rerank_documents = [] + for result in results['results']: + rerank_document = RerankDocument( + index=result['index'], + text=result['document']['text'], + score=result['relevance_score'], + ) + if score_threshold is None or result['relevance_score'] >= score_threshold: + rerank_documents.append(rerank_document) + + return RerankResult(model=model, docs=rerank_documents) + except httpx.HTTPStatusError as e: + raise InvokeServerUnavailableError(str(e)) + + def validate_credentials(self, model: str, credentials: dict) -> None: + """ + Validate model credentials + + :param model: model name + :param credentials: model credentials + :return: + """ + try: + + self._invoke( + model=model, + credentials=credentials, + query="What is the capital of the United States?", + docs=[ + "Carson City is the capital city of the American state of Nevada. At the 2010 United States " + "Census, Carson City had a population of 55,274.", + "The Commonwealth of the Northern Mariana Islands is a group of islands in the Pacific Ocean that " + "are a political division controlled by the United States. Its capital is Saipan.", + ], + score_threshold=0.8 + ) + except Exception as ex: + raise CredentialsValidateFailedError(str(ex)) + + @property + def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + """ + Map model invoke error to unified error + """ + return { + InvokeConnectionError: [httpx.ConnectError], + InvokeServerUnavailableError: [httpx.RemoteProtocolError], + InvokeRateLimitError: [], + InvokeAuthorizationError: [httpx.HTTPStatusError], + InvokeBadRequestError: [httpx.RequestError] + } diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 0fc8ed5a26..203b7eff37 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -140,7 +140,8 @@ class MilvusVector(BaseVector): connections.connect(alias=alias, uri=uri, user=self._client_config.user, password=self._client_config.password) from pymilvus import utility - utility.drop_collection(self._collection_name, None, using=alias) + if utility.has_collection(self._collection_name, using=alias): + utility.drop_collection(self._collection_name, None, using=alias) def text_exists(self, id: str) -> bool: diff --git a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py index 2432931228..6bd4b5c340 100644 --- a/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py +++ b/api/core/rag/datasource/vdb/qdrant/qdrant_vector.py @@ -231,21 +231,30 @@ class QdrantVector(BaseVector): def delete(self): from qdrant_client.http import models - filter = models.Filter( - must=[ - models.FieldCondition( - key="group_id", - match=models.MatchValue(value=self._group_id), + from qdrant_client.http.exceptions import UnexpectedResponse + + try: + filter = models.Filter( + must=[ + models.FieldCondition( + key="group_id", + match=models.MatchValue(value=self._group_id), + ), + ], + ) + self._client.delete( + collection_name=self._collection_name, + points_selector=FilterSelector( + filter=filter ), - ], - ) - self._client.delete( - collection_name=self._collection_name, - points_selector=FilterSelector( - filter=filter - ), - ) - + ) + except UnexpectedResponse as e: + # Collection does not exist, so return + if e.status_code == 404: + return + # Some other error occurred, so re-raise the exception + else: + raise e def delete_by_ids(self, ids: list[str]) -> None: from qdrant_client.http import models diff --git a/api/core/tools/provider/builtin/bing/bing.py b/api/core/tools/provider/builtin/bing/bing.py index 76ce630f22..ff131b26cd 100644 --- a/api/core/tools/provider/builtin/bing/bing.py +++ b/api/core/tools/provider/builtin/bing/bing.py @@ -16,7 +16,8 @@ class BingProvider(BuiltinToolProviderController): user_id='', tool_parameters={ "query": "test", - "result_type": "link" + "result_type": "link", + "enable_webpages": True, }, ) except Exception as e: diff --git a/api/core/tools/provider/builtin/wecom/wecom.py b/api/core/tools/provider/builtin/wecom/wecom.py index b50d3ff68a..6380061b4f 100644 --- a/api/core/tools/provider/builtin/wecom/wecom.py +++ b/api/core/tools/provider/builtin/wecom/wecom.py @@ -2,7 +2,7 @@ from core.tools.provider.builtin.wecom.tools.wecom_group_bot import WecomReposit from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController -class GaodeProvider(BuiltinToolProviderController): +class WecomProvider(BuiltinToolProviderController): def _validate_credentials(self, credentials: dict) -> None: WecomRepositoriesTool() pass diff --git a/api/core/tools/tool/api_tool.py b/api/core/tools/tool/api_tool.py index d5a4bf20bd..31519734ed 100644 --- a/api/core/tools/tool/api_tool.py +++ b/api/core/tools/tool/api_tool.py @@ -1,6 +1,7 @@ import json from json import dumps from typing import Any, Union +from urllib.parse import urlencode import httpx import requests @@ -203,6 +204,8 @@ class ApiTool(Tool): if 'Content-Type' in headers: if headers['Content-Type'] == 'application/json': body = dumps(body) + elif headers['Content-Type'] == 'application/x-www-form-urlencoded': + body = urlencode(body) else: body = body diff --git a/api/requirements.txt b/api/requirements.txt index fbadcdbf04..ae5c77137a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -52,7 +52,7 @@ safetensors==0.3.2 zhipuai==1.0.7 werkzeug~=3.0.1 pymilvus==2.3.0 -qdrant-client==1.6.4 +qdrant-client==1.7.3 cohere~=4.44 pyyaml~=6.0.1 numpy~=1.25.2 diff --git a/api/services/tools_manage_service.py b/api/services/tools_manage_service.py index 7e305c3f7b..0e3d481640 100644 --- a/api/services/tools_manage_service.py +++ b/api/services/tools_manage_service.py @@ -498,12 +498,16 @@ class ToolManageService: @staticmethod def test_api_tool_preview( - tenant_id: str, tool_name: str, credentials: dict, parameters: dict, schema_type: str, schema: str + tenant_id: str, + provider_name: str, + tool_name: str, + credentials: dict, + parameters: dict, + schema_type: str, + schema: str ): """ test api tool before adding api tool provider - - 1. parse schema into tool bundle """ if schema_type not in [member.value for member in ApiProviderSchemaType]: raise ValueError(f'invalid schema type {schema_type}') @@ -518,15 +522,21 @@ class ToolManageService: if tool_bundle is None: raise ValueError(f'invalid tool name {tool_name}') - # create a fake db provider - db_provider = ApiToolProvider( - tenant_id='', user_id='', name='', icon='', - schema=schema, - description='', - schema_type_str=ApiProviderSchemaType.OPENAPI.value, - tools_str=serialize_base_model_array(tool_bundles), - credentials_str=json.dumps(credentials), - ) + db_provider: ApiToolProvider = db.session.query(ApiToolProvider).filter( + ApiToolProvider.tenant_id == tenant_id, + ApiToolProvider.name == provider_name, + ).first() + + if not db_provider: + # create a fake db provider + db_provider = ApiToolProvider( + tenant_id='', user_id='', name='', icon='', + schema=schema, + description='', + schema_type_str=ApiProviderSchemaType.OPENAPI.value, + tools_str=serialize_base_model_array(tool_bundles), + credentials_str=json.dumps(credentials), + ) if 'auth_type' not in credentials: raise ValueError('auth_type is required') @@ -539,6 +549,19 @@ class ToolManageService: # load tools into provider entity provider_controller.load_bundled_tools(tool_bundles) + # decrypt credentials + if db_provider.id: + tool_configuration = ToolConfiguration( + tenant_id=tenant_id, + provider_controller=provider_controller + ) + decrypted_credentials = tool_configuration.decrypt_tool_credentials(credentials) + # check if the credential has changed, save the original credential + masked_credentials = tool_configuration.mask_tool_credentials(decrypted_credentials) + for name, value in credentials.items(): + if name in masked_credentials and value == masked_credentials[name]: + credentials[name] = decrypted_credentials[name] + try: provider_controller.validate_credentials_format(credentials) # get tool diff --git a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx index c1aa603f0b..1e476785e7 100644 --- a/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx +++ b/web/app/components/app/configuration/config/automatic/get-automatic-res.tsx @@ -20,7 +20,7 @@ import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' const noDataIcon = ( - + ) @@ -33,9 +33,9 @@ export type IGetAutomaticResProps = { const genIcon = ( - - - + + + ) @@ -74,14 +74,14 @@ const GetAutomaticRes: FC = ({ const [res, setRes] = React.useState(null) const renderLoading = ( -
+
{t('appDebug.automatic.loading')}
) const renderNoData = ( -
+
{noDataIcon}
{t('appDebug.automatic.noData')}
@@ -142,7 +142,7 @@ const GetAutomaticRes: FC = ({
{t('appDebug.automatic.description')}
{/* inputs */} -
+
{t('appDebug.automatic.intendedAudience')}
setAudiences(e.target.value)} /> @@ -167,8 +167,8 @@ const GetAutomaticRes: FC = ({
} {(!isLoading && res) && ( -
-
{t('appDebug.automatic.resTitle')}
+
+
{t('appDebug.automatic.resTitle')}
= ({
)} -
+