From ee7d5e7206bf3d87035822b062ad52954401370a Mon Sep 17 00:00:00 2001 From: sino Date: Tue, 27 Aug 2024 14:43:37 +0800 Subject: [PATCH 001/125] feat: support Moonshot and GLM models tool call for volc ark provider (#7666) --- .../model_providers/volcengine_maas/llm/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/core/model_runtime/model_providers/volcengine_maas/llm/models.py b/api/core/model_runtime/model_providers/volcengine_maas/llm/models.py index 4e2c66a066..a882f68a36 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/llm/models.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/llm/models.py @@ -38,7 +38,7 @@ configs: dict[str, ModelConfig] = { ), 'Doubao-lite-128k': ModelConfig( properties=ModelProperties(context_size=131072, max_tokens=4096, mode=LLMMode.CHAT), - features=[ModelFeature.TOOL_CALL] + features=[] ), 'Skylark2-pro-4k': ModelConfig( properties=ModelProperties(context_size=4096, max_tokens=4096, mode=LLMMode.CHAT), @@ -54,23 +54,23 @@ configs: dict[str, ModelConfig] = { ), 'Moonshot-v1-8k': ModelConfig( properties=ModelProperties(context_size=8192, max_tokens=4096, mode=LLMMode.CHAT), - features=[] + features=[ModelFeature.TOOL_CALL] ), 'Moonshot-v1-32k': ModelConfig( properties=ModelProperties(context_size=32768, max_tokens=16384, mode=LLMMode.CHAT), - features=[] + features=[ModelFeature.TOOL_CALL] ), 'Moonshot-v1-128k': ModelConfig( properties=ModelProperties(context_size=131072, max_tokens=65536, mode=LLMMode.CHAT), - features=[] + features=[ModelFeature.TOOL_CALL] ), 'GLM3-130B': ModelConfig( properties=ModelProperties(context_size=8192, max_tokens=4096, mode=LLMMode.CHAT), - features=[] + features=[ModelFeature.TOOL_CALL] ), 'GLM3-130B-Fin': ModelConfig( properties=ModelProperties(context_size=8192, max_tokens=4096, mode=LLMMode.CHAT), - features=[] + features=[ModelFeature.TOOL_CALL] ), 'Mistral-7B': ModelConfig( properties=ModelProperties(context_size=8192, max_tokens=2048, mode=LLMMode.CHAT), From 60001a62c4d9a6da95e589026f86f962ffbab282 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:38:06 +0800 Subject: [PATCH 002/125] fixed chunk_overlap is None (#7703) --- api/core/rag/index_processor/index_processor_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/rag/index_processor/index_processor_base.py b/api/core/rag/index_processor/index_processor_base.py index 33e78ce8c5..176d0c1ed6 100644 --- a/api/core/rag/index_processor/index_processor_base.py +++ b/api/core/rag/index_processor/index_processor_base.py @@ -57,7 +57,7 @@ class BaseIndexProcessor(ABC): character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder( chunk_size=segmentation["max_tokens"], - chunk_overlap=segmentation.get('chunk_overlap', 0), + chunk_overlap=segmentation.get('chunk_overlap', 0) or 0, fixed_separator=separator, separators=["\n\n", "。", ". ", " ", ""], embedding_model_instance=embedding_model_instance From d9198b5646e4c536836817eec418f430b1738954 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:47:34 +0800 Subject: [PATCH 003/125] feat: remove unused code (#7702) --- .../custom/custom-app-header-brand/index.tsx | 62 ------------------- .../custom-app-header-brand/style.module.css | 3 - .../components/custom/custom-page/index.tsx | 10 --- 3 files changed, 75 deletions(-) delete mode 100644 web/app/components/custom/custom-app-header-brand/index.tsx delete mode 100644 web/app/components/custom/custom-app-header-brand/style.module.css diff --git a/web/app/components/custom/custom-app-header-brand/index.tsx b/web/app/components/custom/custom-app-header-brand/index.tsx deleted file mode 100644 index 9564986c28..0000000000 --- a/web/app/components/custom/custom-app-header-brand/index.tsx +++ /dev/null @@ -1,62 +0,0 @@ -import { useTranslation } from 'react-i18next' -import s from './style.module.css' -import Button from '@/app/components/base/button' -import { Grid01 } from '@/app/components/base/icons/src/vender/solid/layout' -import { Container, Database01 } from '@/app/components/base/icons/src/vender/line/development' -import { ImagePlus } from '@/app/components/base/icons/src/vender/line/images' -import { useProviderContext } from '@/context/provider-context' -import { Plan } from '@/app/components/billing/type' - -const CustomAppHeaderBrand = () => { - const { t } = useTranslation() - const { plan } = useProviderContext() - - return ( -
-
{t('custom.app.title')}
-
-
-
-
-
-
YOUR LOGO
-
-
-
-
-
-
- -
-
-
- -
-
-
- -
-
-
-
-
-
- -
- -
-
{t('custom.app.changeLogoTip')}
-
- ) -} - -export default CustomAppHeaderBrand diff --git a/web/app/components/custom/custom-app-header-brand/style.module.css b/web/app/components/custom/custom-app-header-brand/style.module.css deleted file mode 100644 index 492733ff9f..0000000000 --- a/web/app/components/custom/custom-app-header-brand/style.module.css +++ /dev/null @@ -1,3 +0,0 @@ -.mask { - background: linear-gradient(95deg, rgba(255, 255, 255, 0.00) 43.9%, rgba(255, 255, 255, 0.80) 95.76%); ; -} \ No newline at end of file diff --git a/web/app/components/custom/custom-page/index.tsx b/web/app/components/custom/custom-page/index.tsx index c3b1e93da3..75d592389d 100644 --- a/web/app/components/custom/custom-page/index.tsx +++ b/web/app/components/custom/custom-page/index.tsx @@ -1,6 +1,5 @@ import { useTranslation } from 'react-i18next' import CustomWebAppBrand from '../custom-web-app-brand' -import CustomAppHeaderBrand from '../custom-app-header-brand' import s from '../style.module.css' import GridMask from '@/app/components/base/grid-mask' import UpgradeBtn from '@/app/components/billing/upgrade-btn' @@ -13,7 +12,6 @@ const CustomPage = () => { const { plan, enableBilling } = useProviderContext() const showBillingTip = enableBilling && plan.type === Plan.sandbox - const showCustomAppHeaderBrand = enableBilling && plan.type === Plan.sandbox const showContact = enableBilling && (plan.type === Plan.professional || plan.type === Plan.team) return ( @@ -32,14 +30,6 @@ const CustomPage = () => { ) } - { - showCustomAppHeaderBrand && ( - <> -
- - - ) - } { showContact && (
From da326baa5ec04c2c261bac11ba0b80853f3534b4 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:56:06 +0800 Subject: [PATCH 004/125] fix: tongyi Error: 'NoneType' object is not subscriptable (#7705) --- .../tongyi/text_embedding/text_embedding.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/tongyi/text_embedding/text_embedding.py b/api/core/model_runtime/model_providers/tongyi/text_embedding/text_embedding.py index e7e1b5c764..97dcb72f7c 100644 --- a/api/core/model_runtime/model_providers/tongyi/text_embedding/text_embedding.py +++ b/api/core/model_runtime/model_providers/tongyi/text_embedding/text_embedding.py @@ -137,9 +137,19 @@ class TongyiTextEmbeddingModel(_CommonTongyi, TextEmbeddingModel): input=text, text_type="document", ) - data = response.output["embeddings"][0] - embeddings.append(data["embedding"]) - embedding_used_tokens += response.usage["total_tokens"] + if response.output and "embeddings" in response.output and response.output["embeddings"]: + data = response.output["embeddings"][0] + if "embedding" in data: + embeddings.append(data["embedding"]) + else: + raise ValueError("Embedding data is missing in the response.") + else: + raise ValueError("Response output is missing or does not contain embeddings.") + + if response.usage and "total_tokens" in response.usage: + embedding_used_tokens += response.usage["total_tokens"] + else: + raise ValueError("Response usage is missing or does not contain total tokens.") return [list(map(float, e)) for e in embeddings], embedding_used_tokens From 205d33a813bca1a859da43dccdb1b7510c513b7a Mon Sep 17 00:00:00 2001 From: Bryan Date: Tue, 27 Aug 2024 19:23:56 +0800 Subject: [PATCH 005/125] Fix: read properties of undefined issue (#7708) Co-authored-by: libing --- web/app/components/datasets/documents/list.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index ab602955d7..540474e7a5 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -405,7 +405,7 @@ const DocumentList: FC = ({ embeddingAvailable, documents = {localDocs.map((doc) => { const isFile = doc.data_source_type === DataSourceType.FILE - const fileType = isFile ? doc.data_source_detail_dict?.upload_file.extension : '' + const fileType = isFile ? doc.data_source_detail_dict?.upload_file?.extension : '' return Date: Tue, 27 Aug 2024 19:38:33 +0800 Subject: [PATCH 006/125] feat: support configs for code execution request (#7704) --- api/configs/feature/__init__.py | 19 ++++++++++++-- .../helper/code_executor/code_executor.py | 26 +++++++++---------- api/poetry.lock | 12 ++++----- api/pyproject.toml | 2 +- .../unit_tests/configs/test_dify_config.py | 4 +++ 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index 46ae7a0bc8..f2efa52de3 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,6 +1,6 @@ from typing import Optional -from pydantic import AliasChoices, Field, NegativeInt, NonNegativeInt, PositiveInt, computed_field +from pydantic import AliasChoices, Field, HttpUrl, NegativeInt, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings from configs.feature.hosted_service import HostedServiceConfig @@ -45,7 +45,7 @@ class CodeExecutionSandboxConfig(BaseSettings): Code Execution Sandbox configs """ - CODE_EXECUTION_ENDPOINT: str = Field( + CODE_EXECUTION_ENDPOINT: HttpUrl = Field( description="endpoint URL of code execution servcie", default="http://sandbox:8194", ) @@ -55,6 +55,21 @@ class CodeExecutionSandboxConfig(BaseSettings): default="dify-sandbox", ) + CODE_EXECUTION_CONNECT_TIMEOUT: Optional[float] = Field( + description="connect timeout in seconds for code execution request", + default=10.0, + ) + + CODE_EXECUTION_READ_TIMEOUT: Optional[float] = Field( + description="read timeout in seconds for code execution request", + default=60.0, + ) + + CODE_EXECUTION_WRITE_TIMEOUT: Optional[float] = Field( + description="write timeout in seconds for code execution request", + default=10.0, + ) + CODE_MAX_NUMBER: PositiveInt = Field( description="max depth for code execution", default=9223372036854775807, diff --git a/api/core/helper/code_executor/code_executor.py b/api/core/helper/code_executor/code_executor.py index a829748b48..4662ebb47a 100644 --- a/api/core/helper/code_executor/code_executor.py +++ b/api/core/helper/code_executor/code_executor.py @@ -15,12 +15,6 @@ from core.helper.code_executor.template_transformer import TemplateTransformer logger = logging.getLogger(__name__) -# Code Executor -CODE_EXECUTION_ENDPOINT = dify_config.CODE_EXECUTION_ENDPOINT -CODE_EXECUTION_API_KEY = dify_config.CODE_EXECUTION_API_KEY - -CODE_EXECUTION_TIMEOUT = Timeout(connect=10, write=10, read=60, pool=None) - class CodeExecutionException(Exception): pass @@ -71,10 +65,10 @@ class CodeExecutor: :param code: code :return: """ - url = URL(CODE_EXECUTION_ENDPOINT) / 'v1' / 'sandbox' / 'run' + url = URL(str(dify_config.CODE_EXECUTION_ENDPOINT)) / 'v1' / 'sandbox' / 'run' headers = { - 'X-Api-Key': CODE_EXECUTION_API_KEY + 'X-Api-Key': dify_config.CODE_EXECUTION_API_KEY } data = { @@ -85,7 +79,12 @@ class CodeExecutor: } try: - response = post(str(url), json=data, headers=headers, timeout=CODE_EXECUTION_TIMEOUT) + response = post(str(url), json=data, headers=headers, + timeout=Timeout( + connect=dify_config.CODE_EXECUTION_CONNECT_TIMEOUT, + read=dify_config.CODE_EXECUTION_READ_TIMEOUT, + write=dify_config.CODE_EXECUTION_WRITE_TIMEOUT, + pool=None)) if response.status_code == 503: raise CodeExecutionException('Code execution service is unavailable') elif response.status_code != 200: @@ -96,7 +95,7 @@ class CodeExecutor: raise CodeExecutionException('Failed to execute code, which is likely a network issue,' ' please check if the sandbox service is running.' f' ( Error: {str(e)} )') - + try: response = response.json() except: @@ -104,12 +103,12 @@ class CodeExecutor: if (code := response.get('code')) != 0: raise CodeExecutionException(f"Got error code: {code}. Got error msg: {response.get('message')}") - + response = CodeExecutionResponse(**response) - + if response.data.error: raise CodeExecutionException(response.data.error) - + return response.data.stdout or '' @classmethod @@ -133,4 +132,3 @@ class CodeExecutor: raise e return template_transformer.transform_response(response) - \ No newline at end of file diff --git a/api/poetry.lock b/api/poetry.lock index 224c0bfb8c..7d26dbdc57 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -6821,13 +6821,13 @@ files = [ [[package]] name = "pytest" -version = "8.1.2" +version = "8.3.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.1.2-py3-none-any.whl", hash = "sha256:6c06dc309ff46a05721e6fd48e492a775ed8165d2ecdf57f156a80c7e95bb142"}, - {file = "pytest-8.1.2.tar.gz", hash = "sha256:f3c45d1d5eed96b01a2aea70dee6a4a366d51d38f9957768083e4fecfc77f3ef"}, + {file = "pytest-8.3.2-py3-none-any.whl", hash = "sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5"}, + {file = "pytest-8.3.2.tar.gz", hash = "sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce"}, ] [package.dependencies] @@ -6835,11 +6835,11 @@ colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} iniconfig = "*" packaging = "*" -pluggy = ">=1.4,<2.0" +pluggy = ">=1.5,<2" tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] [[package]] name = "pytest-benchmark" @@ -10063,4 +10063,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.13" -content-hash = "0b912a7d500c4ff3c7f0c51877e55de70cec5317a990f9e882600e32a30a610e" +content-hash = "e4c00268514d26bd07c6b72925e0e3b4558ec972895d252e60e9571e3ac38895" diff --git a/api/pyproject.toml b/api/pyproject.toml index 1f5bafe591..f1d5e213ae 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -243,7 +243,7 @@ optional = true [tool.poetry.group.dev.dependencies] coverage = "~7.2.4" -pytest = "~8.1.1" +pytest = "~8.3.2" pytest-benchmark = "~4.0.0" pytest-env = "~1.1.3" pytest-mock = "~3.14.0" diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index 39f313b513..fb415483dd 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -3,6 +3,7 @@ from textwrap import dedent import pytest from flask import Flask +from yarl import URL from configs.app_config import DifyConfig @@ -84,3 +85,6 @@ def test_flask_configs(example_env_file): assert config["CONSOLE_WEB_URL"] == "https://example.com" assert config["CONSOLE_CORS_ALLOW_ORIGINS"] == ["https://example.com"] assert config["WEB_API_CORS_ALLOW_ORIGINS"] == ["*"] + + assert str(config["CODE_EXECUTION_ENDPOINT"]) == "http://sandbox:8194/" + assert str(URL(str(config["CODE_EXECUTION_ENDPOINT"])) / "v1") == "http://sandbox:8194/v1" From 92cab33b73d42f20d04541fb383dc783ee336de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B0=E5=9C=A8=E4=BF=AE=E8=A1=8C=E7=9A=84=E5=A4=A7?= =?UTF-8?q?=E8=A1=97=E4=B8=8A?= Date: Tue, 27 Aug 2024 20:21:42 +0800 Subject: [PATCH 007/125] feat(Tools): add feishu document and message plugins (#6435) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 黎斌 --- api/core/tools/provider/_position.yaml | 2 + .../builtin/feishu_document/_assets/icon.svg | 9 ++ .../feishu_document/feishu_document.py | 15 ++ .../feishu_document/feishu_document.yaml | 34 +++++ .../feishu_document/tools/create_document.py | 19 +++ .../tools/create_document.yaml | 47 ++++++ .../tools/get_document_raw_content.py | 17 +++ .../tools/get_document_raw_content.yaml | 23 +++ .../tools/list_document_block.py | 19 +++ .../tools/list_document_block.yaml | 48 ++++++ .../feishu_document/tools/write_document.py | 19 +++ .../feishu_document/tools/write_document.yaml | 56 +++++++ .../builtin/feishu_message/_assets/icon.svg | 19 +++ .../builtin/feishu_message/feishu_message.py | 15 ++ .../feishu_message/feishu_message.yaml | 34 +++++ .../feishu_message/tools/send_bot_message.py | 20 +++ .../tools/send_bot_message.yaml | 91 +++++++++++ .../tools/send_webhook_message.py | 19 +++ .../tools/send_webhook_message.yaml | 58 +++++++ api/core/tools/utils/feishu_api_utils.py | 143 ++++++++++++++++++ 20 files changed, 707 insertions(+) create mode 100644 api/core/tools/provider/builtin/feishu_document/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/feishu_document/feishu_document.py create mode 100644 api/core/tools/provider/builtin/feishu_document/feishu_document.yaml create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/create_document.py create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/create_document.yaml create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.py create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.yaml create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/list_document_block.py create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/list_document_block.yaml create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/write_document.py create mode 100644 api/core/tools/provider/builtin/feishu_document/tools/write_document.yaml create mode 100644 api/core/tools/provider/builtin/feishu_message/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/feishu_message/feishu_message.py create mode 100644 api/core/tools/provider/builtin/feishu_message/feishu_message.yaml create mode 100644 api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.py create mode 100644 api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.yaml create mode 100644 api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.py create mode 100644 api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.yaml create mode 100644 api/core/tools/utils/feishu_api_utils.py diff --git a/api/core/tools/provider/_position.yaml b/api/core/tools/provider/_position.yaml index 25d9f403a0..b804089570 100644 --- a/api/core/tools/provider/_position.yaml +++ b/api/core/tools/provider/_position.yaml @@ -30,5 +30,7 @@ - dingtalk - feishu - feishu_base +- feishu_document +- feishu_message - slack - tianditu diff --git a/api/core/tools/provider/builtin/feishu_document/_assets/icon.svg b/api/core/tools/provider/builtin/feishu_document/_assets/icon.svg new file mode 100644 index 0000000000..5a0a6416b3 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/_assets/icon.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/api/core/tools/provider/builtin/feishu_document/feishu_document.py b/api/core/tools/provider/builtin/feishu_document/feishu_document.py new file mode 100644 index 0000000000..c4f8f26e2c --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/feishu_document.py @@ -0,0 +1,15 @@ +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class FeishuDocumentProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict) -> None: + app_id = credentials.get('app_id') + app_secret = credentials.get('app_secret') + if not app_id or not app_secret: + raise ToolProviderCredentialValidationError("app_id and app_secret is required") + try: + assert FeishuRequest(app_id, app_secret).tenant_access_token is not None + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/feishu_document/feishu_document.yaml b/api/core/tools/provider/builtin/feishu_document/feishu_document.yaml new file mode 100644 index 0000000000..8eaa6b2704 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/feishu_document.yaml @@ -0,0 +1,34 @@ +identity: + author: Doug Lea + name: feishu_document + label: + en_US: Lark Cloud Document + zh_Hans: 飞书云文档 + description: + en_US: Lark Cloud Document + zh_Hans: 飞书云文档 + icon: icon.svg + tags: + - social + - productivity +credentials_for_provider: + app_id: + type: text-input + required: true + label: + en_US: APP ID + placeholder: + en_US: Please input your feishu app id + zh_Hans: 请输入你的飞书 app id + help: + en_US: Get your app_id and app_secret from Feishu + zh_Hans: 从飞书获取您的 app_id 和 app_secret + url: https://open.feishu.cn + app_secret: + type: secret-input + required: true + label: + en_US: APP Secret + placeholder: + en_US: Please input your app secret + zh_Hans: 请输入你的飞书 app secret diff --git a/api/core/tools/provider/builtin/feishu_document/tools/create_document.py b/api/core/tools/provider/builtin/feishu_document/tools/create_document.py new file mode 100644 index 0000000000..0ff82e621b --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/create_document.py @@ -0,0 +1,19 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class CreateDocumentTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + title = tool_parameters.get('title') + content = tool_parameters.get('content') + folder_token = tool_parameters.get('folder_token') + + res = client.create_document(title, content, folder_token) + return self.create_json_message(res) diff --git a/api/core/tools/provider/builtin/feishu_document/tools/create_document.yaml b/api/core/tools/provider/builtin/feishu_document/tools/create_document.yaml new file mode 100644 index 0000000000..ddf2729f0e --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/create_document.yaml @@ -0,0 +1,47 @@ +identity: + name: create_document + author: Doug Lea + label: + en_US: Create Lark document + zh_Hans: 创建飞书文档 +description: + human: + en_US: Create Lark document + zh_Hans: 创建飞书文档，支持创建空文档和带内容的文档，支持 markdown 语法创建。 + llm: A tool for creating Feishu documents. +parameters: + - name: title + type: string + required: false + label: + en_US: Document title + zh_Hans: 文档标题 + human_description: + en_US: Document title, only supports plain text content. + zh_Hans: 文档标题，只支持纯文本内容。 + llm_description: 文档标题，只支持纯文本内容，可以为空。 + form: llm + + - name: content + type: string + required: false + label: + en_US: Document content + zh_Hans: 文档内容 + human_description: + en_US: Document content, supports markdown syntax, can be empty. + zh_Hans: 文档内容，支持 markdown 语法，可以为空。 + llm_description: 文档内容，支持 markdown 语法，可以为空。 + form: llm + + - name: folder_token + type: string + required: false + label: + en_US: folder_token + zh_Hans: 文档所在文件夹的 Token + human_description: + en_US: The token of the folder where the document is located. If it is not passed or is empty, it means the root directory. + zh_Hans: 文档所在文件夹的 Token，不传或传空表示根目录。 + llm_description: 文档所在文件夹的 Token，不传或传空表示根目录。 + form: llm diff --git a/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.py b/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.py new file mode 100644 index 0000000000..16ef90908b --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.py @@ -0,0 +1,17 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class GetDocumentRawContentTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + document_id = tool_parameters.get('document_id') + + res = client.get_document_raw_content(document_id) + return self.create_json_message(res) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.yaml b/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.yaml new file mode 100644 index 0000000000..e5b0937e03 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/get_document_raw_content.yaml @@ -0,0 +1,23 @@ +identity: + name: get_document_raw_content + author: Doug Lea + label: + en_US: Get Document Raw Content + zh_Hans: 获取文档纯文本内容 +description: + human: + en_US: Get document raw content + zh_Hans: 获取文档纯文本内容 + llm: A tool for getting the plain text content of Feishu documents +parameters: + - name: document_id + type: string + required: true + label: + en_US: document_id + zh_Hans: 飞书文档的唯一标识 + human_description: + en_US: Unique ID of Feishu document document_id + zh_Hans: 飞书文档的唯一标识 document_id + llm_description: 飞书文档的唯一标识 document_id + form: llm diff --git a/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.py b/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.py new file mode 100644 index 0000000000..97d17bdb04 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.py @@ -0,0 +1,19 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class ListDocumentBlockTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + document_id = tool_parameters.get('document_id') + page_size = tool_parameters.get('page_size', 500) + page_token = tool_parameters.get('page_token', '') + + res = client.list_document_block(document_id, page_token, page_size) + return self.create_json_message(res) diff --git a/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.yaml b/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.yaml new file mode 100644 index 0000000000..d51e5a837c --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/list_document_block.yaml @@ -0,0 +1,48 @@ +identity: + name: list_document_block + author: Doug Lea + label: + en_US: List Document Block + zh_Hans: 获取飞书文档所有块 +description: + human: + en_US: List document block + zh_Hans: 获取飞书文档所有块的富文本内容并分页返回。 + llm: A tool to get all blocks of Feishu documents +parameters: + - name: document_id + type: string + required: true + label: + en_US: document_id + zh_Hans: 飞书文档的唯一标识 + human_description: + en_US: Unique ID of Feishu document document_id + zh_Hans: 飞书文档的唯一标识 document_id + llm_description: 飞书文档的唯一标识 document_id + form: llm + + - name: page_size + type: number + required: false + default: 500 + label: + en_US: page_size + zh_Hans: 分页大小 + human_description: + en_US: Paging size, the default and maximum value is 500. + zh_Hans: 分页大小, 默认值和最大值为 500。 + llm_description: 分页大小, 表示一次请求最多返回多少条数据，默认值和最大值为 500。 + form: llm + + - name: page_token + type: string + required: false + label: + en_US: page_token + zh_Hans: 分页标记 + human_description: + en_US: Pagination tag, used to paginate query results so that more items can be obtained in the next traversal. + zh_Hans: 分页标记，用于分页查询结果，以便下次遍历时获取更多项。 + llm_description: 分页标记，第一次请求不填，表示从头开始遍历；分页查询结果还有更多项时会同时返回新的 page_token，下次遍历可采用该 page_token 获取查询结果。 + form: llm diff --git a/api/core/tools/provider/builtin/feishu_document/tools/write_document.py b/api/core/tools/provider/builtin/feishu_document/tools/write_document.py new file mode 100644 index 0000000000..914a44dce6 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/write_document.py @@ -0,0 +1,19 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class CreateDocumentTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + document_id = tool_parameters.get('document_id') + content = tool_parameters.get('content') + position = tool_parameters.get('position') + + res = client.write_document(document_id, content, position) + return self.create_json_message(res) diff --git a/api/core/tools/provider/builtin/feishu_document/tools/write_document.yaml b/api/core/tools/provider/builtin/feishu_document/tools/write_document.yaml new file mode 100644 index 0000000000..8ee219d4a7 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_document/tools/write_document.yaml @@ -0,0 +1,56 @@ +identity: + name: write_document + author: Doug Lea + label: + en_US: Write Document + zh_Hans: 在飞书文档中新增内容 +description: + human: + en_US: Adding new content to Lark documents + zh_Hans: 在飞书文档中新增内容 + llm: A tool for adding new content to Lark documents. +parameters: + - name: document_id + type: string + required: true + label: + en_US: document_id + zh_Hans: 飞书文档的唯一标识 + human_description: + en_US: Unique ID of Feishu document document_id + zh_Hans: 飞书文档的唯一标识 document_id + llm_description: 飞书文档的唯一标识 document_id + form: llm + + - name: content + type: string + required: true + label: + en_US: document content + zh_Hans: 文档内容 + human_description: + en_US: Document content, supports markdown syntax, can be empty. + zh_Hans: 文档内容，支持 markdown 语法，可以为空。 + llm_description: + form: llm + + - name: position + type: select + required: true + default: start + label: + en_US: Choose where to add content + zh_Hans: 选择添加内容的位置 + human_description: + en_US: Please fill in start or end to add content at the beginning or end of the document respectively. + zh_Hans: 请填入 start 或 end, 分别表示在文档开头(start)或结尾(end)添加内容。 + form: llm + options: + - value: start + label: + en_US: start + zh_Hans: 在文档开头添加内容 + - value: end + label: + en_US: end + zh_Hans: 在文档结尾添加内容 diff --git a/api/core/tools/provider/builtin/feishu_message/_assets/icon.svg b/api/core/tools/provider/builtin/feishu_message/_assets/icon.svg new file mode 100644 index 0000000000..222a1571f9 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/_assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/api/core/tools/provider/builtin/feishu_message/feishu_message.py b/api/core/tools/provider/builtin/feishu_message/feishu_message.py new file mode 100644 index 0000000000..6d7fed330c --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/feishu_message.py @@ -0,0 +1,15 @@ +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class FeishuMessageProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict) -> None: + app_id = credentials.get('app_id') + app_secret = credentials.get('app_secret') + if not app_id or not app_secret: + raise ToolProviderCredentialValidationError("app_id and app_secret is required") + try: + assert FeishuRequest(app_id, app_secret).tenant_access_token is not None + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) \ No newline at end of file diff --git a/api/core/tools/provider/builtin/feishu_message/feishu_message.yaml b/api/core/tools/provider/builtin/feishu_message/feishu_message.yaml new file mode 100644 index 0000000000..1bd8953ddd --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/feishu_message.yaml @@ -0,0 +1,34 @@ +identity: + author: Doug Lea + name: feishu_message + label: + en_US: Lark Message + zh_Hans: 飞书消息 + description: + en_US: Lark Message + zh_Hans: 飞书消息 + icon: icon.svg + tags: + - social + - productivity +credentials_for_provider: + app_id: + type: text-input + required: true + label: + en_US: APP ID + placeholder: + en_US: Please input your feishu app id + zh_Hans: 请输入你的飞书 app id + help: + en_US: Get your app_id and app_secret from Feishu + zh_Hans: 从飞书获取您的 app_id 和 app_secret + url: https://open.feishu.cn + app_secret: + type: secret-input + required: true + label: + en_US: APP Secret + placeholder: + en_US: Please input your app secret + zh_Hans: 请输入你的飞书 app secret diff --git a/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.py b/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.py new file mode 100644 index 0000000000..74f6866ba3 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.py @@ -0,0 +1,20 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class SendBotMessageTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) -> ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + receive_id_type = tool_parameters.get('receive_id_type') + receive_id = tool_parameters.get('receive_id') + msg_type = tool_parameters.get('msg_type') + content = tool_parameters.get('content') + + res = client.send_bot_message(receive_id_type, receive_id, msg_type, content) + return self.create_json_message(res) diff --git a/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.yaml b/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.yaml new file mode 100644 index 0000000000..6e398b18ab --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/tools/send_bot_message.yaml @@ -0,0 +1,91 @@ +identity: + name: send_bot_message + author: Doug Lea + label: + en_US: Send Bot Message + zh_Hans: 发送飞书应用消息 +description: + human: + en_US: Send bot message + zh_Hans: 发送飞书应用消息 + llm: A tool for sending Feishu application messages. +parameters: + - name: receive_id_type + type: select + required: true + options: + - value: open_id + label: + en_US: open id + zh_Hans: open id + - value: union_id + label: + en_US: union id + zh_Hans: union id + - value: user_id + label: + en_US: user id + zh_Hans: user id + - value: email + label: + en_US: email + zh_Hans: email + - value: chat_id + label: + en_US: chat id + zh_Hans: chat id + label: + en_US: User ID Type + zh_Hans: 用户 ID 类型 + human_description: + en_US: User ID Type + zh_Hans: 用户 ID 类型，可选值有 open_id、union_id、user_id、email、chat_id。 + llm_description: 用户 ID 类型，可选值有 open_id、union_id、user_id、email、chat_id。 + form: llm + + - name: receive_id + type: string + required: true + label: + en_US: Receive Id + zh_Hans: 消息接收者的 ID + human_description: + en_US: The ID of the message receiver. The ID type should correspond to the query parameter receive_id_type. + zh_Hans: 消息接收者的 ID，ID 类型应与查询参数 receive_id_type 对应。 + llm_description: 消息接收者的 ID，ID 类型应与查询参数 receive_id_type 对应。 + form: llm + + - name: msg_type + type: string + required: true + options: + - value: text + label: + en_US: text + zh_Hans: 文本 + - value: interactive + label: + en_US: message card + zh_Hans: 消息卡片 + label: + en_US: Message type + zh_Hans: 消息类型 + human_description: + en_US: Message type, optional values are, text (text), interactive (message card). + zh_Hans: 消息类型，可选值有：text(文本)、interactive(消息卡片)。 + llm_description: 消息类型，可选值有：text(文本)、interactive(消息卡片)。 + form: llm + + - name: content + type: string + required: true + label: + en_US: Message content + zh_Hans: 消息内容 + human_description: + en_US: Message content + zh_Hans: | + 消息内容，JSON 结构序列化后的字符串。不同 msg_type 对应不同内容， + 具体格式说明参考：https://open.larkoffice.com/document/server-docs/im-v1/message-content-description/create_json + llm_description: 消息内容，JSON 结构序列化后的字符串。不同 msg_type 对应不同内容。 + form: llm diff --git a/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.py b/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.py new file mode 100644 index 0000000000..7159f59ffa --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.py @@ -0,0 +1,19 @@ +from typing import Any + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool +from core.tools.utils.feishu_api_utils import FeishuRequest + + +class SendWebhookMessageTool(BuiltinTool): + def _invoke(self, user_id: str, tool_parameters: dict[str, Any]) ->ToolInvokeMessage: + app_id = self.runtime.credentials.get('app_id') + app_secret = self.runtime.credentials.get('app_secret') + client = FeishuRequest(app_id, app_secret) + + webhook = tool_parameters.get('webhook') + msg_type = tool_parameters.get('msg_type') + content = tool_parameters.get('content') + + res = client.send_webhook_message(webhook, msg_type, content) + return self.create_json_message(res) diff --git a/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.yaml b/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.yaml new file mode 100644 index 0000000000..8b39ce4874 --- /dev/null +++ b/api/core/tools/provider/builtin/feishu_message/tools/send_webhook_message.yaml @@ -0,0 +1,58 @@ +identity: + name: send_webhook_message + author: Doug Lea + label: + en_US: Send Webhook Message + zh_Hans: 使用自定义机器人发送飞书消息 +description: + human: + en_US: Send webhook message + zh_Hans: 使用自定义机器人发送飞书消息 + llm: A tool for sending Lark messages using a custom robot. +parameters: + - name: webhook + type: string + required: true + label: + en_US: webhook + zh_Hans: webhook 的地址 + human_description: + en_US: The address of the webhook + zh_Hans: webhook 的地址 + llm_description: webhook 的地址 + form: llm + + - name: msg_type + type: string + required: true + options: + - value: text + label: + en_US: text + zh_Hans: 文本 + - value: interactive + label: + en_US: message card + zh_Hans: 消息卡片 + label: + en_US: Message type + zh_Hans: 消息类型 + human_description: + en_US: Message type, optional values are, text (text), interactive (message card). + zh_Hans: 消息类型，可选值有：text(文本)、interactive(消息卡片)。 + llm_description: 消息类型，可选值有：text(文本)、interactive(消息卡片)。 + form: llm + + - name: content + type: string + required: true + label: + en_US: Message content + zh_Hans: 消息内容 + human_description: + en_US: Message content + zh_Hans: | + 消息内容，JSON 结构序列化后的字符串。不同 msg_type 对应不同内容， + 具体格式说明参考：https://open.larkoffice.com/document/server-docs/im-v1/message-content-description/create_json + llm_description: 消息内容，JSON 结构序列化后的字符串。不同 msg_type 对应不同内容。 + form: llm diff --git a/api/core/tools/utils/feishu_api_utils.py b/api/core/tools/utils/feishu_api_utils.py new file mode 100644 index 0000000000..e6b288868f --- /dev/null +++ b/api/core/tools/utils/feishu_api_utils.py @@ -0,0 +1,143 @@ +import httpx + +from extensions.ext_redis import redis_client + + +class FeishuRequest: + def __init__(self, app_id: str, app_secret: str): + self.app_id = app_id + self.app_secret = app_secret + + @property + def tenant_access_token(self): + feishu_tenant_access_token = f"tools:{self.app_id}:feishu_tenant_access_token" + if redis_client.exists(feishu_tenant_access_token): + return redis_client.get(feishu_tenant_access_token).decode() + res = self.get_tenant_access_token(self.app_id, self.app_secret) + redis_client.setex(feishu_tenant_access_token, res.get("expire"), res.get("tenant_access_token")) + return res.get("tenant_access_token") + + def _send_request(self, url: str, method: str = "post", require_token: bool = True, payload: dict = None, + params: dict = None): + headers = { + "Content-Type": "application/json", + "user-agent": "Dify", + } + if require_token: + headers["tenant-access-token"] = f"{self.tenant_access_token}" + res = httpx.request(method=method, url=url, headers=headers, json=payload, params=params, timeout=30).json() + if res.get("code") != 0: + raise Exception(res) + return res + + def get_tenant_access_token(self, app_id: str, app_secret: str) -> dict: + """ + API url: https://open.feishu.cn/document/server-docs/authentication-management/access-token/tenant_access_token_internal + Example Response: + { + "code": 0, + "msg": "ok", + "tenant_access_token": "t-caecc734c2e3328a62489fe0648c4b98779515d3", + "expire": 7200 + } + """ + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/access_token/get_tenant_access_token" + payload = { + "app_id": app_id, + "app_secret": app_secret + } + res = self._send_request(url, require_token=False, payload=payload) + return res + + def create_document(self, title: str, content: str, folder_token: str) -> dict: + """ + API url: https://open.larkoffice.com/document/server-docs/docs/docs/docx-v1/document/create + Example Response: + { + "data": { + "title": "title", + "url": "https://svi136aogf123.feishu.cn/docx/VWbvd4fEdoW0WSxaY1McQTz8n7d", + "type": "docx", + "token": "VWbvd4fEdoW0WSxaY1McQTz8n7d" + }, + "log_id": "021721281231575fdbddc0200ff00060a9258ec0000103df61b5d", + "code": 0, + "msg": "创建飞书文档成功，请查看" + } + """ + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/document/create_document" + payload = { + "title": title, + "content": content, + "folder_token": folder_token, + } + res = self._send_request(url, payload=payload) + return res.get("data") + + def write_document(self, document_id: str, content: str, position: str = "start") -> dict: + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/document/write_document" + payload = { + "document_id": document_id, + "content": content, + "position": position + } + res = self._send_request(url, payload=payload) + return res.get("data") + + def get_document_raw_content(self, document_id: str) -> dict: + """ + API url: https://open.larkoffice.com/document/server-docs/docs/docs/docx-v1/document/raw_content + Example Response: + { + "code": 0, + "msg": "success", + "data": { + "content": "云文档\n多人实时协同，插入一切元素。不仅是在线文档，更是强大的创作和互动工具\n云文档：专为协作而生\n" + } + } + """ + params = { + "document_id": document_id, + } + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/document/get_document_raw_content" + res = self._send_request(url, method="get", params=params) + return res.get("data").get("content") + + def list_document_block(self, document_id: str, page_token: str, page_size: int = 500) -> dict: + """ + API url: https://open.larkoffice.com/document/server-docs/docs/docs/docx-v1/document/list + """ + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/document/list_document_block" + params = { + "document_id": document_id, + "page_size": page_size, + "page_token": page_token, + } + res = self._send_request(url, method="get", params=params) + return res.get("data") + + def send_bot_message(self, receive_id_type: str, receive_id: str, msg_type: str, content: str) -> dict: + """ + API url: https://open.larkoffice.com/document/server-docs/im-v1/message/create + """ + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/message/send_bot_message" + params = { + "receive_id_type": receive_id_type, + } + payload = { + "receive_id": receive_id, + "msg_type": msg_type, + "content": content, + } + res = self._send_request(url, params=params, payload=payload) + return res.get("data") + + def send_webhook_message(self, webhook: str, msg_type: str, content: str) -> dict: + url = "https://lark-plugin-api.solutionsuite.cn/lark-plugin/message/send_webhook_message" + payload = { + "webhook": webhook, + "msg_type": msg_type, + "content": content, + } + res = self._send_request(url, require_token=False, payload=payload) + return res From e38334cfd2535a6981236ae4e8f76759b4aab75f Mon Sep 17 00:00:00 2001 From: Jiakun Xu Date: Wed, 28 Aug 2024 08:45:51 +0800 Subject: [PATCH 008/125] fix: doc_language return null when document segment settings (#7719) --- api/controllers/console/datasets/datasets_document.py | 2 ++ web/app/components/datasets/create/step-two/index.tsx | 4 +++- web/models/datasets.ts | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 7d0b9f0460..6bc29a8643 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -599,6 +599,7 @@ class DocumentDetailApi(DocumentResource): "hit_count": document.hit_count, "display_status": document.display_status, "doc_form": document.doc_form, + "doc_language": document.doc_language, } else: process_rules = DatasetService.get_process_rules(dataset_id) @@ -631,6 +632,7 @@ class DocumentDetailApi(DocumentResource): "hit_count": document.hit_count, "display_status": document.display_status, "doc_form": document.doc_form, + "doc_language": document.doc_language, } return response, 200 diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 890ceea630..10b378d8c5 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -123,7 +123,9 @@ const StepTwo = ({ const [docForm, setDocForm] = useState( (datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT, ) - const [docLanguage, setDocLanguage] = useState(locale !== LanguagesSupported[1] ? 'English' : 'Chinese') + const [docLanguage, setDocLanguage] = useState( + (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'), + ) const [QATipHide, setQATipHide] = useState(false) const [previewSwitched, setPreviewSwitched] = useState(false) const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean() diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 5731ec7646..0ae7831245 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -189,6 +189,7 @@ export type InitialDocumentDetail = { completed_segments?: number total_segments?: number doc_form: 'text_model' | 'qa_model' + doc_language: string } export type SimpleDocumentDetail = InitialDocumentDetail & { From bc3a8e0ca2ad4117a2fe3899f36a59f6c2effc85 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Wed, 28 Aug 2024 09:47:30 +0900 Subject: [PATCH 009/125] feat: store created_by and updated_by for apps, modelconfigs, and sites (#7613) --- api/controllers/console/app/model_config.py | 2 + api/controllers/console/app/site.py | 6 +++ .../create_site_record_when_app_created.py | 2 + api/fields/app_fields.py | 24 +++++++++ api/fields/workflow_fields.py | 8 +++ ...d_add_created_by_and_updated_by_to_app_.py | 52 +++++++++++++++++++ api/models/model.py | 6 +++ api/services/app_dsl_service.py | 4 ++ api/services/app_service.py | 9 ++++ api/services/workflow/workflow_converter.py | 2 + 10 files changed, 115 insertions(+) create mode 100644 api/migrations/versions/2024_08_25_0441-d0187d6a88dd_add_created_by_and_updated_by_to_app_.py diff --git a/api/controllers/console/app/model_config.py b/api/controllers/console/app/model_config.py index 702afe9864..f5068a4cd8 100644 --- a/api/controllers/console/app/model_config.py +++ b/api/controllers/console/app/model_config.py @@ -32,6 +32,8 @@ class ModelConfigResource(Resource): new_app_model_config = AppModelConfig( app_id=app_model.id, + created_by=current_user.id, + updated_by=current_user.id, ) new_app_model_config = new_app_model_config.from_model_config_dict(model_configuration) diff --git a/api/controllers/console/app/site.py b/api/controllers/console/app/site.py index d903a2609f..f936642acd 100644 --- a/api/controllers/console/app/site.py +++ b/api/controllers/console/app/site.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + from flask_login import current_user from flask_restful import Resource, marshal_with, reqparse from werkzeug.exceptions import Forbidden, NotFound @@ -71,6 +73,8 @@ class AppSite(Resource): if value is not None: setattr(site, attr_name, value) + site.updated_by = current_user.id + site.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return site @@ -93,6 +97,8 @@ class AppSiteAccessTokenReset(Resource): raise NotFound site.code = Site.generate_code(16) + site.updated_by = current_user.id + site.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() return site diff --git a/api/events/event_handlers/create_site_record_when_app_created.py b/api/events/event_handlers/create_site_record_when_app_created.py index ab07c5d366..1515661b2d 100644 --- a/api/events/event_handlers/create_site_record_when_app_created.py +++ b/api/events/event_handlers/create_site_record_when_app_created.py @@ -17,6 +17,8 @@ def handle(sender, **kwargs): default_language=account.interface_language, customize_token_strategy="not_allow", code=Site.generate_code(16), + created_by=app.created_by, + updated_by=app.updated_by, ) db.session.add(site) diff --git a/api/fields/app_fields.py b/api/fields/app_fields.py index 26ed686783..45fcb128ce 100644 --- a/api/fields/app_fields.py +++ b/api/fields/app_fields.py @@ -1,5 +1,6 @@ from flask_restful import fields +from fields.workflow_fields import workflow_partial_fields from libs.helper import AppIconUrlField, TimestampField app_detail_kernel_fields = { @@ -39,7 +40,10 @@ model_config_fields = { "completion_prompt_config": fields.Raw(attribute="completion_prompt_config_dict"), "dataset_configs": fields.Raw(attribute="dataset_configs_dict"), "file_upload": fields.Raw(attribute="file_upload_dict"), + "created_by": fields.String, "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, } app_detail_fields = { @@ -52,8 +56,12 @@ app_detail_fields = { "enable_site": fields.Boolean, "enable_api": fields.Boolean, "model_config": fields.Nested(model_config_fields, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_fields, allow_null=True), "tracing": fields.Raw, + "created_by": fields.String, "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, } prompt_config_fields = { @@ -63,6 +71,10 @@ prompt_config_fields = { model_config_partial_fields = { "model": fields.Raw(attribute="model_dict"), "pre_prompt": fields.String, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, } tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String} @@ -78,7 +90,11 @@ app_partial_fields = { "icon_background": fields.String, "icon_url": AppIconUrlField, "model_config": fields.Nested(model_config_partial_fields, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_fields, allow_null=True), + "created_by": fields.String, "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, "tags": fields.List(fields.Nested(tag_fields)), } @@ -124,6 +140,10 @@ site_fields = { "prompt_public": fields.Boolean, "app_base_url": fields.String, "show_workflow_steps": fields.Boolean, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, } app_detail_fields_with_site = { @@ -138,9 +158,13 @@ app_detail_fields_with_site = { "enable_site": fields.Boolean, "enable_api": fields.Boolean, "model_config": fields.Nested(model_config_fields, attribute="app_model_config", allow_null=True), + "workflow": fields.Nested(workflow_partial_fields, allow_null=True), "site": fields.Nested(site_fields), "api_base_url": fields.String, + "created_by": fields.String, "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, "deleted_tools": fields.List(fields.String), } diff --git a/api/fields/workflow_fields.py b/api/fields/workflow_fields.py index 240b8f2eb0..2adef63ada 100644 --- a/api/fields/workflow_fields.py +++ b/api/fields/workflow_fields.py @@ -53,3 +53,11 @@ workflow_fields = { "environment_variables": fields.List(EnvironmentVariableField()), "conversation_variables": fields.List(fields.Nested(conversation_variable_fields)), } + +workflow_partial_fields = { + "id": fields.String, + "created_by": fields.String, + "created_at": TimestampField, + "updated_by": fields.String, + "updated_at": TimestampField, +} diff --git a/api/migrations/versions/2024_08_25_0441-d0187d6a88dd_add_created_by_and_updated_by_to_app_.py b/api/migrations/versions/2024_08_25_0441-d0187d6a88dd_add_created_by_and_updated_by_to_app_.py new file mode 100644 index 0000000000..e0066a302c --- /dev/null +++ b/api/migrations/versions/2024_08_25_0441-d0187d6a88dd_add_created_by_and_updated_by_to_app_.py @@ -0,0 +1,52 @@ +"""add created_by and updated_by to app, modelconfig, and site + +Revision ID: d0187d6a88dd +Revises: 2dbe42621d96 +Create Date: 2024-08-25 04:41:18.157397 + +""" + +import sqlalchemy as sa +from alembic import op + +import models as models + +# revision identifiers, used by Alembic. +revision = "d0187d6a88dd" +down_revision = "2dbe42621d96" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("app_model_configs", schema=None) as batch_op: + batch_op.add_column(sa.Column("created_by", models.types.StringUUID(), nullable=True)) + batch_op.add_column(sa.Column("updated_by", models.types.StringUUID(), nullable=True)) + + with op.batch_alter_table("apps", schema=None) as batch_op: + batch_op.add_column(sa.Column("created_by", models.types.StringUUID(), nullable=True)) + batch_op.add_column(sa.Column("updated_by", models.types.StringUUID(), nullable=True)) + + with op.batch_alter_table("sites", schema=None) as batch_op: + batch_op.add_column(sa.Column("created_by", models.types.StringUUID(), nullable=True)) + batch_op.add_column(sa.Column("updated_by", models.types.StringUUID(), nullable=True)) + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("sites", schema=None) as batch_op: + batch_op.drop_column("updated_by") + batch_op.drop_column("created_by") + + with op.batch_alter_table("apps", schema=None) as batch_op: + batch_op.drop_column("updated_by") + batch_op.drop_column("created_by") + + with op.batch_alter_table("app_model_configs", schema=None) as batch_op: + batch_op.drop_column("updated_by") + batch_op.drop_column("created_by") + + # ### end Alembic commands ### diff --git a/api/models/model.py b/api/models/model.py index 83c56363e3..e2d1fcfc23 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -82,7 +82,9 @@ class App(db.Model): is_universal = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) tracing = db.Column(db.Text, nullable=True) max_active_requests = db.Column(db.Integer, nullable=True) + created_by = db.Column(StringUUID, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_by = db.Column(StringUUID, nullable=True) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) @property @@ -221,7 +223,9 @@ class AppModelConfig(db.Model): provider = db.Column(db.String(255), nullable=True) model_id = db.Column(db.String(255), nullable=True) configs = db.Column(db.JSON, nullable=True) + created_by = db.Column(StringUUID, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_by = db.Column(StringUUID, nullable=True) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) opening_statement = db.Column(db.Text) suggested_questions = db.Column(db.Text) @@ -1115,7 +1119,9 @@ class Site(db.Model): customize_token_strategy = db.Column(db.String(255), nullable=False) prompt_public = db.Column(db.Boolean, nullable=False, server_default=db.text('false')) status = db.Column(db.String(255), nullable=False, server_default=db.text("'normal'::character varying")) + created_by = db.Column(StringUUID, nullable=True) created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) + updated_by = db.Column(StringUUID, nullable=True) updated_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)')) code = db.Column(db.String(255)) diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index a938b4f93b..a2aa15ed4b 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -346,6 +346,8 @@ class AppDslService: app_model_config = AppModelConfig() app_model_config = app_model_config.from_model_config_dict(model_config_data) app_model_config.app_id = app.id + app_model_config.created_by = account.id + app_model_config.updated_by = account.id db.session.add(app_model_config) db.session.commit() @@ -390,6 +392,8 @@ class AppDslService: icon_background=icon_background, enable_site=True, enable_api=True, + created_by=account.id, + updated_by=account.id, ) db.session.add(app) diff --git a/api/services/app_service.py b/api/services/app_service.py index 8a2f8c0535..462613fb7d 100644 --- a/api/services/app_service.py +++ b/api/services/app_service.py @@ -127,6 +127,8 @@ class AppService: app.tenant_id = tenant_id app.api_rph = args.get("api_rph", 0) app.api_rpm = args.get("api_rpm", 0) + app.created_by = account.id + app.updated_by = account.id db.session.add(app) db.session.flush() @@ -134,6 +136,8 @@ class AppService: if default_model_config: app_model_config = AppModelConfig(**default_model_config) app_model_config.app_id = app.id + app_model_config.created_by = account.id + app_model_config.updated_by = account.id db.session.add(app_model_config) db.session.flush() @@ -217,6 +221,7 @@ class AppService: app.icon_type = args.get("icon_type", "emoji") app.icon = args.get("icon") app.icon_background = args.get("icon_background") + app.updated_by = current_user.id app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() @@ -233,6 +238,7 @@ class AppService: :return: App instance """ app.name = name + app.updated_by = current_user.id app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() @@ -248,6 +254,7 @@ class AppService: """ app.icon = icon app.icon_background = icon_background + app.updated_by = current_user.id app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() @@ -264,6 +271,7 @@ class AppService: return app app.enable_site = enable_site + app.updated_by = current_user.id app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() @@ -280,6 +288,7 @@ class AppService: return app app.enable_api = enable_api + app.updated_by = current_user.id app.updated_at = datetime.now(timezone.utc).replace(tzinfo=None) db.session.commit() diff --git a/api/services/workflow/workflow_converter.py b/api/services/workflow/workflow_converter.py index c9057bd0e5..4b845be2f4 100644 --- a/api/services/workflow/workflow_converter.py +++ b/api/services/workflow/workflow_converter.py @@ -74,6 +74,8 @@ class WorkflowConverter: new_app.api_rph = app_model.api_rph new_app.is_demo = False new_app.is_public = app_model.is_public + new_app.created_by = account.id + new_app.updated_by = account.id db.session.add(new_app) db.session.flush() db.session.commit() From 693fe912f283c633677cc1b85dfc5876b7a790e2 Mon Sep 17 00:00:00 2001 From: KVOJJJin Date: Wed, 28 Aug 2024 09:42:54 +0800 Subject: [PATCH 010/125] Fix annotation reply settings (#7696) --- web/app/components/app/annotation/index.tsx | 2 +- .../toolbox/annotation/config-param.tsx | 2 +- .../components/app/configuration/toolbox/index.tsx | 2 +- web/app/components/app/log-annotation/index.tsx | 14 +++++++++----- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/web/app/components/app/annotation/index.tsx b/web/app/components/app/annotation/index.tsx index 1e65d7a94f..0f54f5bfc3 100644 --- a/web/app/components/app/annotation/index.tsx +++ b/web/app/components/app/annotation/index.tsx @@ -280,7 +280,7 @@ const Annotation: FC = ({ onSave={async (embeddingModel, score) => { if ( embeddingModel.embedding_model_name !== annotationConfig?.embedding_model?.embedding_model_name - && embeddingModel.embedding_provider_name !== annotationConfig?.embedding_model?.embedding_provider_name + || embeddingModel.embedding_provider_name !== annotationConfig?.embedding_model?.embedding_provider_name ) { const { job_id: jobId }: any = await updateAnnotationStatus(appDetail.id, AnnotationEnableStatus.enable, embeddingModel, score) await ensureJobCompleted(jobId, AnnotationEnableStatus.enable) diff --git a/web/app/components/app/configuration/toolbox/annotation/config-param.tsx b/web/app/components/app/configuration/toolbox/annotation/config-param.tsx index e49c69e373..e418a76c34 100644 --- a/web/app/components/app/configuration/toolbox/annotation/config-param.tsx +++ b/web/app/components/app/configuration/toolbox/annotation/config-param.tsx @@ -98,7 +98,7 @@ const AnnotationReplyConfig: FC = ({ let isEmbeddingModelChanged = false if ( embeddingModel.embedding_model_name !== annotationConfig.embedding_model.embedding_model_name - && embeddingModel.embedding_provider_name !== annotationConfig.embedding_model.embedding_provider_name + || embeddingModel.embedding_provider_name !== annotationConfig.embedding_model.embedding_provider_name ) { await onEmbeddingChange(embeddingModel) isEmbeddingModelChanged = true diff --git a/web/app/components/app/configuration/toolbox/index.tsx b/web/app/components/app/configuration/toolbox/index.tsx index 488ca86b90..00ea301a42 100644 --- a/web/app/components/app/configuration/toolbox/index.tsx +++ b/web/app/components/app/configuration/toolbox/index.tsx @@ -32,7 +32,7 @@ const Toolbox: FC = ({ ) } { - (showAnnotation || true) && ( + showAnnotation && ( = ({ const router = useRouter() const appDetail = useAppStore(state => state.appDetail) - const options = [ - { value: PageType.log, text: t('appLog.title') }, - { value: PageType.annotation, text: t('appAnnotation.title') }, - ] + const options = useMemo(() => { + if (appDetail?.mode === 'completion') + return [{ value: PageType.log, text: t('appLog.title') }] + return [ + { value: PageType.log, text: t('appLog.title') }, + { value: PageType.annotation, text: t('appAnnotation.title') }, + ] + }, [appDetail]) if (!appDetail) { return ( From 7cfebffbb883f942dc683a3739a299ea4839f0a6 Mon Sep 17 00:00:00 2001 From: sino Date: Wed, 28 Aug 2024 13:56:50 +0800 Subject: [PATCH 011/125] chore: update default endpoint for ark provider (#7741) --- .../model_providers/volcengine_maas/client.py | 21 ++++++++++++++----- .../volcengine_maas/volcengine_maas.yaml | 2 +- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/api/core/model_runtime/model_providers/volcengine_maas/client.py b/api/core/model_runtime/model_providers/volcengine_maas/client.py index 5100494e58..61f3521a43 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/client.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/client.py @@ -32,6 +32,9 @@ from core.model_runtime.entities.message_entities import ( UserPromptMessage, ) +DEFAULT_V2_ENDPOINT = "maas-api.ml-platform-cn-beijing.volces.com" +DEFAULT_V3_ENDPOINT = "https://ark.cn-beijing.volces.com/api/v3" + class ArkClientV3: endpoint_id: Optional[str] = None @@ -43,16 +46,24 @@ class ArkClientV3: @staticmethod def is_legacy(credentials: dict) -> bool: + # match default v2 endpoint if ArkClientV3.is_compatible_with_legacy(credentials): return False - sdk_version = credentials.get("sdk_version", "v2") - return sdk_version != "v3" + # match default v3 endpoint + if credentials.get("api_endpoint_host") == DEFAULT_V3_ENDPOINT: + return False + # only v3 support api_key + if credentials.get("auth_method") == "api_key": + return False + # these cases are considered as sdk v2 + # - modified default v2 endpoint + # - modified default v3 endpoint and auth without api_key + return True @staticmethod def is_compatible_with_legacy(credentials: dict) -> bool: - sdk_version = credentials.get("sdk_version") endpoint = credentials.get("api_endpoint_host") - return sdk_version is None and endpoint == "maas-api.ml-platform-cn-beijing.volces.com" + return endpoint == DEFAULT_V2_ENDPOINT @classmethod def from_credentials(cls, credentials): @@ -64,7 +75,7 @@ class ArkClientV3: "sk": credentials['volc_secret_access_key'], } if cls.is_compatible_with_legacy(credentials): - args["base_url"] = "https://ark.cn-beijing.volces.com/api/v3" + args["base_url"] = DEFAULT_V3_ENDPOINT client = ArkClientV3( **args diff --git a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml index a00c1b7994..735ba2b314 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml +++ b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml @@ -64,7 +64,7 @@ model_credential_schema: en_US: API Endpoint Host zh_Hans: API Endpoint Host type: text-input - default: maas-api.ml-platform-cn-beijing.volces.com + default: https://ark.cn-beijing.volces.com/api/v3 placeholder: en_US: Enter your API Endpoint Host zh_Hans: 输入 API Endpoint Host From 4682e0ac7c415a3ae5675b15f066f01bc1884bcd Mon Sep 17 00:00:00 2001 From: Vimpas Date: Wed, 28 Aug 2024 13:57:45 +0800 Subject: [PATCH 012/125] =?UTF-8?q?fix(storage):=20=F0=9F=90=9B=20HeadBuck?= =?UTF-8?q?et=20Operation=20Permission=20(#7733)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 莫岳恒 --- api/extensions/storage/s3_storage.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api/extensions/storage/s3_storage.py b/api/extensions/storage/s3_storage.py index 424d441cdc..0858be3af6 100644 --- a/api/extensions/storage/s3_storage.py +++ b/api/extensions/storage/s3_storage.py @@ -35,6 +35,9 @@ class S3Storage(BaseStorage): # if bucket not exists, create it if e.response["Error"]["Code"] == "404": self.client.create_bucket(Bucket=self.bucket_name) + # if bucket is not accessible, pass, maybe the bucket is existing but not accessible + elif e.response["Error"]["Code"] == "403": + pass else: # other error, raise exception raise From 9342b4b95192c0372ce9a3a79aa485c4962ba2fe Mon Sep 17 00:00:00 2001 From: snickerjp Date: Wed, 28 Aug 2024 15:44:05 +0900 Subject: [PATCH 013/125] Update package "libldap-2.5-0" for docker build. (#7726) --- api/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/Dockerfile b/api/Dockerfile index cca6488679..82b89ad77b 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -55,7 +55,7 @@ RUN apt-get update \ && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \ && apt-get update \ # For Security - && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-1 libldap-2.5-0=2.5.18+dfsg-2 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \ + && apt-get install -y --no-install-recommends zlib1g=1:1.3.dfsg+really1.3.1-1 expat=2.6.2-1 libldap-2.5-0=2.5.18+dfsg-3 perl=5.38.2-5 libsqlite3-0=3.46.0-1 \ && apt-get autoremove -y \ && rm -rf /var/lib/apt/lists/* From 3a071b8db98c004fdd8049a6ed0fce5321978267 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:36:11 +0800 Subject: [PATCH 014/125] fix: datasets permission is missing (#7751) --- api/services/dataset_service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 8649d0fea5..ce5fa632dc 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -136,7 +136,9 @@ class DatasetService: return datasets.items, datasets.total @staticmethod - def create_empty_dataset(tenant_id: str, name: str, indexing_technique: Optional[str], account: Account): + def create_empty_dataset( + tenant_id: str, name: str, indexing_technique: Optional[str], account: Account, permission: Optional[str] + ): # check if dataset name already exists if Dataset.query.filter_by(name=name, tenant_id=tenant_id).first(): raise DatasetNameDuplicateError(f"Dataset with name {name} already exists.") @@ -153,6 +155,7 @@ class DatasetService: dataset.tenant_id = tenant_id dataset.embedding_model_provider = embedding_model.provider if embedding_model else None dataset.embedding_model = embedding_model.model if embedding_model else None + dataset.permission = permission if permission else DatasetPermissionEnum.ONLY_ME db.session.add(dataset) db.session.commit() return dataset From 7541a492b79dbb5ad6bf6043eeafc75767fe14ed Mon Sep 17 00:00:00 2001 From: Joel Date: Wed, 28 Aug 2024 17:16:07 +0800 Subject: [PATCH 015/125] fix: crawl options max length can not set 0 (#7758) Co-authored-by: Yi --- .../components/datasets/create/website/firecrawl/base/field.tsx | 2 +- .../components/datasets/create/website/firecrawl/base/input.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/app/components/datasets/create/website/firecrawl/base/field.tsx b/web/app/components/datasets/create/website/firecrawl/base/field.tsx index cac40798c1..5b5ca90c5d 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/field.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/field.tsx @@ -38,7 +38,7 @@ const Field: FC = ({ popupContent={
{tooltip}
} - popupClassName='relative top-[3px] w-3 h-3 ml-1' + triggerClassName='ml-0.5 w-4 h-4' /> )}
diff --git a/web/app/components/datasets/create/website/firecrawl/base/input.tsx b/web/app/components/datasets/create/website/firecrawl/base/input.tsx index 06249f57e7..7d2d2b609f 100644 --- a/web/app/components/datasets/create/website/firecrawl/base/input.tsx +++ b/web/app/components/datasets/create/website/firecrawl/base/input.tsx @@ -9,7 +9,7 @@ type Props = { isNumber?: boolean } -const MIN_VALUE = 1 +const MIN_VALUE = 0 const Input: FC = ({ value, From 5d0914daea66aaef9d0ee4f86672dae198483043 Mon Sep 17 00:00:00 2001 From: Leheng Lu <55642493+Leheng620@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:25:20 +0800 Subject: [PATCH 016/125] fix: not able to pass array of string/number/object into variable aggregator groups (#7757) --- api/core/workflow/nodes/variable_aggregator/entities.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/workflow/nodes/variable_aggregator/entities.py b/api/core/workflow/nodes/variable_aggregator/entities.py index cea88334b9..e5de38dc0f 100644 --- a/api/core/workflow/nodes/variable_aggregator/entities.py +++ b/api/core/workflow/nodes/variable_aggregator/entities.py @@ -17,7 +17,7 @@ class AdvancedSettings(BaseModel): """ Group. """ - output_type: Literal['string', 'number', 'array', 'object'] + output_type: Literal['string', 'number', 'object', 'array[string]', 'array[number]', 'array[object]'] variables: list[list[str]] group_name: str @@ -30,4 +30,4 @@ class VariableAssignerNodeData(BaseNodeData): type: str = 'variable-assigner' output_type: str variables: list[list[str]] - advanced_settings: Optional[AdvancedSettings] = None \ No newline at end of file + advanced_settings: Optional[AdvancedSettings] = None From 26abbe8e5bf7d957617dc1e00b648d897f4e96db Mon Sep 17 00:00:00 2001 From: zhuhao <37029601+hwzhuhao@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:27:20 +0800 Subject: [PATCH 017/125] feat(Tools): add a tool to query the stock price from Alpha Vantage (#7019) (#7752) --- api/core/tools/provider/_position.yaml | 1 + .../builtin/alphavantage/_assets/icon.svg | 7 +++ .../builtin/alphavantage/alphavantage.py | 22 +++++++++ .../builtin/alphavantage/alphavantage.yaml | 31 ++++++++++++ .../builtin/alphavantage/tools/query_stock.py | 49 +++++++++++++++++++ .../alphavantage/tools/query_stock.yaml | 27 ++++++++++ 6 files changed, 137 insertions(+) create mode 100644 api/core/tools/provider/builtin/alphavantage/_assets/icon.svg create mode 100644 api/core/tools/provider/builtin/alphavantage/alphavantage.py create mode 100644 api/core/tools/provider/builtin/alphavantage/alphavantage.yaml create mode 100644 api/core/tools/provider/builtin/alphavantage/tools/query_stock.py create mode 100644 api/core/tools/provider/builtin/alphavantage/tools/query_stock.yaml diff --git a/api/core/tools/provider/_position.yaml b/api/core/tools/provider/_position.yaml index b804089570..9b90dda3b2 100644 --- a/api/core/tools/provider/_position.yaml +++ b/api/core/tools/provider/_position.yaml @@ -10,6 +10,7 @@ - wikipedia - nominatim - yahoo +- alphavantage - arxiv - pubmed - stablediffusion diff --git a/api/core/tools/provider/builtin/alphavantage/_assets/icon.svg b/api/core/tools/provider/builtin/alphavantage/_assets/icon.svg new file mode 100644 index 0000000000..785432943b --- /dev/null +++ b/api/core/tools/provider/builtin/alphavantage/_assets/icon.svg @@ -0,0 +1,7 @@ + + + 形状结合 + + + + \ No newline at end of file diff --git a/api/core/tools/provider/builtin/alphavantage/alphavantage.py b/api/core/tools/provider/builtin/alphavantage/alphavantage.py new file mode 100644 index 0000000000..01f2acfb5b --- /dev/null +++ b/api/core/tools/provider/builtin/alphavantage/alphavantage.py @@ -0,0 +1,22 @@ +from typing import Any + +from core.tools.errors import ToolProviderCredentialValidationError +from core.tools.provider.builtin.alphavantage.tools.query_stock import QueryStockTool +from core.tools.provider.builtin_tool_provider import BuiltinToolProviderController + + +class AlphaVantageProvider(BuiltinToolProviderController): + def _validate_credentials(self, credentials: dict[str, Any]) -> None: + try: + QueryStockTool().fork_tool_runtime( + runtime={ + "credentials": credentials, + } + ).invoke( + user_id='', + tool_parameters={ + "code": "AAPL", # Apple Inc. + }, + ) + except Exception as e: + raise ToolProviderCredentialValidationError(str(e)) diff --git a/api/core/tools/provider/builtin/alphavantage/alphavantage.yaml b/api/core/tools/provider/builtin/alphavantage/alphavantage.yaml new file mode 100644 index 0000000000..710510cfd8 --- /dev/null +++ b/api/core/tools/provider/builtin/alphavantage/alphavantage.yaml @@ -0,0 +1,31 @@ +identity: + author: zhuhao + name: alphavantage + label: + en_US: AlphaVantage + zh_Hans: AlphaVantage + pt_BR: AlphaVantage + description: + en_US: AlphaVantage is an online platform that provides financial market data and APIs, making it convenient for individual investors and developers to access stock quotes, technical indicators, and stock analysis. + zh_Hans: AlphaVantage是一个在线平台，它提供金融市场数据和API，便于个人投资者和开发者获取股票报价、技术指标和股票分析。 + pt_BR: AlphaVantage is an online platform that provides financial market data and APIs, making it convenient for individual investors and developers to access stock quotes, technical indicators, and stock analysis. + icon: icon.svg + tags: + - finance +credentials_for_provider: + api_key: + type: secret-input + required: true + label: + en_US: AlphaVantage API key + zh_Hans: AlphaVantage API key + pt_BR: AlphaVantage API key + placeholder: + en_US: Please input your AlphaVantage API key + zh_Hans: 请输入你的 AlphaVantage API key + pt_BR: Please input your AlphaVantage API key + help: + en_US: Get your AlphaVantage API key from AlphaVantage + zh_Hans: 从 AlphaVantage 获取您的 AlphaVantage API key + pt_BR: Get your AlphaVantage API key from AlphaVantage + url: https://www.alphavantage.co/support/#api-key diff --git a/api/core/tools/provider/builtin/alphavantage/tools/query_stock.py b/api/core/tools/provider/builtin/alphavantage/tools/query_stock.py new file mode 100644 index 0000000000..5c379b746d --- /dev/null +++ b/api/core/tools/provider/builtin/alphavantage/tools/query_stock.py @@ -0,0 +1,49 @@ +from typing import Any, Union + +import requests + +from core.tools.entities.tool_entities import ToolInvokeMessage +from core.tools.tool.builtin_tool import BuiltinTool + +ALPHAVANTAGE_API_URL = "https://www.alphavantage.co/query" + + +class QueryStockTool(BuiltinTool): + + def _invoke(self, + user_id: str, + tool_parameters: dict[str, Any], + ) -> Union[ToolInvokeMessage, list[ToolInvokeMessage]]: + + stock_code = tool_parameters.get('code', '') + if not stock_code: + return self.create_text_message('Please tell me your stock code') + + if 'api_key' not in self.runtime.credentials or not self.runtime.credentials.get('api_key'): + return self.create_text_message("Alpha Vantage API key is required.") + + params = { + "function": "TIME_SERIES_DAILY", + "symbol": stock_code, + "outputsize": "compact", + "datatype": "json", + "apikey": self.runtime.credentials['api_key'] + } + response = requests.get(url=ALPHAVANTAGE_API_URL, params=params) + response.raise_for_status() + result = self._handle_response(response.json()) + return self.create_json_message(result) + + def _handle_response(self, response: dict[str, Any]) -> dict[str, Any]: + result = response.get('Time Series (Daily)', {}) + if not result: + return {} + stock_result = {} + for k, v in result.items(): + stock_result[k] = {} + stock_result[k]['open'] = v.get('1. open') + stock_result[k]['high'] = v.get('2. high') + stock_result[k]['low'] = v.get('3. low') + stock_result[k]['close'] = v.get('4. close') + stock_result[k]['volume'] = v.get('5. volume') + return stock_result diff --git a/api/core/tools/provider/builtin/alphavantage/tools/query_stock.yaml b/api/core/tools/provider/builtin/alphavantage/tools/query_stock.yaml new file mode 100644 index 0000000000..d89f34e373 --- /dev/null +++ b/api/core/tools/provider/builtin/alphavantage/tools/query_stock.yaml @@ -0,0 +1,27 @@ +identity: + name: query_stock + author: zhuhao + label: + en_US: query_stock + zh_Hans: query_stock + pt_BR: query_stock +description: + human: + en_US: Retrieve information such as daily opening price, daily highest price, daily lowest price, daily closing price, and daily trading volume for a specified stock symbol. + zh_Hans: 获取指定股票代码的每日开盘价、每日最高价、每日最低价、每日收盘价和每日交易量等信息。 + pt_BR: Retrieve information such as daily opening price, daily highest price, daily lowest price, daily closing price, and daily trading volume for a specified stock symbol + llm: Retrieve information such as daily opening price, daily highest price, daily lowest price, daily closing price, and daily trading volume for a specified stock symbol +parameters: + - name: code + type: string + required: true + label: + en_US: stock code + zh_Hans: 股票代码 + pt_BR: stock code + human_description: + en_US: stock code + zh_Hans: 股票代码 + pt_BR: stock code + llm_description: stock code for query from alphavantage + form: llm From 3a67fc6c5af17345abc947409e65f56d10b31282 Mon Sep 17 00:00:00 2001 From: YidaHu Date: Wed, 28 Aug 2024 17:30:13 +0800 Subject: [PATCH 018/125] feat: add support for array types in available variable list (#7715) --- .../workflow/nodes/http/components/edit-body/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/http/components/edit-body/index.tsx b/web/app/components/workflow/nodes/http/components/edit-body/index.tsx index 645bcdcdf9..6e8f4eac3b 100644 --- a/web/app/components/workflow/nodes/http/components/edit-body/index.tsx +++ b/web/app/components/workflow/nodes/http/components/edit-body/index.tsx @@ -44,7 +44,7 @@ const EditBody: FC = ({ const { availableVars, availableNodes } = useAvailableVarList(nodeId, { onlyLeafNodeVar: false, filterVar: (varPayload: Var) => { - return [VarType.string, VarType.number, VarType.secret].includes(varPayload.type) + return [VarType.string, VarType.number, VarType.secret, VarType.arrayNumber, VarType.arrayString].includes(varPayload.type) }, }) From 1262277714ba00120f61c71cbae309e06a520622 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 28 Aug 2024 17:46:37 +0800 Subject: [PATCH 019/125] chore: improve http executor configs (#7730) --- .../workflow/nodes/http_request/entities.py | 10 ++---- .../nodes/http_request/http_request_node.py | 31 +++++++++---------- 2 files changed, 18 insertions(+), 23 deletions(-) diff --git a/api/core/workflow/nodes/http_request/entities.py b/api/core/workflow/nodes/http_request/entities.py index 90d644e0e2..c066d469d8 100644 --- a/api/core/workflow/nodes/http_request/entities.py +++ b/api/core/workflow/nodes/http_request/entities.py @@ -5,10 +5,6 @@ from pydantic import BaseModel, ValidationInfo, field_validator from configs import dify_config from core.workflow.entities.base_node_data_entities import BaseNodeData -MAX_CONNECT_TIMEOUT = dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT -MAX_READ_TIMEOUT = dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT -MAX_WRITE_TIMEOUT = dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT - class HttpRequestNodeAuthorizationConfig(BaseModel): type: Literal[None, 'basic', 'bearer', 'custom'] @@ -41,9 +37,9 @@ class HttpRequestNodeBody(BaseModel): class HttpRequestNodeTimeout(BaseModel): - connect: int = MAX_CONNECT_TIMEOUT - read: int = MAX_READ_TIMEOUT - write: int = MAX_WRITE_TIMEOUT + connect: int = dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT + read: int = dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT + write: int = dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT class HttpRequestNodeData(BaseNodeData): diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index 1facf8a4f4..f6c8ea3c83 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -3,6 +3,7 @@ from mimetypes import guess_extension from os import path from typing import cast +from configs import dify_config from core.app.segments import parser from core.file.file_obj import FileTransferMethod, FileType, FileVar from core.tools.tool_file_manager import ToolFileManager @@ -11,9 +12,6 @@ from core.workflow.entities.node_entities import NodeRunResult, NodeType from core.workflow.entities.variable_pool import VariablePool from core.workflow.nodes.base_node import BaseNode from core.workflow.nodes.http_request.entities import ( - MAX_CONNECT_TIMEOUT, - MAX_READ_TIMEOUT, - MAX_WRITE_TIMEOUT, HttpRequestNodeData, HttpRequestNodeTimeout, ) @@ -21,9 +19,9 @@ from core.workflow.nodes.http_request.http_executor import HttpExecutor, HttpExe from models.workflow import WorkflowNodeExecutionStatus HTTP_REQUEST_DEFAULT_TIMEOUT = HttpRequestNodeTimeout( - connect=min(10, MAX_CONNECT_TIMEOUT), - read=min(60, MAX_READ_TIMEOUT), - write=min(20, MAX_WRITE_TIMEOUT), + connect=min(10, dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT), + read=min(60, dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT), + write=min(20, dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT), ) @@ -43,9 +41,9 @@ class HttpRequestNode(BaseNode): 'body': {'type': 'none'}, 'timeout': { **HTTP_REQUEST_DEFAULT_TIMEOUT.model_dump(), - 'max_connect_timeout': MAX_CONNECT_TIMEOUT, - 'max_read_timeout': MAX_READ_TIMEOUT, - 'max_write_timeout': MAX_WRITE_TIMEOUT, + 'max_connect_timeout': dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT, + 'max_read_timeout': dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT, + 'max_write_timeout': dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT, }, }, } @@ -92,17 +90,18 @@ class HttpRequestNode(BaseNode): }, ) - def _get_request_timeout(self, node_data: HttpRequestNodeData) -> HttpRequestNodeTimeout: + @staticmethod + def _get_request_timeout(node_data: HttpRequestNodeData) -> HttpRequestNodeTimeout: timeout = node_data.timeout if timeout is None: return HTTP_REQUEST_DEFAULT_TIMEOUT - timeout.connect = timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect - timeout.connect = min(timeout.connect, MAX_CONNECT_TIMEOUT) - timeout.read = timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read - timeout.read = min(timeout.read, MAX_READ_TIMEOUT) - timeout.write = timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write - timeout.write = min(timeout.write, MAX_WRITE_TIMEOUT) + timeout.connect = min(timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect, + dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT) + timeout.read = min(timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read, + dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT) + timeout.write = min(timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write, + dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT) return timeout @classmethod From 54b693d5b1811239ca34144f026f8d4914d00317 Mon Sep 17 00:00:00 2001 From: Garfield Dai Date: Wed, 28 Aug 2024 18:55:47 +0800 Subject: [PATCH 020/125] feat: update saas billing hint. (#7760) --- api/controllers/console/datasets/file.py | 2 +- api/controllers/console/wraps.py | 26 +++++++++++------------- api/controllers/service_api/wraps.py | 22 ++++++++------------ 3 files changed, 22 insertions(+), 28 deletions(-) diff --git a/api/controllers/console/datasets/file.py b/api/controllers/console/datasets/file.py index d6a464545e..846aa70e86 100644 --- a/api/controllers/console/datasets/file.py +++ b/api/controllers/console/datasets/file.py @@ -39,7 +39,7 @@ class FileApi(Resource): @login_required @account_initialization_required @marshal_with(file_fields) - @cloud_edition_billing_resource_check(resource="documents") + @cloud_edition_billing_resource_check("documents") def post(self): # get file from request file = request.files["file"] diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 5a964c84fa..7667b30e34 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -46,9 +46,7 @@ def only_edition_self_hosted(view): return decorated -def cloud_edition_billing_resource_check( - resource: str, error_msg: str = "You have reached the limit of your subscription." -): +def cloud_edition_billing_resource_check(resource: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -60,22 +58,22 @@ def cloud_edition_billing_resource_check( documents_upload_quota = features.documents_upload_quota annotation_quota_limit = features.annotation_quota_limit if resource == "members" and 0 < members.limit <= members.size: - abort(403, error_msg) + abort(403, "The number of members has reached the limit of your subscription.") elif resource == "apps" and 0 < apps.limit <= apps.size: - abort(403, error_msg) + abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort(403, error_msg) + abort(403, "The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, so we need to check the source of the request from datasets source = request.args.get("source") if source == "datasets": - abort(403, error_msg) + abort(403, "The number of documents has reached the limit of your subscription.") else: return view(*args, **kwargs) elif resource == "workspace_custom" and not features.can_replace_logo: - abort(403, error_msg) + abort(403, "The workspace custom feature has reached the limit of your subscription.") elif resource == "annotation" and 0 < annotation_quota_limit.limit < annotation_quota_limit.size: - abort(403, error_msg) + abort(403, "The annotation quota has reached the limit of your subscription.") else: return view(*args, **kwargs) @@ -86,10 +84,7 @@ def cloud_edition_billing_resource_check( return interceptor -def cloud_edition_billing_knowledge_limit_check( - resource: str, - error_msg: str = "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", -): +def cloud_edition_billing_knowledge_limit_check(resource: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -97,7 +92,10 @@ def cloud_edition_billing_knowledge_limit_check( if features.billing.enabled: if resource == "add_segment": if features.billing.subscription.plan == "sandbox": - abort(403, error_msg) + abort( + 403, + "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", + ) else: return view(*args, **kwargs) diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index a596c6f287..b935b23ed6 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -83,9 +83,7 @@ def validate_app_token(view: Optional[Callable] = None, *, fetch_user_arg: Optio return decorator(view) -def cloud_edition_billing_resource_check( - resource: str, api_token_type: str, error_msg: str = "You have reached the limit of your subscription." -): +def cloud_edition_billing_resource_check(resource: str, api_token_type: str): def interceptor(view): def decorated(*args, **kwargs): api_token = validate_and_get_api_token(api_token_type) @@ -98,13 +96,13 @@ def cloud_edition_billing_resource_check( documents_upload_quota = features.documents_upload_quota if resource == "members" and 0 < members.limit <= members.size: - raise Forbidden(error_msg) + raise Forbidden("The number of members has reached the limit of your subscription.") elif resource == "apps" and 0 < apps.limit <= apps.size: - raise Forbidden(error_msg) + raise Forbidden("The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - raise Forbidden(error_msg) + raise Forbidden("The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: - raise Forbidden(error_msg) + raise Forbidden("The number of documents has reached the limit of your subscription.") else: return view(*args, **kwargs) @@ -115,11 +113,7 @@ def cloud_edition_billing_resource_check( return interceptor -def cloud_edition_billing_knowledge_limit_check( - resource: str, - api_token_type: str, - error_msg: str = "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan.", -): +def cloud_edition_billing_knowledge_limit_check(resource: str, api_token_type: str): def interceptor(view): @wraps(view) def decorated(*args, **kwargs): @@ -128,7 +122,9 @@ def cloud_edition_billing_knowledge_limit_check( if features.billing.enabled: if resource == "add_segment": if features.billing.subscription.plan == "sandbox": - raise Forbidden(error_msg) + raise Forbidden( + "To unlock this feature and elevate your Dify experience, please upgrade to a paid plan." + ) else: return view(*args, **kwargs) From 0e0a70349668a9c40b6835a1a5a016a4fdec423a Mon Sep 17 00:00:00 2001 From: takatost Date: Wed, 28 Aug 2024 23:26:11 +0800 Subject: [PATCH 021/125] chore: ignore openai error record in sentry (#7770) --- api/extensions/ext_sentry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/extensions/ext_sentry.py b/api/extensions/ext_sentry.py index 227c6635f0..3b7b0a37f4 100644 --- a/api/extensions/ext_sentry.py +++ b/api/extensions/ext_sentry.py @@ -1,3 +1,4 @@ +import openai import sentry_sdk from sentry_sdk.integrations.celery import CeleryIntegration from sentry_sdk.integrations.flask import FlaskIntegration @@ -9,7 +10,7 @@ def init_app(app): sentry_sdk.init( dsn=app.config.get("SENTRY_DSN"), integrations=[FlaskIntegration(), CeleryIntegration()], - ignore_errors=[HTTPException, ValueError], + ignore_errors=[HTTPException, ValueError, openai.APIStatusError], traces_sample_rate=app.config.get("SENTRY_TRACES_SAMPLE_RATE", 1.0), profiles_sample_rate=app.config.get("SENTRY_PROFILES_SAMPLE_RATE", 1.0), environment=app.config.get("DEPLOY_ENV"), From ec1408346eb121f72bfcba9e0f4018efcd31b849 Mon Sep 17 00:00:00 2001 From: kanoshiou <73424326+kanoshiou@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:18:49 +0800 Subject: [PATCH 022/125] docs: navigate to open issues in contributing documents (#7781) --- CONTRIBUTING.md | 2 +- CONTRIBUTING_CN.md | 2 +- CONTRIBUTING_JA.md | 2 +- CONTRIBUTING_VI.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f810584f24..8f57cd545e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ In terms of licensing, please take a minute to read our short [License and Contr ## Before you jump in -[Find](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) an existing issue, or [open](https://github.com/langgenius/dify/issues/new/choose) a new one. We categorize issues into 2 types: +[Find](https://github.com/langgenius/dify/issues?q=is:issue+is:open) an existing issue, or [open](https://github.com/langgenius/dify/issues/new/choose) a new one. We categorize issues into 2 types: ### Feature requests: diff --git a/CONTRIBUTING_CN.md b/CONTRIBUTING_CN.md index 303c2513f5..7cd2bb60eb 100644 --- a/CONTRIBUTING_CN.md +++ b/CONTRIBUTING_CN.md @@ -8,7 +8,7 @@ ## 在开始之前 -[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:closed)现有问题，或 [创建](https://github.com/langgenius/dify/issues/new/choose) 一个新问题。我们将问题分为两类： +[查找](https://github.com/langgenius/dify/issues?q=is:issue+is:open)现有问题，或 [创建](https://github.com/langgenius/dify/issues/new/choose) 一个新问题。我们将问题分为两类： ### 功能请求： diff --git a/CONTRIBUTING_JA.md b/CONTRIBUTING_JA.md index 6d5bfb205c..a68bdeddbc 100644 --- a/CONTRIBUTING_JA.md +++ b/CONTRIBUTING_JA.md @@ -10,7 +10,7 @@ Dify にコントリビュートしたいとお考えなのですね。それは ## 飛び込む前に -[既存の Issue](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) を探すか、[新しい Issue](https://github.com/langgenius/dify/issues/new/choose) を作成してください。私たちは Issue を 2 つのタイプに分類しています。 +[既存の Issue](https://github.com/langgenius/dify/issues?q=is:issue+is:open) を探すか、[新しい Issue](https://github.com/langgenius/dify/issues/new/choose) を作成してください。私たちは Issue を 2 つのタイプに分類しています。 ### 機能リクエスト diff --git a/CONTRIBUTING_VI.md b/CONTRIBUTING_VI.md index 2521256d24..80e68a046e 100644 --- a/CONTRIBUTING_VI.md +++ b/CONTRIBUTING_VI.md @@ -8,7 +8,7 @@ Về vấn đề cấp phép, xin vui lòng dành chút thời gian đọc qua [ ## Trước khi bắt đầu -[Tìm kiếm](https://github.com/langgenius/dify/issues?q=is:issue+is:closed) một vấn đề hiện có, hoặc [tạo mới](https://github.com/langgenius/dify/issues/new/choose) một vấn đề. Chúng tôi phân loại các vấn đề thành 2 loại: +[Tìm kiếm](https://github.com/langgenius/dify/issues?q=is:issue+is:open) một vấn đề hiện có, hoặc [tạo mới](https://github.com/langgenius/dify/issues/new/choose) một vấn đề. Chúng tôi phân loại các vấn đề thành 2 loại: ### Yêu cầu tính năng: From 62f48015234ade1a6c9aa7e1befdfb4d90427ba2 Mon Sep 17 00:00:00 2001 From: Huang YunKun Date: Thu, 29 Aug 2024 11:22:39 +0800 Subject: [PATCH 023/125] Update ssrf_proxy related doc link in docker-compose file (#7778) --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index c1ed2ce5f8..a319d6b45a 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -294,7 +294,7 @@ services: # ssrf_proxy server # for more information, please refer to - # https://docs.dify.ai/learn-more/faq/self-host-faq#id-18.-why-is-ssrf_proxy-needed + # https://docs.dify.ai/learn-more/faq/install-faq#id-18.-why-is-ssrf_proxy-needed ssrf_proxy: image: ubuntu/squid:latest restart: always From ad30668eb62c1c023d3426f38f01ee3ceb637b11 Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 29 Aug 2024 11:23:16 +0800 Subject: [PATCH 024/125] Sync Input component from feat/attachments branch (#7782) --- web/app/components/base/input/index.tsx | 104 ++++++++++++------ .../empty-dataset-creation-modal/index.tsx | 2 +- .../documents/detail/completed/index.tsx | 2 +- .../documents/detail/metadata/index.tsx | 2 +- .../components/datasets/documents/index.tsx | 4 +- .../key-value/key-value-edit/item.tsx | 2 +- 6 files changed, 78 insertions(+), 38 deletions(-) diff --git a/web/app/components/base/input/index.tsx b/web/app/components/base/input/index.tsx index 5ab8249446..cac5efd879 100644 --- a/web/app/components/base/input/index.tsx +++ b/web/app/components/base/input/index.tsx @@ -1,43 +1,83 @@ -'use client' -import type { SVGProps } from 'react' -import React, { useState } from 'react' +import type { CSSProperties } from 'react' +import React from 'react' import { useTranslation } from 'react-i18next' -import cn from 'classnames' +import { RiCloseCircleFill, RiErrorWarningLine, RiSearchLine } from '@remixicon/react' +import { type VariantProps, cva } from 'class-variance-authority' +import cn from '@/utils/classnames' -type InputProps = { - placeholder?: string - value?: string - defaultValue?: string - onChange?: (v: string) => void - className?: string - wrapperClassName?: string - type?: string - showPrefix?: React.ReactNode - prefixIcon?: React.ReactNode -} - -const GlassIcon = ({ className }: SVGProps) => ( - - - +export const inputVariants = cva( + '', + { + variants: { + size: { + regular: 'px-3 radius-md system-sm-regular', + large: 'px-4 radius-lg system-md-regular', + }, + }, + defaultVariants: { + size: 'regular', + }, + }, ) -const Input = ({ value, defaultValue, onChange, className = '', wrapperClassName = '', placeholder, type, showPrefix, prefixIcon }: InputProps) => { - const [localValue, setLocalValue] = useState(value ?? defaultValue) +export type InputProps = { + showLeftIcon?: boolean + showClearIcon?: boolean + onClear?: () => void + disabled?: boolean + destructive?: boolean + wrapperClassName?: string + styleCss?: CSSProperties +} & React.InputHTMLAttributes & VariantProps + +const Input = ({ + size, + disabled, + destructive, + showLeftIcon, + showClearIcon, + onClear, + wrapperClassName, + className, + styleCss, + value, + placeholder, + onChange, + ...props +}: InputProps) => { const { t } = useTranslation() return ( -
- {showPrefix && {prefixIcon ?? }} +
+ {showLeftIcon && } { - setLocalValue(e.target.value) - onChange && onChange(e.target.value) - }} + style={styleCss} + className={cn( + 'w-full py-[7px] bg-components-input-bg-normal border border-transparent text-components-input-text-filled hover:bg-components-input-bg-hover hover:border-components-input-border-hover focus:bg-components-input-bg-active focus:border-components-input-border-active focus:shadow-xs placeholder:text-components-input-text-placeholder appearance-none outline-none caret-primary-600', + inputVariants({ size }), + showLeftIcon && 'pl-[26px]', + showLeftIcon && size === 'large' && 'pl-7', + showClearIcon && value && 'pr-[26px]', + showClearIcon && value && size === 'large' && 'pr-7', + destructive && 'pr-[26px]', + destructive && size === 'large' && 'pr-7', + disabled && 'bg-components-input-bg-disabled border-transparent text-components-input-text-filled-disabled cursor-not-allowed hover:bg-components-input-bg-disabled hover:border-transparent', + destructive && 'bg-components-input-bg-destructive border-components-input-border-destructive text-components-input-text-filled hover:bg-components-input-bg-destructive hover:border-components-input-border-destructive focus:bg-components-input-bg-destructive focus:border-components-input-border-destructive', + className, + )} + placeholder={placeholder ?? (showLeftIcon ? t('common.operation.search') ?? '' : 'please input')} + value={value} + onChange={onChange} + disabled={disabled} + {...props} /> + {showClearIcon && value && !disabled && !destructive && ( +
+ +
+ )} + {destructive && ( + + )}
) } diff --git a/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx b/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx index e9247c49df..c490d50b88 100644 --- a/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx +++ b/web/app/components/datasets/create/empty-dataset-creation-modal/index.tsx @@ -58,7 +58,7 @@ const EmptyDatasetCreationModal = ({
{t('datasetCreation.stepOne.modal.tip')}
{t('datasetCreation.stepOne.modal.input')}
- + setInputValue(e.target.value)} />
diff --git a/web/app/components/datasets/documents/detail/completed/index.tsx b/web/app/components/datasets/documents/detail/completed/index.tsx index f2addac2e2..195b75b0a2 100644 --- a/web/app/components/datasets/documents/detail/completed/index.tsx +++ b/web/app/components/datasets/documents/detail/completed/index.tsx @@ -391,7 +391,7 @@ const Completed: FC = ({ defaultValue={'all'} className={s.select} wrapperClassName='h-fit w-[120px] mr-2' /> - + setSearchValue(e.target.value), 500)} />
= ({ /> : onUpdate?.(e.target.value)} value={value} defaultValue={defaultValue} placeholder={`${t('datasetDocuments.metadata.placeholder.add')}${label}`} diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index ce81e44792..32c045eccd 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -201,10 +201,10 @@ const Documents: FC = ({ datasetId }) => {
setSearchValue(e.target.value), 500)} value={searchValue} />
diff --git a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx index 93c2696b98..0f28db96c3 100644 --- a/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx +++ b/web/app/components/workflow/nodes/http/components/key-value/key-value-edit/item.tsx @@ -73,7 +73,7 @@ const KeyValueItem: FC = ({ handleChange('key')(e.target.value)} /> )}
From c441bea4d14312fc1c535e63e67cbbfb3ea0c368 Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:46:33 +0800 Subject: [PATCH 025/125] fix: datasets permission is missing (#7787) --- api/controllers/console/datasets/datasets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index d369730594..44c1390c14 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -122,6 +122,7 @@ class DatasetListApi(Resource): name=args["name"], indexing_technique=args["indexing_technique"], account=current_user, + permission=DatasetPermissionEnum.ONLY_ME, ) except services.errors.dataset.DatasetNameDuplicateError: raise DatasetNameDuplicateError() From 8e311cc45cfeabf2fc88527619fef1c8337be94a Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 29 Aug 2024 12:46:42 +0800 Subject: [PATCH 026/125] fixed permission is None (#7788) --- api/services/dataset_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index ce5fa632dc..d10df3f600 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -137,7 +137,7 @@ class DatasetService: @staticmethod def create_empty_dataset( - tenant_id: str, name: str, indexing_technique: Optional[str], account: Account, permission: Optional[str] + tenant_id: str, name: str, indexing_technique: Optional[str], account: Account, permission: Optional[str] = None ): # check if dataset name already exists if Dataset.query.filter_by(name=name, tenant_id=tenant_id).first(): From 2c51e3a3275092603ba5f9548e4aea7f5e99ebf7 Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 29 Aug 2024 15:57:43 +0800 Subject: [PATCH 027/125] fix: webapp sso setting may not the latest value when refresh (#7795) --- .../app/(appDetailLayout)/[appId]/layout.tsx | 2 +- .../app/(appDetailLayout)/[appId]/overview/cardView.tsx | 2 +- web/app/components/app/overview/appCard.tsx | 9 ++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx index 84ec157323..8723420d84 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/layout.tsx @@ -128,7 +128,7 @@ const AppDetailLayout: FC = (props) => { if (e.status === 404) router.replace('/apps') }) - }, [appId, isCurrentWorkspaceEditor]) + }, [appId, isCurrentWorkspaceEditor, systemFeatures]) useUnmount(() => { setAppDetail() diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx index 3584e13733..8f3ee510b8 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx @@ -95,7 +95,7 @@ const CardView: FC = ({ appId }) => { if (systemFeatures.enable_web_sso_switch_component) { const [sso_err] = await asyncRunSafe( - updateAppSSO({ id: appId, enabled: params.enable_sso }) as Promise, + updateAppSSO({ id: appId, enabled: Boolean(params.enable_sso) }) as Promise, ) if (sso_err) { handleCallbackResult(sso_err) diff --git a/web/app/components/app/overview/appCard.tsx b/web/app/components/app/overview/appCard.tsx index 0d0b95c0a0..f9f5c1fbff 100644 --- a/web/app/components/app/overview/appCard.tsx +++ b/web/app/components/app/overview/appCard.tsx @@ -134,8 +134,8 @@ function AppCard({ return (
@@ -176,7 +176,6 @@ function AppCard({ {isApp && } {/* button copy link/ button regenerate */} @@ -202,8 +201,8 @@ function AppCard({ onClick={() => setShowConfirmDelete(true)} >
From 962cdbbebd9e0614ea05593aed785faa063d7cf4 Mon Sep 17 00:00:00 2001 From: Yeuoly <45712896+Yeuoly@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:04:01 +0800 Subject: [PATCH 028/125] chore: add app generator overload (#7792) --- .../app/apps/advanced_chat/app_generator.py | 22 ++++++++++++++++- api/core/app/apps/agent_chat/app_generator.py | 20 +++++++++++++++- api/core/app/apps/chat/app_generator.py | 22 +++++++++++++++-- api/core/app/apps/completion/app_generator.py | 24 ++++++++++++++++--- api/core/app/apps/workflow/app_generator.py | 24 +++++++++++++++++-- 5 files changed, 103 insertions(+), 9 deletions(-) diff --git a/api/core/app/apps/advanced_chat/app_generator.py b/api/core/app/apps/advanced_chat/app_generator.py index 5a1e5973cd..e7c9ebe097 100644 --- a/api/core/app/apps/advanced_chat/app_generator.py +++ b/api/core/app/apps/advanced_chat/app_generator.py @@ -4,7 +4,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -39,6 +39,26 @@ logger = logging.getLogger(__name__) class AdvancedChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, workflow: Workflow, diff --git a/api/core/app/apps/agent_chat/app_generator.py b/api/core/app/apps/agent_chat/app_generator.py index 53780bdfb0..daf37f4a50 100644 --- a/api/core/app/apps/agent_chat/app_generator.py +++ b/api/core/app/apps/agent_chat/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -28,6 +28,24 @@ logger = logging.getLogger(__name__) class AgentChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[dict, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate(self, app_model: App, user: Union[Account, EndUser], args: Any, diff --git a/api/core/app/apps/chat/app_generator.py b/api/core/app/apps/chat/app_generator.py index 5b896e2845..ab15928b74 100644 --- a/api/core/app/apps/chat/app_generator.py +++ b/api/core/app/apps/chat/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -28,13 +28,31 @@ logger = logging.getLogger(__name__) class ChatAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: Any, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, stream: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ) -> Union[dict, Generator[str, None, None]]: """ Generate App response. diff --git a/api/core/app/apps/completion/app_generator.py b/api/core/app/apps/completion/app_generator.py index c4e1caf65a..c0b13b40fd 100644 --- a/api/core/app/apps/completion/app_generator.py +++ b/api/core/app/apps/completion/app_generator.py @@ -3,7 +3,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Any, Union +from typing import Any, Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -30,12 +30,30 @@ logger = logging.getLogger(__name__) class CompletionAppGenerator(MessageBasedAppGenerator): + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate(self, app_model: App, user: Union[Account, EndUser], args: Any, invoke_from: InvokeFrom, stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + -> Union[dict, Generator[str, None, None]]: """ Generate App response. @@ -203,7 +221,7 @@ class CompletionAppGenerator(MessageBasedAppGenerator): user: Union[Account, EndUser], invoke_from: InvokeFrom, stream: bool = True) \ - -> Union[dict, Generator[dict, None, None]]: + -> Union[dict, Generator[str, None, None]]: """ Generate App response. diff --git a/api/core/app/apps/workflow/app_generator.py b/api/core/app/apps/workflow/app_generator.py index df40aec154..26bb6c0f4f 100644 --- a/api/core/app/apps/workflow/app_generator.py +++ b/api/core/app/apps/workflow/app_generator.py @@ -4,7 +4,7 @@ import os import threading import uuid from collections.abc import Generator -from typing import Union +from typing import Literal, Union, overload from flask import Flask, current_app from pydantic import ValidationError @@ -32,6 +32,26 @@ logger = logging.getLogger(__name__) class WorkflowAppGenerator(BaseAppGenerator): + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[True] = True, + ) -> Generator[str, None, None]: ... + + @overload + def generate( + self, app_model: App, + workflow: Workflow, + user: Union[Account, EndUser], + args: dict, + invoke_from: InvokeFrom, + stream: Literal[False] = False, + ) -> dict: ... + def generate( self, app_model: App, workflow: Workflow, @@ -107,7 +127,7 @@ class WorkflowAppGenerator(BaseAppGenerator): application_generate_entity: WorkflowAppGenerateEntity, invoke_from: InvokeFrom, stream: bool = True, - ) -> Union[dict, Generator[dict, None, None]]: + ) -> Union[dict, Generator[str, None, None]]: """ Generate App response. From f0273f00e18d023fd6270307f9eb5c1030567e1b Mon Sep 17 00:00:00 2001 From: hisir Date: Thu, 29 Aug 2024 18:58:19 +0800 Subject: [PATCH 029/125] Fixed when testing the openai compatible interface model, an error is reported when no object is returned (#7808) --- .../model_providers/openai_api_compatible/llm/llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 2b729d4293..6279125f46 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -150,9 +150,9 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): except json.JSONDecodeError as e: raise CredentialsValidateFailedError('Credentials validation failed: JSON decode error') - if (completion_type is LLMMode.CHAT and json_result['object'] == ''): + if (completion_type is LLMMode.CHAT and json_result.get('object','') == ''): json_result['object'] = 'chat.completion' - elif (completion_type is LLMMode.COMPLETION and json_result['object'] == ''): + elif (completion_type is LLMMode.COMPLETION and json_result.get('object','') == ''): json_result['object'] = 'text_completion' if (completion_type is LLMMode.CHAT From bd6d4d0553696f711e2f4ab101b952a34c3ca33c Mon Sep 17 00:00:00 2001 From: YidaHu Date: Thu, 29 Aug 2024 19:03:08 +0800 Subject: [PATCH 030/125] fix: filter out installed apps without an app (#7799) --- api/controllers/console/explore/installed_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/controllers/console/explore/installed_app.py b/api/controllers/console/explore/installed_app.py index b71078760c..3f1e64a247 100644 --- a/api/controllers/console/explore/installed_app.py +++ b/api/controllers/console/explore/installed_app.py @@ -35,6 +35,7 @@ class InstalledAppsListApi(Resource): "uninstallable": current_tenant_id == installed_app.app_owner_tenant_id, } for installed_app in installed_apps + if installed_app.app is not None ] installed_apps.sort( key=lambda app: ( From c9e0f0bf20a587e640284a02070b95a1611f37a9 Mon Sep 17 00:00:00 2001 From: Benjamin Date: Fri, 30 Aug 2024 00:03:40 +0800 Subject: [PATCH 031/125] fix: correct typo in environment variable description (#7817) --- web/i18n/zh-Hans/workflow.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/i18n/zh-Hans/workflow.ts b/web/i18n/zh-Hans/workflow.ts index f57eb40bb0..56d1de6ceb 100644 --- a/web/i18n/zh-Hans/workflow.ts +++ b/web/i18n/zh-Hans/workflow.ts @@ -80,7 +80,7 @@ const translation = { }, env: { envPanelTitle: '环境变量', - envDescription: '环境变量是一种存储敏感信息的方法，如 API 密钥、数据库密码等。它们被存储在工作流程中，而不是代码中，以便在不同环墋中共享。', + envDescription: '环境变量是一种存储敏感信息的方法，如 API 密钥、数据库密码等。它们被存储在工作流程中，而不是代码中，以便在不同环境中共享。', envPanelButton: '添加环境变量', modal: { title: '添加环境变量', From dc015c380a94543da572222685bcfbf1af766eee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Fri, 30 Aug 2024 15:08:31 +0800 Subject: [PATCH 032/125] feat: add zhipu glm_4_plus and glm_4v_plus model (#7824) --- .../zhipuai/llm/glm_4_plus.yaml | 39 +++++++++++++++++++ .../zhipuai/llm/glm_4v_plus.yaml | 37 ++++++++++++++++++ .../model_providers/zhipuai/llm/llm.py | 9 +++-- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml create mode 100644 api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml new file mode 100644 index 0000000000..d9132640b2 --- /dev/null +++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4_plus.yaml @@ -0,0 +1,39 @@ +model: glm-4-plus +label: + en_US: glm-4-plus +model_type: llm +features: + - multi-tool-call + - agent-thought + - stream-tool-call +model_properties: + mode: chat +parameter_rules: + - name: temperature + use_template: temperature + default: 0.95 + min: 0.0 + max: 1.0 + help: + zh_Hans: 采样温度，控制输出的随机性，必须为正数取值范围是：(0.0,1.0]，不能等于 0,默认值为 0.95 值越大，会使输出更随机，更具创造性；值越小，输出会更加稳定或确定建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Sampling temperature, controls the randomness of the output, must be a positive number. The value range is (0.0,1.0], which cannot be equal to 0. The default value is 0.95. The larger the value, the more random and creative the output will be; the smaller the value, The output will be more stable or certain. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: top_p + use_template: top_p + default: 0.7 + help: + zh_Hans: 用温度取样的另一种方法，称为核取样取值范围是：(0.0, 1.0) 开区间，不能等于 0 或 1，默认值为 0.7 模型考虑具有 top_p 概率质量tokens的结果例如：0.1 意味着模型解码器只考虑从前 10% 的概率的候选集中取 tokens 建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Another method of temperature sampling is called kernel sampling. The value range is (0.0, 1.0) open interval, which cannot be equal to 0 or 1. The default value is 0.7. The model considers the results with top_p probability mass tokens. For example 0.1 means The model decoder only considers tokens from the candidate set with the top 10% probability. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: incremental + label: + zh_Hans: 增量返回 + en_US: Incremental + type: boolean + help: + zh_Hans: SSE接口调用时，用于控制每次返回内容方式是增量还是全量，不提供此参数时默认为增量返回，true 为增量返回，false 为全量返回。 + en_US: When the SSE interface is called, it is used to control whether the content is returned incrementally or in full. If this parameter is not provided, the default is incremental return. true means incremental return, false means full return. + required: false + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 8192 diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml new file mode 100644 index 0000000000..4a45c65f64 --- /dev/null +++ b/api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml @@ -0,0 +1,37 @@ +model: glm-4v-plus +label: + en_US: glm-4v-plus +model_type: llm +model_properties: + mode: chat +features: + - vision +parameter_rules: + - name: temperature + use_template: temperature + default: 0.95 + min: 0.0 + max: 1.0 + help: + zh_Hans: 采样温度，控制输出的随机性，必须为正数取值范围是：(0.0,1.0]，不能等于 0,默认值为 0.95 值越大，会使输出更随机，更具创造性；值越小，输出会更加稳定或确定建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Sampling temperature, controls the randomness of the output, must be a positive number. The value range is (0.0,1.0], which cannot be equal to 0. The default value is 0.95. The larger the value, the more random and creative the output will be; the smaller the value, The output will be more stable or certain. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: top_p + use_template: top_p + default: 0.7 + help: + zh_Hans: 用温度取样的另一种方法，称为核取样取值范围是：(0.0, 1.0) 开区间，不能等于 0 或 1，默认值为 0.7 模型考虑具有 top_p 概率质量tokens的结果例如：0.1 意味着模型解码器只考虑从前 10% 的概率的候选集中取 tokens 建议您根据应用场景调整 top_p 或 temperature 参数，但不要同时调整两个参数。 + en_US: Another method of temperature sampling is called kernel sampling. The value range is (0.0, 1.0) open interval, which cannot be equal to 0 or 1. The default value is 0.7. The model considers the results with top_p probability mass tokens. For example 0.1 means The model decoder only considers tokens from the candidate set with the top 10% probability. It is recommended that you adjust the top_p or temperature parameters according to the application scenario, but do not adjust both parameters at the same time. + - name: incremental + label: + zh_Hans: 增量返回 + en_US: Incremental + type: boolean + help: + zh_Hans: SSE接口调用时，用于控制每次返回内容方式是增量还是全量，不提供此参数时默认为增量返回，true 为增量返回，false 为全量返回。 + en_US: When the SSE interface is called, it is used to control whether the content is returned incrementally or in full. If this parameter is not provided, the default is incremental return. true means incremental return, false means full return. + required: false + - name: max_tokens + use_template: max_tokens + default: 1024 + min: 1 + max: 8192 diff --git a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py index 13d8f5e5c3..b2cdc7ad7a 100644 --- a/api/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/api/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -153,7 +153,8 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): :return: full response or stream response chunk generator result """ extra_model_kwargs = {} - if stop: + # request to glm-4v-plus with stop words will always response "finish_reason":"network_error" + if stop and model!= 'glm-4v-plus': extra_model_kwargs['stop'] = stop client = ZhipuAI( @@ -174,7 +175,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): if copy_prompt_message.role in [PromptMessageRole.USER, PromptMessageRole.SYSTEM, PromptMessageRole.TOOL]: if isinstance(copy_prompt_message.content, list): # check if model is 'glm-4v' - if model != 'glm-4v': + if model not in ('glm-4v', 'glm-4v-plus'): # not support list message continue # get image and @@ -207,7 +208,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): else: new_prompt_messages.append(copy_prompt_message) - if model == 'glm-4v': + if model == 'glm-4v' or model == 'glm-4v-plus': params = self._construct_glm_4v_parameter(model, new_prompt_messages, model_parameters) else: params = { @@ -304,7 +305,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): return params - def _construct_glm_4v_messages(self, prompt_message: Union[str | list[PromptMessageContent]]) -> list[dict]: + def _construct_glm_4v_messages(self, prompt_message: Union[str, list[PromptMessageContent]]) -> list[dict]: if isinstance(prompt_message, str): return [{'type': 'text', 'text': prompt_message}] From ceb2b150fff62fa515ec26e1aeffce0095d5c7be Mon Sep 17 00:00:00 2001 From: Zhi Date: Fri, 30 Aug 2024 15:53:50 +0800 Subject: [PATCH 033/125] enhance: include workspace name in create-tenant command (#7834) --- api/commands.py | 7 +++++-- api/services/account_service.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/commands.py b/api/commands.py index 41f1a6444c..3bf8bc0ecc 100644 --- a/api/commands.py +++ b/api/commands.py @@ -559,8 +559,9 @@ def add_qdrant_doc_id_index(field: str): @click.command("create-tenant", help="Create account and tenant.") @click.option("--email", prompt=True, help="The email address of the tenant account.") +@click.option("--name", prompt=True, help="The workspace name of the tenant account.") @click.option("--language", prompt=True, help="Account language, default: en-US.") -def create_tenant(email: str, language: Optional[str] = None): +def create_tenant(email: str, language: Optional[str] = None, name: Optional[str] = None): """ Create tenant account """ @@ -580,13 +581,15 @@ def create_tenant(email: str, language: Optional[str] = None): if language not in languages: language = "en-US" + name = name.strip() + # generate random password new_password = secrets.token_urlsafe(16) # register account account = RegisterService.register(email=email, name=account_name, password=new_password, language=language) - TenantService.create_owner_tenant_if_not_exist(account) + TenantService.create_owner_tenant_if_not_exist(account, name) click.echo( click.style( diff --git a/api/services/account_service.py b/api/services/account_service.py index cd501c9792..e1b70fc9ed 100644 --- a/api/services/account_service.py +++ b/api/services/account_service.py @@ -265,7 +265,7 @@ class TenantService: return tenant @staticmethod - def create_owner_tenant_if_not_exist(account: Account): + def create_owner_tenant_if_not_exist(account: Account, name: Optional[str] = None): """Create owner tenant if not exist""" available_ta = ( TenantAccountJoin.query.filter_by(account_id=account.id).order_by(TenantAccountJoin.id.asc()).first() @@ -274,7 +274,10 @@ class TenantService: if available_ta: return - tenant = TenantService.create_tenant(f"{account.name}'s Workspace") + if name: + tenant = TenantService.create_tenant(name) + else: + tenant = TenantService.create_tenant(f"{account.name}'s Workspace") TenantService.create_tenant_member(tenant, account, role="owner") account.current_tenant = tenant db.session.commit() From 62bfc4dba6e5e99ddf162b659b4cf07054c16559 Mon Sep 17 00:00:00 2001 From: Yi Xiao <54782454+YIXIAO0@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:13:54 +0800 Subject: [PATCH 034/125] fix: tooltip size sets improperly (#7836) --- .../common/retrieval-param-config/index.tsx | 2 +- .../system-model-selector/index.tsx | 13 +++++-------- .../config-credentials.tsx | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/web/app/components/datasets/common/retrieval-param-config/index.tsx b/web/app/components/datasets/common/retrieval-param-config/index.tsx index 54a0963f59..323e47f3b4 100644 --- a/web/app/components/datasets/common/retrieval-param-config/index.tsx +++ b/web/app/components/datasets/common/retrieval-param-config/index.tsx @@ -191,7 +191,7 @@ const RetrievalParamConfig: FC = ({
{option.label}
{option.tips}
} - triggerClassName='ml-0.5 w-3.5 h-4.5' + triggerClassName='ml-0.5 w-3.5 h-3.5' />
)) diff --git a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx index 6e181425e2..1574785898 100644 --- a/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx +++ b/web/app/components/header/account-setting/model-provider-page/system-model-selector/index.tsx @@ -148,7 +148,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.systemReasoningModel.tip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -168,8 +168,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.embeddingModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -189,8 +188,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.rerankModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -210,8 +208,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.speechToTextModel.tip')}
} - needsDelay={false} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
@@ -231,7 +228,7 @@ const SystemModel: FC = ({ {t('common.modelProvider.ttsModel.tip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4 shrink-0' />
diff --git a/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx b/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx index c7685d496d..d580c00102 100644 --- a/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx +++ b/web/app/components/tools/edit-custom-collection-modal/config-credentials.tsx @@ -114,7 +114,7 @@ const ConfigCredential: FC = ({ {t('tools.createTool.authMethod.keyTooltip')}
} - triggerClassName='ml-0.5' + triggerClassName='ml-0.5 w-4 h-4' /> Date: Fri, 30 Aug 2024 18:48:05 +0800 Subject: [PATCH 035/125] fix: an issue of keyword search feature in application log list (#7816) --- api/controllers/console/app/conversation.py | 27 +++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/api/controllers/console/app/conversation.py b/api/controllers/console/app/conversation.py index 753a6be20c..c3aac6690e 100644 --- a/api/controllers/console/app/conversation.py +++ b/api/controllers/console/app/conversation.py @@ -173,21 +173,18 @@ class ChatConversationApi(Resource): if args["keyword"]: keyword_filter = "%{}%".format(args["keyword"]) - query = ( - query.join( - Message, - Message.conversation_id == Conversation.id, - ) - .join(subquery, subquery.c.conversation_id == Conversation.id) - .filter( - or_( - Message.query.ilike(keyword_filter), - Message.answer.ilike(keyword_filter), - Conversation.name.ilike(keyword_filter), - Conversation.introduction.ilike(keyword_filter), - subquery.c.from_end_user_session_id.ilike(keyword_filter), - ), - ) + message_subquery = ( + db.session.query(Message.conversation_id) + .filter(or_(Message.query.ilike(keyword_filter), Message.answer.ilike(keyword_filter))) + .subquery() + ) + query = query.join(subquery, subquery.c.conversation_id == Conversation.id).filter( + or_( + Conversation.id.in_(message_subquery), + Conversation.name.ilike(keyword_filter), + Conversation.introduction.ilike(keyword_filter), + subquery.c.from_end_user_session_id.ilike(keyword_filter), + ), ) account = current_user From 0c2a62f8479d4cfcc18751b5859290af7d24370b Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Fri, 30 Aug 2024 19:09:10 +0800 Subject: [PATCH 036/125] =?UTF-8?q?fix:=20correct=20http=20timeout=20confi?= =?UTF-8?q?gs=E2=80=98=20default=20values=20and=20ignorance=20by=20HttpReq?= =?UTF-8?q?uestNode=20(#7762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/configs/feature/__init__.py | 23 ++++++++----------- .../nodes/http_request/http_request_node.py | 15 +++++------- .../unit_tests/configs/test_dify_config.py | 7 ++++++ 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/api/configs/feature/__init__.py b/api/configs/feature/__init__.py index f2efa52de3..303bce2aa5 100644 --- a/api/configs/feature/__init__.py +++ b/api/configs/feature/__init__.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Annotated, Optional from pydantic import AliasChoices, Field, HttpUrl, NegativeInt, NonNegativeInt, PositiveInt, computed_field from pydantic_settings import BaseSettings @@ -217,20 +217,17 @@ class HttpConfig(BaseSettings): def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]: return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",") - HTTP_REQUEST_MAX_CONNECT_TIMEOUT: NonNegativeInt = Field( - description="", - default=300, - ) + HTTP_REQUEST_MAX_CONNECT_TIMEOUT: Annotated[ + PositiveInt, Field(ge=10, description="connect timeout in seconds for HTTP request") + ] = 10 - HTTP_REQUEST_MAX_READ_TIMEOUT: NonNegativeInt = Field( - description="", - default=600, - ) + HTTP_REQUEST_MAX_READ_TIMEOUT: Annotated[ + PositiveInt, Field(ge=60, description="read timeout in seconds for HTTP request") + ] = 60 - HTTP_REQUEST_MAX_WRITE_TIMEOUT: NonNegativeInt = Field( - description="", - default=600, - ) + HTTP_REQUEST_MAX_WRITE_TIMEOUT: Annotated[ + PositiveInt, Field(ge=10, description="read timeout in seconds for HTTP request") + ] = 20 HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field( description="", diff --git a/api/core/workflow/nodes/http_request/http_request_node.py b/api/core/workflow/nodes/http_request/http_request_node.py index f6c8ea3c83..037a7a1848 100644 --- a/api/core/workflow/nodes/http_request/http_request_node.py +++ b/api/core/workflow/nodes/http_request/http_request_node.py @@ -19,9 +19,9 @@ from core.workflow.nodes.http_request.http_executor import HttpExecutor, HttpExe from models.workflow import WorkflowNodeExecutionStatus HTTP_REQUEST_DEFAULT_TIMEOUT = HttpRequestNodeTimeout( - connect=min(10, dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT), - read=min(60, dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT), - write=min(20, dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT), + connect=dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT, + read=dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT, + write=dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT, ) @@ -96,12 +96,9 @@ class HttpRequestNode(BaseNode): if timeout is None: return HTTP_REQUEST_DEFAULT_TIMEOUT - timeout.connect = min(timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect, - dify_config.HTTP_REQUEST_MAX_CONNECT_TIMEOUT) - timeout.read = min(timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read, - dify_config.HTTP_REQUEST_MAX_READ_TIMEOUT) - timeout.write = min(timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write, - dify_config.HTTP_REQUEST_MAX_WRITE_TIMEOUT) + timeout.connect = timeout.connect or HTTP_REQUEST_DEFAULT_TIMEOUT.connect + timeout.read = timeout.read or HTTP_REQUEST_DEFAULT_TIMEOUT.read + timeout.write = timeout.write or HTTP_REQUEST_DEFAULT_TIMEOUT.write return timeout @classmethod diff --git a/api/tests/unit_tests/configs/test_dify_config.py b/api/tests/unit_tests/configs/test_dify_config.py index fb415483dd..3f639ccacc 100644 --- a/api/tests/unit_tests/configs/test_dify_config.py +++ b/api/tests/unit_tests/configs/test_dify_config.py @@ -19,6 +19,7 @@ def example_env_file(tmp_path, monkeypatch) -> str: """ CONSOLE_API_URL=https://example.com CONSOLE_WEB_URL=https://example.com + HTTP_REQUEST_MAX_WRITE_TIMEOUT=30 """ ) ) @@ -48,6 +49,12 @@ def test_dify_config(example_env_file): assert config.API_COMPRESSION_ENABLED is False assert config.SENTRY_TRACES_SAMPLE_RATE == 1.0 + # annotated field with default value + assert config.HTTP_REQUEST_MAX_READ_TIMEOUT == 60 + + # annotated field with configured value + assert config.HTTP_REQUEST_MAX_WRITE_TIMEOUT == 30 + # NOTE: If there is a `.env` file in your Workspace, this test might not succeed as expected. # This is due to `pymilvus` loading all the variables from the `.env` file into `os.environ`. From 1f56a20b626d2e721404ecab179fac81a5dffe90 Mon Sep 17 00:00:00 2001 From: sino Date: Sat, 31 Aug 2024 10:56:32 +0800 Subject: [PATCH 037/125] feat: support auth by api key for ark provider (#7845) --- .../model_providers/volcengine_maas/client.py | 19 +++++++++-- .../volcengine_maas/volcengine_maas.yaml | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/api/core/model_runtime/model_providers/volcengine_maas/client.py b/api/core/model_runtime/model_providers/volcengine_maas/client.py index 61f3521a43..a4d89dabcb 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/client.py +++ b/api/core/model_runtime/model_providers/volcengine_maas/client.py @@ -71,11 +71,24 @@ class ArkClientV3: args = { "base_url": credentials['api_endpoint_host'], "region": credentials['volc_region'], - "ak": credentials['volc_access_key_id'], - "sk": credentials['volc_secret_access_key'], } + if credentials.get("auth_method") == "api_key": + args = { + **args, + "api_key": credentials['volc_api_key'], + } + else: + args = { + **args, + "ak": credentials['volc_access_key_id'], + "sk": credentials['volc_secret_access_key'], + } + if cls.is_compatible_with_legacy(credentials): - args["base_url"] = DEFAULT_V3_ENDPOINT + args = { + **args, + "base_url": DEFAULT_V3_ENDPOINT + } client = ArkClientV3( **args diff --git a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml index 735ba2b314..13e00da76f 100644 --- a/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml +++ b/api/core/model_runtime/model_providers/volcengine_maas/volcengine_maas.yaml @@ -30,8 +30,28 @@ model_credential_schema: en_US: Enter your Model Name zh_Hans: 输入模型名称 credential_form_schemas: + - variable: auth_method + required: true + label: + en_US: Authentication Method + zh_Hans: 鉴权方式 + type: select + default: aksk + options: + - label: + en_US: API Key + value: api_key + - label: + en_US: Access Key / Secret Access Key + value: aksk + placeholder: + en_US: Enter your Authentication Method + zh_Hans: 选择鉴权方式 - variable: volc_access_key_id required: true + show_on: + - variable: auth_method + value: aksk label: en_US: Access Key zh_Hans: Access Key @@ -41,6 +61,9 @@ model_credential_schema: zh_Hans: 输入您的 Access Key - variable: volc_secret_access_key required: true + show_on: + - variable: auth_method + value: aksk label: en_US: Secret Access Key zh_Hans: Secret Access Key @@ -48,6 +71,17 @@ model_credential_schema: placeholder: en_US: Enter your Secret Access Key zh_Hans: 输入您的 Secret Access Key + - variable: volc_api_key + required: true + show_on: + - variable: auth_method + value: api_key + label: + en_US: API Key + type: secret-input + placeholder: + en_US: Enter your API Key + zh_Hans: 输入您的 API Key - variable: volc_region required: true label: From ef82a29e2307f08c3ce6ce51720afc43e2d5bfc1 Mon Sep 17 00:00:00 2001 From: zhujinle <32472522+zhujinle@users.noreply.github.com> Date: Sun, 1 Sep 2024 14:52:27 +0800 Subject: [PATCH 038/125] fix: crash when ECharts accesses undefined objects (#7853) --- web/app/components/base/markdown.tsx | 30 ++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/web/app/components/base/markdown.tsx b/web/app/components/base/markdown.tsx index af4b13ff70..6e8ae6c9e6 100644 --- a/web/app/components/base/markdown.tsx +++ b/web/app/components/base/markdown.tsx @@ -8,7 +8,7 @@ import RemarkGfm from 'remark-gfm' import SyntaxHighlighter from 'react-syntax-highlighter' import { atelierHeathLight } from 'react-syntax-highlighter/dist/esm/styles/hljs' import type { RefObject } from 'react' -import { memo, useEffect, useMemo, useRef, useState } from 'react' +import { Component, memo, useEffect, useMemo, useRef, useState } from 'react' import type { CodeComponent } from 'react-markdown/lib/ast-to-react' import cn from '@/utils/classnames' import CopyBtn from '@/app/components/base/copy-btn' @@ -104,7 +104,7 @@ const CodeBlock: CodeComponent = memo(({ inline, className, children, ...props } const match = /language-(\w+)/.exec(className || '') const language = match?.[1] const languageShowName = getCorrectCapitalizationLanguageName(language || '') - let chartData = JSON.parse(String('{"title":{"text":"Something went wrong."}}').replace(/\n$/, '')) + let chartData = JSON.parse(String('{"title":{"text":"ECharts error - Wrong JSON format."}}').replace(/\n$/, '')) if (language === 'echarts') { try { chartData = JSON.parse(String(children).replace(/\n$/, '')) @@ -143,10 +143,10 @@ const CodeBlock: CodeComponent = memo(({ inline, className, children, ...props } ? () : ( (language === 'echarts') - ? (
-
) + ) : ( ) } + +// **Add an ECharts runtime error handler +// Avoid error #7832 (Crash when ECharts accesses undefined objects) +// This can happen when a component attempts to access an undefined object that references an unregistered map, causing the program to crash. + +export default class ErrorBoundary extends Component { + constructor(props) { + super(props) + this.state = { hasError: false } + } + + componentDidCatch(error, errorInfo) { + this.setState({ hasError: true }) + console.error(error, errorInfo) + } + + render() { + if (this.state.hasError) + return
Oops! ECharts reported a runtime error.
(see the browser console for more information)
+ return this.props.children + } +} From 024d688b77f7fafdabf66498ee77efec9631d769 Mon Sep 17 00:00:00 2001 From: Hirotaka Miyagi <31152321+MH4GF@users.noreply.github.com> Date: Sun, 1 Sep 2024 15:57:50 +0900 Subject: [PATCH 039/125] fix(RetrievalConfig): Fix score threshold assignment for zero value (#7865) --- .../nodes/knowledge-retrieval/components/retrieval-config.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx index ebe9d2151e..b335b62e33 100644 --- a/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx +++ b/web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx @@ -70,7 +70,7 @@ const RetrievalConfig: FC = ({ } onMultipleRetrievalConfigChange({ top_k: configs.top_k, - score_threshold: configs.score_threshold_enabled ? (configs.score_threshold || DATASET_DEFAULT.score_threshold) : null, + score_threshold: configs.score_threshold_enabled ? (configs.score_threshold ?? DATASET_DEFAULT.score_threshold) : null, reranking_model: payload.retrieval_mode === RETRIEVE_TYPE.oneWay ? undefined : (!configs.reranking_model?.reranking_provider_name From 1510bdbcf66dfdb14a269eeb22605a0182fdd450 Mon Sep 17 00:00:00 2001 From: Hirotaka Miyagi <31152321+MH4GF@users.noreply.github.com> Date: Sun, 1 Sep 2024 15:58:12 +0900 Subject: [PATCH 040/125] refactor: Remove typecasting by any (#7862) --- web/app/(commonLayout)/apps/Apps.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index c16512bd50..132096c6b4 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -139,7 +139,7 @@ const Apps = () => {