opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/node-execution-retry

This commit is contained in:
Novice Lee 2024-12-18 15:33:24 +08:00
parent 357769c72e 56434db4f5
commit b411087bb7
30 changed files with 403 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/controllers/console/files.py
View File

@ -1,6 +1,7 @@
from flask import request
from flask_login import current_user
from flask_restful import Resource, marshal_with
from werkzeug.exceptions import Forbidden
import services
from configs import dify_config
@ -58,6 +59,9 @@ class FileApi(Resource):
if not file.filename:
raise FilenameNotExistsError
if source == "datasets" and not current_user.is_dataset_editor:
raise Forbidden()
if source not in ("datasets", None):
source = None

4
api/core/model_runtime/model_providers/hunyuan/hunyuan.yaml
View File

@ -3,8 +3,8 @@ label:
zh_Hans: 腾讯混元
en_US: Hunyuan
description:
en_US: Models provided by Tencent Hunyuan, such as hunyuan-standard, hunyuan-standard-256k, hunyuan-pro and hunyuan-lite.
zh_Hans: 腾讯混元提供的模型，例如 hunyuan-standard、 hunyuan-standard-256k, hunyuan-pro 和 hunyuan-lite。
en_US: Models provided by Tencent Hunyuan, such as hunyuan-standard, hunyuan-standard-256k, hunyuan-pro, hunyuan-role, hunyuan-large, hunyuan-large-role, hunyuan-turbo-latest, hunyuan-large-longcontext, hunyuan-turbo, hunyuan-vision, hunyuan-turbo-vision, hunyuan-functioncall and hunyuan-lite.
zh_Hans: 腾讯混元提供的模型，例如 hunyuan-standard、 hunyuan-standard-256k, hunyuan-pro, hunyuan-role, hunyuan-large, hunyuan-large-role, hunyuan-turbo-latest, hunyuan-large-longcontext, hunyuan-turbo, hunyuan-vision, hunyuan-turbo-vision, hunyuan-functioncall 和 hunyuan-lite。
icon_small:
en_US: icon_s_en.png
icon_large:

7
api/core/model_runtime/model_providers/hunyuan/llm/_position.yaml
View File

@ -4,3 +4,10 @@
- hunyuan-pro
- hunyuan-turbo
- hunyuan-vision
- hunyuan-role
- hunyuan-large
- hunyuan-large-role
- hunyuan-large-longcontext
- hunyuan-turbo-latest
- hunyuan-turbo-vision
- hunyuan-functioncall

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-functioncall.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-functioncall
label:
zh_Hans: hunyuan-functioncall
en_US: hunyuan-functioncall
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 32000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 32000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.004'
output: '0.008'
unit: '0.001'
currency: RMB

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-longcontext.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-large-longcontext
label:
zh_Hans: hunyuan-large-longcontext
en_US: hunyuan-large-longcontext
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 134000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 134000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.006'
output: '0.018'
unit: '0.001'
currency: RMB

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large-role.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-large-role
label:
zh_Hans: hunyuan-large-role
en_US: hunyuan-large-role
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 32000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 32000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.004'
output: '0.008'
unit: '0.001'
currency: RMB

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-large.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-large
label:
zh_Hans: hunyuan-large
en_US: hunyuan-large
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 32000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 32000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.004'
output: '0.012'
unit: '0.001'
currency: RMB

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-role.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-role
label:
zh_Hans: hunyuan-role
en_US: hunyuan-role
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 32000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 32000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.004'
output: '0.008'
unit: '0.001'
currency: RMB

38
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-latest.yaml Normal file
View File

@ -0,0 +1,38 @@
model: hunyuan-turbo-latest
label:
zh_Hans: hunyuan-turbo-latest
en_US: hunyuan-turbo-latest
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
model_properties:
mode: chat
context_size: 32000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 32000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.015'
output: '0.05'
unit: '0.001'
currency: RMB

39
api/core/model_runtime/model_providers/hunyuan/llm/hunyuan-turbo-vision.yaml Normal file
View File

@ -0,0 +1,39 @@
model: hunyuan-turbo-vision
label:
zh_Hans: hunyuan-turbo-vision
en_US: hunyuan-turbo-vision
model_type: llm
features:
- agent-thought
- tool-call
- multi-tool-call
- stream-tool-call
- vision
model_properties:
mode: chat
context_size: 8000
parameter_rules:
- name: temperature
use_template: temperature
- name: top_p
use_template: top_p
- name: max_tokens
use_template: max_tokens
default: 1024
min: 1
max: 8000
- name: enable_enhance
label:
zh_Hans: 功能增强
en_US: Enable Enhancement
type: boolean
help:
zh_Hans: 功能增强（如搜索）开关，关闭时将直接由主模型生成回复内容，可以降低响应时延（对于流式输出时的首字时延尤为明显）。但在少数场景里，回复效果可能会下降。
en_US: Allow the model to perform external search to enhance the generation results.
required: false
default: true
pricing:
input: '0.08'
output: '0.08'
unit: '0.001'
currency: RMB

12
api/core/model_runtime/model_providers/openai/llm/_position.yaml
View File

@ -1,4 +1,7 @@
- gpt-4o-audio-preview
- o1
- o1-2024-12-17
- o1-mini
- o1-mini-2024-09-12
- gpt-4
- gpt-4o
- gpt-4o-2024-05-13
@ -7,10 +10,6 @@
- chatgpt-4o-latest
- gpt-4o-mini
- gpt-4o-mini-2024-07-18
- o1-preview
- o1-preview-2024-09-12
- o1-mini
- o1-mini-2024-09-12
- gpt-4-turbo
- gpt-4-turbo-2024-04-09
- gpt-4-turbo-preview
@ -25,4 +24,7 @@
- gpt-3.5-turbo-1106
- gpt-3.5-turbo-0613
- gpt-3.5-turbo-instruct
- gpt-4o-audio-preview
- o1-preview
- o1-preview-2024-09-12
- text-davinci-003

2
api/core/model_runtime/model_providers/openai/llm/chatgpt-4o-latest.yaml
View File

@ -22,7 +22,7 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 16384
- name: response_format

4
api/core/model_runtime/model_providers/openai/llm/gpt-4o-2024-05-13.yaml
View File

@ -22,9 +22,9 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 4096
max: 16384
- name: response_format
label:
zh_Hans: 回复格式

2
api/core/model_runtime/model_providers/openai/llm/gpt-4o-2024-08-06.yaml
View File

@ -22,7 +22,7 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 16384
- name: response_format

2
api/core/model_runtime/model_providers/openai/llm/gpt-4o-2024-11-20.yaml
View File

@ -22,7 +22,7 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 16384
- name: response_format

4
api/core/model_runtime/model_providers/openai/llm/gpt-4o-audio-preview.yaml
View File

@ -22,9 +22,9 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 4096
max: 16384
- name: response_format
label:
zh_Hans: 回复格式

2
api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini-2024-07-18.yaml
View File

@ -22,7 +22,7 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 16384
- name: response_format

2
api/core/model_runtime/model_providers/openai/llm/gpt-4o-mini.yaml
View File

@ -22,7 +22,7 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 16384
- name: response_format

8
api/core/model_runtime/model_providers/openai/llm/gpt-4o.yaml
View File

@ -22,9 +22,9 @@ parameter_rules:
use_template: frequency_penalty
- name: max_tokens
use_template: max_tokens
default: 512
default: 16384
min: 1
max: 4096
max: 16384
- name: response_format
label:
zh_Hans: 回复格式
@ -38,7 +38,7 @@ parameter_rules:
- text
- json_object
pricing:
input: '5.00'
output: '15.00'
input: '2.50'
output: '10.00'
unit: '0.000001'
currency: USD

35
api/core/model_runtime/model_providers/openai/llm/o1-2024-12-17.yaml Normal file
View File

@ -0,0 +1,35 @@
model: o1-2024-12-17
label:
en_US: o1-2024-12-17
model_type: llm
features:
- multi-tool-call
- agent-thought
- stream-tool-call
- vision
model_properties:
mode: chat
context_size: 200000
parameter_rules:
- name: max_tokens
use_template: max_tokens
default: 50000
min: 1
max: 50000
- name: response_format
label:
zh_Hans: 回复格式
en_US: response_format
type: string
help:
zh_Hans: 指定模型必须输出的格式
en_US: specifying the format that the model must output
required: false
options:
- text
- json_object
pricing:
input: '15.00'
output: '60.00'
unit: '0.000001'
currency: USD

36
api/core/model_runtime/model_providers/openai/llm/o1.yaml Normal file
View File

@ -0,0 +1,36 @@
model: o1
label:
zh_Hans: o1
en_US: o1
model_type: llm
features:
- multi-tool-call
- agent-thought
- stream-tool-call
- vision
model_properties:
mode: chat
context_size: 200000
parameter_rules:
- name: max_tokens
use_template: max_tokens
default: 50000
min: 1
max: 50000
- name: response_format
label:
zh_Hans: 回复格式
en_US: response_format
type: string
help:
zh_Hans: 指定模型必须输出的格式
en_US: specifying the format that the model must output
required: false
options:
- text
- json_object
pricing:
input: '15.00'
output: '60.00'
unit: '0.000001'
currency: USD

17
api/core/rag/datasource/vdb/lindorm/lindorm_vector.py
View File

@ -49,10 +49,10 @@ class LindormVectorStoreConfig(BaseModel):
class LindormVectorStore(BaseVector):
def __init__(self, collection_name: str, config: LindormVectorStoreConfig, **kwargs):
def __init__(self, collection_name: str, config: LindormVectorStoreConfig, using_ugc: bool, **kwargs):
self._routing = None
self._routing_field = None
if config.using_ugc:
if using_ugc:
routing_value: str = kwargs.get("routing_value")
if routing_value is None:
raise ValueError("UGC index should init vector with valid 'routing_value' parameter value")
@ -64,7 +64,7 @@ class LindormVectorStore(BaseVector):
super().__init__(collection_name.lower())
self._client_config = config
self._client = OpenSearch(**config.to_opensearch_params())
self._using_ugc = config.using_ugc
self._using_ugc = using_ugc
self.kwargs = kwargs
def get_type(self) -> str:
@ -467,12 +467,16 @@ class LindormVectorStoreFactory(AbstractVectorFactory):
using_ugc = dify_config.USING_UGC_INDEX
routing_value = None
if dataset.index_struct:
if using_ugc:
# if an existed record's index_struct_dict doesn't contain using_ugc field,
# it actually stores in the normal index format
stored_in_ugc = dataset.index_struct_dict.get("using_ugc", False)
using_ugc = stored_in_ugc
if stored_in_ugc:
dimension = dataset.index_struct_dict["dimension"]
index_type = dataset.index_struct_dict["index_type"]
distance_type = dataset.index_struct_dict["distance_type"]
index_name = f"{UGC_INDEX_PREFIX}_{dimension}_{index_type}_{distance_type}"
routing_value = dataset.index_struct_dict["vector_store"]["class_prefix"]
index_name = f"{UGC_INDEX_PREFIX}_{dimension}_{index_type}_{distance_type}"
else:
index_name = dataset.index_struct_dict["vector_store"]["class_prefix"]
else:
@ -487,6 +491,7 @@ class LindormVectorStoreFactory(AbstractVectorFactory):
"index_type": index_type,
"dimension": dimension,
"distance_type": distance_type,
"using_ugc": using_ugc,
}
dataset.index_struct = json.dumps(index_struct_dict)
if using_ugc:
@ -494,4 +499,4 @@ class LindormVectorStoreFactory(AbstractVectorFactory):
routing_value = class_prefix
else:
index_name = class_prefix
return LindormVectorStore(index_name, lindorm_config, routing_value=routing_value)
return LindormVectorStore(index_name, lindorm_config, routing_value=routing_value, using_ugc=using_ugc)

1
api/extensions/storage/opendal_storage.py
View File

@ -34,7 +34,6 @@ class OpenDALStorage(BaseStorage):
root = kwargs.get("root", "storage")
Path(root).mkdir(parents=True, exist_ok=True)
# self.op = opendal.Operator(scheme=scheme, **kwargs)
self.op = opendal.Operator(scheme=scheme, **kwargs)
logger.debug(f"opendal operator created with scheme {scheme}")
retry_layer = opendal.layers.RetryLayer(max_times=3, factor=2.0, jitter=True)

2
api/services/feature_service.py
View File

@ -63,6 +63,7 @@ class SystemFeatureModel(BaseModel):
enable_social_oauth_login: bool = False
is_allow_register: bool = False
is_allow_create_workspace: bool = False
is_email_setup: bool = False
license: LicenseModel = LicenseModel()
@ -98,6 +99,7 @@ class FeatureService:
system_features.enable_social_oauth_login = dify_config.ENABLE_SOCIAL_OAUTH_LOGIN
system_features.is_allow_register = dify_config.ALLOW_REGISTER
system_features.is_allow_create_workspace = dify_config.ALLOW_CREATE_WORKSPACE
system_features.is_email_setup = dify_config.MAIL_TYPE is not None and dify_config.MAIL_TYPE != ""
@classmethod
def _fulfill_params_from_env(cls, features: FeatureModel):

9
docker/.env.example
View File

@ -284,8 +284,11 @@ CONSOLE_CORS_ALLOW_ORIGINS=*
# The type of storage to use for storing user files.
STORAGE_TYPE=opendal
# Apache OpenDAL Configuration, refer to https://github.com/apache/opendal
# The scheme for the OpenDAL storage.
# Apache OpenDAL Configuration
# The configuration for OpenDAL consists of the following format: OPENDAL_<SCHEME_NAME>_<CONFIG_NAME>.
# You can find all the service configurations (CONFIG_NAME) in the repository at: https://github.com/apache/opendal/tree/main/core/src/services.
# Dify will scan configurations starting with OPENDAL_<SCHEME_NAME> and automatically apply them.
# The scheme name for the OpenDAL storage.
OPENDAL_SCHEME=fs
# Configurations for OpenDAL Local File System.
OPENDAL_FS_ROOT=storage
@ -919,4 +922,4 @@ MAX_SUBMIT_COUNT=100
# Proxy
HTTP_PROXY=
HTTPS_PROXY=
HTTPS_PROXY=

2
web/app/components/develop/template/template_advanced_chat.zh.mdx
View File

@ -3,7 +3,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
# 工作流编排对话型应用 API
对话应用支持会话持久化，可将之前的聊天记录作为上下进行回答，可适用于聊天/客服 AI 等。
对话应用支持会话持久化，可将之前的聊天记录作为上下文进行回答，可适用于聊天/客服 AI 等。
<div>
### 基础 URL

2
web/app/components/develop/template/template_chat.zh.mdx
View File

@ -3,7 +3,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx'
# 对话型应用 API
对话应用支持会话持久化，可将之前的聊天记录作为上下进行回答，可适用于聊天/客服 AI 等。
对话应用支持会话持久化，可将之前的聊天记录作为上下文进行回答，可适用于聊天/客服 AI 等。
<div>
### 基础 URL

10
web/app/signin/components/mail-and-password-auth.tsx
View File

@ -12,12 +12,13 @@ import I18NContext from '@/context/i18n'
type MailAndPasswordAuthProps = {
isInvite: boolean
isEmailSetup: boolean
allowRegistration: boolean
}
const passwordRegex = /^(?=.*[a-zA-Z])(?=.*\d).{8,}$/
export default function MailAndPasswordAuth({ isInvite, allowRegistration }: MailAndPasswordAuthProps) {
export default function MailAndPasswordAuth({ isInvite, isEmailSetup, allowRegistration }: MailAndPasswordAuthProps) {
const { t } = useTranslation()
const { locale } = useContext(I18NContext)
const router = useRouter()
@ -124,7 +125,12 @@ export default function MailAndPasswordAuth({ isInvite, allowRegistration }: Mai
<div className='mb-3'>
<label htmlFor="password" className="my-2 flex items-center justify-between">
<span className='system-md-semibold text-text-secondary'>{t('login.password')}</span>
<Link href={`/reset-password?${searchParams.toString()}`} className='system-xs-regular text-components-button-secondary-accent-text'>
<Link
href={`/reset-password?${searchParams.toString()}`}
className={`system-xs-regular ${isEmailSetup ? 'text-components-button-secondary-accent-text' : 'text-components-button-secondary-accent-text-disabled pointer-events-none'}`}
tabIndex={isEmailSetup ? 0 : -1}
aria-disabled={!isEmailSetup}
>
{t('login.forget')}
</Link>
</label>

2
web/app/signin/normalForm.tsx
View File

@ -163,7 +163,7 @@ const NormalForm = () => {
</div>}
</>}
{systemFeatures.enable_email_password_login && authType === 'password' && <>
<MailAndPasswordAuth isInvite={isInviteLink} allowRegistration={systemFeatures.is_allow_register} />
<MailAndPasswordAuth isInvite={isInviteLink} isEmailSetup={systemFeatures.is_email_setup} allowRegistration={systemFeatures.is_allow_register} />
{systemFeatures.enable_email_code_login && <div className='cursor-pointer py-1 text-center' onClick={() => { updateAuthType('code') }}>
<span className='system-xs-medium text-components-button-secondary-accent-text'>{t('login.useVerificationCode')}</span>
</div>}

2
web/types/feature.ts
View File

@ -29,6 +29,7 @@ export type SystemFeatures = {
enable_social_oauth_login: boolean
is_allow_create_workspace: boolean
is_allow_register: boolean
is_email_setup: boolean
license: License
}
@ -43,6 +44,7 @@ export const defaultSystemFeatures: SystemFeatures = {
enable_social_oauth_login: false,
is_allow_create_workspace: false,
is_allow_register: false,
is_email_setup: false,
license: {
status: LicenseStatus.NONE,
expired_at: '',