From 8ac3bd17688fa85e951854eb261edfb77396263a Mon Sep 17 00:00:00 2001 From: croatialu Date: Wed, 18 Jun 2025 11:30:30 +0800 Subject: [PATCH 01/37] feat: Add support for hidden attributes to form item types (#20956) --- web/types/app.ts | 3 +++ web/utils/model-config.ts | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/web/types/app.ts b/web/types/app.ts index e4227adbe9..3de5c446ec 100644 --- a/web/types/app.ts +++ b/web/types/app.ts @@ -90,6 +90,7 @@ export type TextTypeFormItem = { variable: string required: boolean max_length: number + hide: boolean } export type SelectTypeFormItem = { @@ -98,6 +99,7 @@ export type SelectTypeFormItem = { variable: string required: boolean options: string[] + hide: boolean } export type ParagraphTypeFormItem = { @@ -105,6 +107,7 @@ export type ParagraphTypeFormItem = { label: string variable: string required: boolean + hide: boolean } /** * User Input Form Item diff --git a/web/utils/model-config.ts b/web/utils/model-config.ts index 74d8848a98..330d8f9b52 100644 --- a/web/utils/model-config.ts +++ b/web/utils/model-config.ts @@ -40,6 +40,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | max_length: content.max_length, options: [], is_context_var, + hide: content.hide, }) } else if (type === 'number') { @@ -49,6 +50,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | required: content.required, type, options: [], + hide: content.hide, }) } else if (type === 'select') { @@ -59,6 +61,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | type: 'select', options: content.options, is_context_var, + hide: content.hide, }) } else if (type === 'file') { @@ -73,6 +76,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | allowed_file_upload_methods: content.allowed_file_upload_methods, number_limits: 1, }, + hide: content.hide, }) } else if (type === 'file-list') { @@ -87,6 +91,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | allowed_file_upload_methods: content.allowed_file_upload_methods, number_limits: content.max_length, }, + hide: content.hide, }) } else { @@ -100,6 +105,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] | icon: content.icon, icon_background: content.icon_background, is_context_var, + hide: content.hide, }) } }) @@ -119,6 +125,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required !== false, // default true max_length: item.max_length, default: '', + hide: item.hide, }, } as any) return @@ -130,6 +137,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ variable: item.key, required: item.required !== false, // default true default: '', + hide: item.hide, }, } as any) } @@ -141,6 +149,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required !== false, // default true options: item.options, default: '', + hide: item.hide, }, } as any) } @@ -155,6 +164,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[ required: item.required, icon: item.icon, icon_background: item.icon_background, + hide: item.hide, }, } as any) } From 45c89bd6def304333625a455d482ef02135562e6 Mon Sep 17 00:00:00 2001 From: kazuya-awano <43840049+kazuya-awano@users.noreply.github.com> Date: Wed, 18 Jun 2025 12:30:55 +0900 Subject: [PATCH 02/37] feat: add pagenation to notion extractor (#20919) --- api/core/rag/extractor/notion_extractor.py | 108 ++++++++++++--------- 1 file changed, 62 insertions(+), 46 deletions(-) diff --git a/api/core/rag/extractor/notion_extractor.py b/api/core/rag/extractor/notion_extractor.py index 4e14800d0a..eca955ddd1 100644 --- a/api/core/rag/extractor/notion_extractor.py +++ b/api/core/rag/extractor/notion_extractor.py @@ -79,55 +79,71 @@ class NotionExtractor(BaseExtractor): def _get_notion_database_data(self, database_id: str, query_dict: dict[str, Any] = {}) -> list[Document]: """Get all the pages from a Notion database.""" assert self._notion_access_token is not None, "Notion access token is required" - res = requests.post( - DATABASE_URL_TMPL.format(database_id=database_id), - headers={ - "Authorization": "Bearer " + self._notion_access_token, - "Content-Type": "application/json", - "Notion-Version": "2022-06-28", - }, - json=query_dict, - ) - - data = res.json() database_content = [] - if "results" not in data or data["results"] is None: + next_cursor = None + has_more = True + + while has_more: + current_query = query_dict.copy() + if next_cursor: + current_query["start_cursor"] = next_cursor + + res = requests.post( + DATABASE_URL_TMPL.format(database_id=database_id), + headers={ + "Authorization": "Bearer " + self._notion_access_token, + "Content-Type": "application/json", + "Notion-Version": "2022-06-28", + }, + json=current_query, + ) + + response_data = res.json() + + if "results" not in response_data or response_data["results"] is None: + break + + for result in response_data["results"]: + properties = result["properties"] + data = {} + value: Any + for property_name, property_value in properties.items(): + type = property_value["type"] + if type == "multi_select": + value = [] + multi_select_list = property_value[type] + for multi_select in multi_select_list: + value.append(multi_select["name"]) + elif type in {"rich_text", "title"}: + if len(property_value[type]) > 0: + value = property_value[type][0]["plain_text"] + else: + value = "" + elif type in {"select", "status"}: + if property_value[type]: + value = property_value[type]["name"] + else: + value = "" + else: + value = property_value[type] + data[property_name] = value + row_dict = {k: v for k, v in data.items() if v} + row_content = "" + for key, value in row_dict.items(): + if isinstance(value, dict): + value_dict = {k: v for k, v in value.items() if v} + value_content = "".join(f"{k}:{v} " for k, v in value_dict.items()) + row_content = row_content + f"{key}:{value_content}\n" + else: + row_content = row_content + f"{key}:{value}\n" + database_content.append(row_content) + + has_more = response_data.get("has_more", False) + next_cursor = response_data.get("next_cursor") + + if not database_content: return [] - for result in data["results"]: - properties = result["properties"] - data = {} - value: Any - for property_name, property_value in properties.items(): - type = property_value["type"] - if type == "multi_select": - value = [] - multi_select_list = property_value[type] - for multi_select in multi_select_list: - value.append(multi_select["name"]) - elif type in {"rich_text", "title"}: - if len(property_value[type]) > 0: - value = property_value[type][0]["plain_text"] - else: - value = "" - elif type in {"select", "status"}: - if property_value[type]: - value = property_value[type]["name"] - else: - value = "" - else: - value = property_value[type] - data[property_name] = value - row_dict = {k: v for k, v in data.items() if v} - row_content = "" - for key, value in row_dict.items(): - if isinstance(value, dict): - value_dict = {k: v for k, v in value.items() if v} - value_content = "".join(f"{k}:{v} " for k, v in value_dict.items()) - row_content = row_content + f"{key}:{value_content}\n" - else: - row_content = row_content + f"{key}:{value}\n" - database_content.append(row_content) return [Document(page_content="\n".join(database_content))] From 0784c6295d9de9ae5d2a761a102c8702553357b8 Mon Sep 17 00:00:00 2001 From: XiaoCC <284617787@qq.com> Date: Wed, 18 Jun 2025 11:31:04 +0800 Subject: [PATCH 03/37] fix Multiple \n Interface rendering exception (#20977) --- web/app/components/base/markdown/markdown-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/base/markdown/markdown-utils.ts b/web/app/components/base/markdown/markdown-utils.ts index 0aa385a1d1..dc3c7a9784 100644 --- a/web/app/components/base/markdown/markdown-utils.ts +++ b/web/app/components/base/markdown/markdown-utils.ts @@ -28,7 +28,7 @@ export const preprocessLaTeX = (content: string) => { } export const preprocessThinkTag = (content: string) => { - const thinkOpenTagRegex = /\n/g + const thinkOpenTagRegex = /(\n)+/g const thinkCloseTagRegex = /\n<\/think>/g return flow([ (str: string) => str.replace(thinkOpenTagRegex, '
\n'), From b69f952e3e617f929a92065635302a1f7d26d601 Mon Sep 17 00:00:00 2001 From: yangzheli <43645580+yangzheli@users.noreply.github.com> Date: Wed, 18 Jun 2025 11:33:10 +0800 Subject: [PATCH 04/37] fix(web): number type prompt variable required validation not effective (#20898) --- web/app/components/app/configuration/debug/index.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/app/components/app/configuration/debug/index.tsx b/web/app/components/app/configuration/debug/index.tsx index 477328dad3..38b0c890e2 100644 --- a/web/app/components/app/configuration/debug/index.tsx +++ b/web/app/components/app/configuration/debug/index.tsx @@ -156,12 +156,11 @@ const Debug: FC = ({ } let hasEmptyInput = '' const requiredVars = modelConfig.configs.prompt_variables.filter(({ key, name, required, type }) => { - if (type !== 'string' && type !== 'paragraph' && type !== 'select') + if (type !== 'string' && type !== 'paragraph' && type !== 'select' && type !== 'number') return false const res = (!key || !key.trim()) || (!name || !name.trim()) || (required || required === undefined || required === null) return res }) // compatible with old version - // debugger requiredVars.forEach(({ key, name }) => { if (hasEmptyInput) return From ce3e2e5eb85b6b80a95b002f69b67d67e9a8b672 Mon Sep 17 00:00:00 2001 From: "Byron.wang" Date: Wed, 18 Jun 2025 12:29:14 +0800 Subject: [PATCH 05/37] Set a default value for the PLUGIN_S3_USE_AWS environment variable in the dify-plugin-daemon. (#21152) --- docker/.env.example | 2 +- docker/docker-compose.middleware.yaml | 2 +- docker/docker-compose.yaml | 2 +- docker/middleware.env.example | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index d4d59936eb..195446b7ba 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1067,7 +1067,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets # Plugin oss bucket PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials -PLUGIN_S3_USE_AWS= +PLUGIN_S3_USE_AWS=false PLUGIN_S3_USE_AWS_MANAGED_IAM=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false diff --git a/docker/docker-compose.middleware.yaml b/docker/docker-compose.middleware.yaml index 4081bfd818..5308a1f978 100644 --- a/docker/docker-compose.middleware.yaml +++ b/docker/docker-compose.middleware.yaml @@ -104,7 +104,7 @@ services: PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-} + S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index dddce106b9..2b98d098b3 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -467,7 +467,7 @@ x-shared-env: &shared-api-worker-env PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages} PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets} PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-} - PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-} + PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false} PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false} PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-} PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false} diff --git a/docker/middleware.env.example b/docker/middleware.env.example index 338b057ae8..f261d88d48 100644 --- a/docker/middleware.env.example +++ b/docker/middleware.env.example @@ -133,7 +133,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets PLUGIN_STORAGE_OSS_BUCKET= # Plugin oss s3 credentials PLUGIN_S3_USE_AWS_MANAGED_IAM=false -PLUGIN_S3_USE_AWS= +PLUGIN_S3_USE_AWS=false PLUGIN_S3_ENDPOINT= PLUGIN_S3_USE_PATH_STYLE=false PLUGIN_AWS_ACCESS_KEY= From 1da80274451d2e19fefff79dd7beaf7b075d2014 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=AF=97=E6=B5=93?= Date: Wed, 18 Jun 2025 13:58:57 +0800 Subject: [PATCH 06/37] feat: Support drop DSL file into the browser to create app (#20706) Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- web/app/(commonLayout)/apps/Apps.tsx | 129 ++++++++++++------ ...sQueryState.ts => use-apps-query-state.ts} | 0 .../apps/hooks/use-dsl-drag-drop.ts | 72 ++++++++++ .../app/create-from-dsl-modal/index.tsx | 12 +- web/i18n/en-US/app.ts | 1 + web/i18n/zh-Hans/app.ts | 1 + 6 files changed, 173 insertions(+), 42 deletions(-) rename web/app/(commonLayout)/apps/hooks/{useAppsQueryState.ts => use-apps-query-state.ts} (100%) create mode 100644 web/app/(commonLayout)/apps/hooks/use-dsl-drag-drop.ts diff --git a/web/app/(commonLayout)/apps/Apps.tsx b/web/app/(commonLayout)/apps/Apps.tsx index d0cc7ff91f..2aa192fb02 100644 --- a/web/app/(commonLayout)/apps/Apps.tsx +++ b/web/app/(commonLayout)/apps/Apps.tsx @@ -9,6 +9,7 @@ import { useTranslation } from 'react-i18next' import { useDebounceFn } from 'ahooks' import { RiApps2Line, + RiDragDropLine, RiExchange2Line, RiFile4Line, RiMessage3Line, @@ -16,7 +17,8 @@ import { } from '@remixicon/react' import AppCard from './AppCard' import NewAppCard from './NewAppCard' -import useAppsQueryState from './hooks/useAppsQueryState' +import useAppsQueryState from './hooks/use-apps-query-state' +import { useDSLDragDrop } from './hooks/use-dsl-drag-drop' import type { AppListResponse } from '@/models/app' import { fetchAppList } from '@/service/apps' import { useAppContext } from '@/context/app-context' @@ -29,6 +31,7 @@ import { useStore as useTagStore } from '@/app/components/base/tag-management/st import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' import CheckboxWithLabel from '@/app/components/datasets/create/website/base/checkbox-with-label' +import CreateFromDSLModal from '@/app/components/app/create-from-dsl-modal' const getKey = ( pageIndex: number, @@ -67,6 +70,9 @@ const Apps = () => { const [tagFilterValue, setTagFilterValue] = useState(tagIDs) const [searchKeywords, setSearchKeywords] = useState(keywords) const newAppCardRef = useRef(null) + const containerRef = useRef(null) + const [showCreateFromDSLModal, setShowCreateFromDSLModal] = useState(false) + const [droppedDSLFile, setDroppedDSLFile] = useState() const setKeywords = useCallback((keywords: string) => { setQuery(prev => ({ ...prev, keywords })) }, [setQuery]) @@ -74,6 +80,17 @@ const Apps = () => { setQuery(prev => ({ ...prev, tagIDs })) }, [setQuery]) + const handleDSLFileDropped = useCallback((file: File) => { + setDroppedDSLFile(file) + setShowCreateFromDSLModal(true) + }, []) + + const { dragging } = useDSLDragDrop({ + onDSLFileDropped: handleDSLFileDropped, + containerRef, + enabled: isCurrentWorkspaceEditor, + }) + const { data, isLoading, error, setSize, mutate } = useSWRInfinite( (pageIndex: number, previousPageData: AppListResponse) => getKey(pageIndex, previousPageData, activeTab, isCreatedByMe, tagIDs, searchKeywords), fetchAppList, @@ -151,47 +168,81 @@ const Apps = () => { return ( <> -
- -
- - - handleKeywordsChange(e.target.value)} - onClear={() => handleKeywordsChange('')} +
+ {dragging && ( +
+
+ )} + +
+ +
+ + + handleKeywordsChange(e.target.value)} + onClear={() => handleKeywordsChange('')} + /> +
+ {(data && data[0].total > 0) + ?
+ {isCurrentWorkspaceEditor + && } + {data.map(({ data: apps }) => apps.map(app => ( + + )))} +
+ :
+ {isCurrentWorkspaceEditor + && } + +
} + + {isCurrentWorkspaceEditor && ( +
+ + {t('app.newApp.dropDSLToCreateApp')} +
+ )} + +
+ {showTagManagementModal && ( + + )}
- {(data && data[0].total > 0) - ?
- {isCurrentWorkspaceEditor - && } - {data.map(({ data: apps }) => apps.map(app => ( - - )))} -
- :
- {isCurrentWorkspaceEditor - && } - -
} - -
- {showTagManagementModal && ( - + + {showCreateFromDSLModal && ( + { + setShowCreateFromDSLModal(false) + setDroppedDSLFile(undefined) + }} + onSuccess={() => { + setShowCreateFromDSLModal(false) + setDroppedDSLFile(undefined) + mutate() + }} + droppedFile={droppedDSLFile} + /> )} ) diff --git a/web/app/(commonLayout)/apps/hooks/useAppsQueryState.ts b/web/app/(commonLayout)/apps/hooks/use-apps-query-state.ts similarity index 100% rename from web/app/(commonLayout)/apps/hooks/useAppsQueryState.ts rename to web/app/(commonLayout)/apps/hooks/use-apps-query-state.ts diff --git a/web/app/(commonLayout)/apps/hooks/use-dsl-drag-drop.ts b/web/app/(commonLayout)/apps/hooks/use-dsl-drag-drop.ts new file mode 100644 index 0000000000..96942ec54e --- /dev/null +++ b/web/app/(commonLayout)/apps/hooks/use-dsl-drag-drop.ts @@ -0,0 +1,72 @@ +import { useEffect, useState } from 'react' + +type DSLDragDropHookProps = { + onDSLFileDropped: (file: File) => void + containerRef: React.RefObject + enabled?: boolean +} + +export const useDSLDragDrop = ({ onDSLFileDropped, containerRef, enabled = true }: DSLDragDropHookProps) => { + const [dragging, setDragging] = useState(false) + + const handleDragEnter = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + if (e.dataTransfer?.types.includes('Files')) + setDragging(true) + } + + const handleDragOver = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + } + + const handleDragLeave = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + if (e.relatedTarget === null || !containerRef.current?.contains(e.relatedTarget as Node)) + setDragging(false) + } + + const handleDrop = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragging(false) + + if (!e.dataTransfer) + return + + const files = [...e.dataTransfer.files] + if (files.length === 0) + return + + const file = files[0] + if (file.name.toLowerCase().endsWith('.yaml') || file.name.toLowerCase().endsWith('.yml')) + onDSLFileDropped(file) + } + + useEffect(() => { + if (!enabled) + return + + const current = containerRef.current + if (current) { + current.addEventListener('dragenter', handleDragEnter) + current.addEventListener('dragover', handleDragOver) + current.addEventListener('dragleave', handleDragLeave) + current.addEventListener('drop', handleDrop) + } + return () => { + if (current) { + current.removeEventListener('dragenter', handleDragEnter) + current.removeEventListener('dragover', handleDragOver) + current.removeEventListener('dragleave', handleDragLeave) + current.removeEventListener('drop', handleDrop) + } + } + }, [containerRef, enabled]) + + return { + dragging: enabled ? dragging : false, + } +} diff --git a/web/app/components/app/create-from-dsl-modal/index.tsx b/web/app/components/app/create-from-dsl-modal/index.tsx index 9739ac47ea..8faafe05a8 100644 --- a/web/app/components/app/create-from-dsl-modal/index.tsx +++ b/web/app/components/app/create-from-dsl-modal/index.tsx @@ -1,7 +1,7 @@ 'use client' import type { MouseEventHandler } from 'react' -import { useMemo, useRef, useState } from 'react' +import { useEffect, useMemo, useRef, useState } from 'react' import { useRouter } from 'next/navigation' import { useContext } from 'use-context-selector' import { useTranslation } from 'react-i18next' @@ -35,6 +35,7 @@ type CreateFromDSLModalProps = { onClose: () => void activeTab?: string dslUrl?: string + droppedFile?: File } export enum CreateFromDSLModalTab { @@ -42,11 +43,11 @@ export enum CreateFromDSLModalTab { FROM_URL = 'from-url', } -const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDSLModalTab.FROM_FILE, dslUrl = '' }: CreateFromDSLModalProps) => { +const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDSLModalTab.FROM_FILE, dslUrl = '', droppedFile }: CreateFromDSLModalProps) => { const { push } = useRouter() const { t } = useTranslation() const { notify } = useContext(ToastContext) - const [currentFile, setDSLFile] = useState() + const [currentFile, setDSLFile] = useState(droppedFile) const [fileContent, setFileContent] = useState() const [currentTab, setCurrentTab] = useState(activeTab) const [dslUrlValue, setDslUrlValue] = useState(dslUrl) @@ -78,6 +79,11 @@ const CreateFromDSLModal = ({ show, onSuccess, onClose, activeTab = CreateFromDS const isCreatingRef = useRef(false) + useEffect(() => { + if (droppedFile) + handleFile(droppedFile) + }, [droppedFile]) + const onCreate: MouseEventHandler = async () => { if (currentTab === CreateFromDSLModalTab.FROM_FILE && !currentFile) return diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index ccfe23ead6..e75a9d5358 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -87,6 +87,7 @@ const translation = { appCreateDSLErrorPart3: 'Current application DSL version: ', appCreateDSLErrorPart4: 'System-supported DSL version: ', appCreateFailed: 'Failed to create app', + dropDSLToCreateApp: 'Drop DSL file here to create app', }, newAppFromTemplate: { byCategories: 'BY CATEGORIES', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 4ec1e65059..c5bfb39f4f 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -87,6 +87,7 @@ const translation = { appCreateDSLErrorPart3: '当前应用 DSL 版本：', appCreateDSLErrorPart4: '系统支持 DSL 版本：', appCreateFailed: '应用创建失败', + dropDSLToCreateApp: '拖放 DSL 文件到此处创建应用', Confirm: '确认', }, newAppFromTemplate: { From 37f26c412f06cf0435566456da0818cad3cf9ce6 Mon Sep 17 00:00:00 2001 From: He Wang Date: Wed, 18 Jun 2025 14:00:59 +0800 Subject: [PATCH 07/37] add healthcheck to oceanbase container (#20989) --- .github/workflows/vdb-tests.yml | 6 +-- .../vdb/oceanbase/check_oceanbase_ready.py | 49 ------------------- docker/docker-compose-template.yaml | 8 +++ docker/docker-compose.yaml | 8 +++ 4 files changed, 18 insertions(+), 53 deletions(-) delete mode 100644 api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py diff --git a/.github/workflows/vdb-tests.yml b/.github/workflows/vdb-tests.yml index 512d14b2ee..7d0a873ebd 100644 --- a/.github/workflows/vdb-tests.yml +++ b/.github/workflows/vdb-tests.yml @@ -84,10 +84,8 @@ jobs: elasticsearch oceanbase - - name: Check VDB Ready (TiDB, Oceanbase) - run: | - uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py - uv run --project api python api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py + - name: Check VDB Ready (TiDB) + run: uv run --project api python api/tests/integration_tests/vdb/tidb_vector/check_tiflash_ready.py - name: Test Vector Stores run: uv run --project api bash dev/pytest/pytest_vdb.sh diff --git a/api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py b/api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py deleted file mode 100644 index 94a51292ff..0000000000 --- a/api/tests/integration_tests/vdb/oceanbase/check_oceanbase_ready.py +++ /dev/null @@ -1,49 +0,0 @@ -import time - -import pymysql - - -def check_oceanbase_ready() -> bool: - try: - connection = pymysql.connect( - host="localhost", - port=2881, - user="root", - password="difyai123456", - ) - affected_rows = connection.query("SELECT 1") - return affected_rows == 1 - except Exception as e: - print(f"Oceanbase is not ready. Exception: {e}") - return False - finally: - if connection: - connection.close() - - -def main(): - max_attempts = 50 - retry_interval_seconds = 2 - is_oceanbase_ready = False - for attempt in range(max_attempts): - try: - is_oceanbase_ready = check_oceanbase_ready() - except Exception as e: - print(f"Oceanbase is not ready. Exception: {e}") - is_oceanbase_ready = False - - if is_oceanbase_ready: - break - else: - print(f"Attempt {attempt + 1} failed, retry in {retry_interval_seconds} seconds...") - time.sleep(retry_interval_seconds) - - if is_oceanbase_ready: - print("Oceanbase is ready.") - else: - print(f"Oceanbase is not ready after {max_attempts} attempting checks.") - exit(1) - - -if __name__ == "__main__": - main() diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 55e1b55599..158ede88cf 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -451,6 +451,14 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini + ports: + - "${OCEANBASE_VECTOR_PORT:-2881}:2881" + healthcheck: + test: [ 'CMD-SHELL', 'obclient -h127.0.0.1 -P2881 -uroot@test -p$${OB_TENANT_PASSWORD} -e "SELECT 1;"' ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s # Oracle vector database oracle: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 2b98d098b3..99aa87bcc0 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -958,6 +958,14 @@ services: OB_CLUSTER_NAME: ${OCEANBASE_CLUSTER_NAME:-difyai} OB_SERVER_IP: 127.0.0.1 MODE: mini + ports: + - "${OCEANBASE_VECTOR_PORT:-2881}:2881" + healthcheck: + test: [ 'CMD-SHELL', 'obclient -h127.0.0.1 -P2881 -uroot@test -p$${OB_TENANT_PASSWORD} -e "SELECT 1;"' ] + interval: 10s + retries: 30 + start_period: 30s + timeout: 10s # Oracle vector database oracle: From ea44b895e20589949ad6ddb9025c2fc6f9bb98e0 Mon Sep 17 00:00:00 2001 From: Bowen Liang Date: Wed, 18 Jun 2025 14:02:45 +0800 Subject: [PATCH 08/37] chore: cancel enforcing uppercase of the text of plugin navigation button on the header bar (#20890) --- web/app/components/header/plugins-nav/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/header/plugins-nav/index.tsx b/web/app/components/header/plugins-nav/index.tsx index b22e717c94..b1f903bafb 100644 --- a/web/app/components/header/plugins-nav/index.tsx +++ b/web/app/components/header/plugins-nav/index.tsx @@ -31,7 +31,7 @@ const PluginsNav = ({ )}>
Date: Wed, 18 Jun 2025 14:05:28 +0800 Subject: [PATCH 09/37] add service api ratelimit check (#20878) --- api/controllers/service_api/dataset/dataset.py | 9 ++++++++- api/controllers/service_api/dataset/document.py | 11 ++++++++++- api/controllers/service_api/dataset/hit_testing.py | 3 ++- api/controllers/service_api/dataset/metadata.py | 7 ++++++- api/controllers/service_api/dataset/segment.py | 7 +++++++ 5 files changed, 33 insertions(+), 4 deletions(-) diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index 27e8dd3fa6..1467dfb6b3 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -5,7 +5,11 @@ from werkzeug.exceptions import Forbidden, NotFound import services.dataset_service from controllers.service_api import api from controllers.service_api.dataset.error import DatasetInUseError, DatasetNameDuplicateError -from controllers.service_api.wraps import DatasetApiResource, validate_dataset_token +from controllers.service_api.wraps import ( + DatasetApiResource, + cloud_edition_billing_rate_limit_check, + validate_dataset_token, +) from core.model_runtime.entities.model_entities import ModelType from core.plugin.entities.plugin import ModelProviderID from core.provider_manager import ProviderManager @@ -70,6 +74,7 @@ class DatasetListApi(DatasetApiResource): response = {"data": data, "has_more": len(datasets) == limit, "limit": limit, "total": total, "page": page} return response, 200 + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id): """Resource for creating datasets.""" parser = reqparse.RequestParser() @@ -193,6 +198,7 @@ class DatasetApi(DatasetApiResource): return data, 200 + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def patch(self, _, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -293,6 +299,7 @@ class DatasetApi(DatasetApiResource): return result_data, 200 + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def delete(self, _, dataset_id): """ Deletes a dataset given its ID. diff --git a/api/controllers/service_api/dataset/document.py b/api/controllers/service_api/dataset/document.py index ab7ab4dcf0..e4779f3bdf 100644 --- a/api/controllers/service_api/dataset/document.py +++ b/api/controllers/service_api/dataset/document.py @@ -19,7 +19,11 @@ from controllers.service_api.dataset.error import ( ArchivedDocumentImmutableError, DocumentIndexingError, ) -from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_resource_check +from controllers.service_api.wraps import ( + DatasetApiResource, + cloud_edition_billing_rate_limit_check, + cloud_edition_billing_resource_check, +) from core.errors.error import ProviderTokenNotInitError from extensions.ext_database import db from fields.document_fields import document_fields, document_status_fields @@ -35,6 +39,7 @@ class DocumentAddByTextApi(DatasetApiResource): @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_resource_check("documents", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id): """Create document by text.""" parser = reqparse.RequestParser() @@ -99,6 +104,7 @@ class DocumentUpdateByTextApi(DatasetApiResource): """Resource for update documents.""" @cloud_edition_billing_resource_check("vector_space", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, document_id): """Update document by text.""" parser = reqparse.RequestParser() @@ -158,6 +164,7 @@ class DocumentAddByFileApi(DatasetApiResource): @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_resource_check("documents", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id): """Create document by upload file.""" args = {} @@ -232,6 +239,7 @@ class DocumentUpdateByFileApi(DatasetApiResource): """Resource for update documents.""" @cloud_edition_billing_resource_check("vector_space", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, document_id): """Update document by upload file.""" args = {} @@ -302,6 +310,7 @@ class DocumentUpdateByFileApi(DatasetApiResource): class DocumentDeleteApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def delete(self, tenant_id, dataset_id, document_id): """Delete document.""" document_id = str(document_id) diff --git a/api/controllers/service_api/dataset/hit_testing.py b/api/controllers/service_api/dataset/hit_testing.py index 465f71bf03..52e9bca5da 100644 --- a/api/controllers/service_api/dataset/hit_testing.py +++ b/api/controllers/service_api/dataset/hit_testing.py @@ -1,9 +1,10 @@ from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase from controllers.service_api import api -from controllers.service_api.wraps import DatasetApiResource +from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check class HitTestingApi(DatasetApiResource, DatasetsHitTestingBase): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/service_api/dataset/metadata.py b/api/controllers/service_api/dataset/metadata.py index 35582feea0..1968696ee5 100644 --- a/api/controllers/service_api/dataset/metadata.py +++ b/api/controllers/service_api/dataset/metadata.py @@ -3,7 +3,7 @@ from flask_restful import marshal, reqparse from werkzeug.exceptions import NotFound from controllers.service_api import api -from controllers.service_api.wraps import DatasetApiResource +from controllers.service_api.wraps import DatasetApiResource, cloud_edition_billing_rate_limit_check from fields.dataset_fields import dataset_metadata_fields from services.dataset_service import DatasetService from services.entities.knowledge_entities.knowledge_entities import ( @@ -14,6 +14,7 @@ from services.metadata_service import MetadataService class DatasetMetadataCreateServiceApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id): parser = reqparse.RequestParser() parser.add_argument("type", type=str, required=True, nullable=True, location="json") @@ -39,6 +40,7 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource): class DatasetMetadataServiceApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def patch(self, tenant_id, dataset_id, metadata_id): parser = reqparse.RequestParser() parser.add_argument("name", type=str, required=True, nullable=True, location="json") @@ -54,6 +56,7 @@ class DatasetMetadataServiceApi(DatasetApiResource): metadata = MetadataService.update_metadata_name(dataset_id_str, metadata_id_str, args.get("name")) return marshal(metadata, dataset_metadata_fields), 200 + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def delete(self, tenant_id, dataset_id, metadata_id): dataset_id_str = str(dataset_id) metadata_id_str = str(metadata_id) @@ -73,6 +76,7 @@ class DatasetMetadataBuiltInFieldServiceApi(DatasetApiResource): class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, action): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -88,6 +92,7 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource): class DocumentMetadataEditServiceApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) diff --git a/api/controllers/service_api/dataset/segment.py b/api/controllers/service_api/dataset/segment.py index 337752275a..403b7f0a0c 100644 --- a/api/controllers/service_api/dataset/segment.py +++ b/api/controllers/service_api/dataset/segment.py @@ -8,6 +8,7 @@ from controllers.service_api.app.error import ProviderNotInitializeError from controllers.service_api.wraps import ( DatasetApiResource, cloud_edition_billing_knowledge_limit_check, + cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, ) from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError @@ -35,6 +36,7 @@ class SegmentApi(DatasetApiResource): @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, document_id): """Create single segment.""" # check dataset @@ -139,6 +141,7 @@ class SegmentApi(DatasetApiResource): class DatasetSegmentApi(DatasetApiResource): + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def delete(self, tenant_id, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -162,6 +165,7 @@ class DatasetSegmentApi(DatasetApiResource): return 204 @cloud_edition_billing_resource_check("vector_space", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -236,6 +240,7 @@ class ChildChunkApi(DatasetApiResource): @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def post(self, tenant_id, dataset_id, document_id, segment_id): """Create child chunk.""" # check dataset @@ -332,6 +337,7 @@ class DatasetChildChunkApi(DatasetApiResource): """Resource for updating child chunks.""" @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def delete(self, tenant_id, dataset_id, document_id, segment_id, child_chunk_id): """Delete child chunk.""" # check dataset @@ -370,6 +376,7 @@ class DatasetChildChunkApi(DatasetApiResource): @cloud_edition_billing_resource_check("vector_space", "dataset") @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset") + @cloud_edition_billing_rate_limit_check("knowledge", "dataset") def patch(self, tenant_id, dataset_id, document_id, segment_id, child_chunk_id): """Update child chunk.""" # check dataset From 15800c6108960c1865a766876db465496b85bd28 Mon Sep 17 00:00:00 2001 From: croatialu Date: Wed, 18 Jun 2025 14:27:02 +0800 Subject: [PATCH 10/37] feat: Embedded chat window supports userVariables configuration. (#20983) --- .../components/app/overview/embedded/index.tsx | 4 ++++ .../base/chat/embedded-chatbot/chat-wrapper.tsx | 10 ++++++++++ .../base/chat/embedded-chatbot/context.tsx | 5 +++++ .../base/chat/embedded-chatbot/hooks.tsx | 6 +++++- .../base/chat/embedded-chatbot/index.tsx | 2 ++ web/app/components/base/chat/utils.ts | 17 ++++++++++++++++- web/public/embed.js | 14 +++++++++++++- web/public/embed.min.js | 6 +++--- 8 files changed, 58 insertions(+), 6 deletions(-) diff --git a/web/app/components/app/overview/embedded/index.tsx b/web/app/components/app/overview/embedded/index.tsx index 691b727b8e..b48eac5458 100644 --- a/web/app/components/app/overview/embedded/index.tsx +++ b/web/app/components/app/overview/embedded/index.tsx @@ -50,6 +50,10 @@ const OPTION_MAP = { // user_id: 'YOU CAN DEFINE USER ID HERE', // conversation_id: 'YOU CAN DEFINE CONVERSATION ID HERE, IT MUST BE A VALID UUID', }, + userVariables: { + // avatar_url: 'YOU CAN DEFINE USER AVATAR URL HERE', + // name: 'YOU CAN DEFINE USER NAME HERE', + }, }