Merge branch 'main' into feat/rag-pipeline

api/controllers/web/passport.py

@@ -163,7 +163,7 @@ def exchange_token_for_existing_web_user(app_code: str, enterprise_user_decoded:
         )
         db.session.add(end_user)
         db.session.commit()
-    exp_dt = datetime.now(UTC) + timedelta(hours=dify_config.ACCESS_TOKEN_EXPIRE_MINUTES * 24)
+    exp_dt = datetime.now(UTC) + timedelta(minutes=dify_config.ACCESS_TOKEN_EXPIRE_MINUTES)
     exp = int(exp_dt.timestamp())
     payload = {
         "iss": site.id,

api/core/rag/datasource/vdb/weaviate/weaviate_vector.py

@@ -41,6 +41,12 @@ class WeaviateVector(BaseVector):
 
         weaviate.connect.connection.has_grpc = False
 
+        # Fix to minimize the performance impact of the deprecation check in weaviate-client 3.24.0,
+        # by changing the connection timeout to pypi.org from 1 second to 0.001 seconds.
+        # TODO: This can be removed once weaviate-client is updated to 3.26.7 or higher,
+        #       which does not contain the deprecation check.
+        weaviate.connect.connection.PYPI_TIMEOUT = 0.001
+
         try:
             client = weaviate.Client(
                 url=config.endpoint, auth_client_secret=auth_config, timeout_config=(5, 60), startup_period=None

api/core/rag/extractor/notion_extractor.py

@@ -79,55 +79,71 @@ class NotionExtractor(BaseExtractor):
     def _get_notion_database_data(self, database_id: str, query_dict: dict[str, Any] = {}) -> list[Document]:
         """Get all the pages from a Notion database."""
         assert self._notion_access_token is not None, "Notion access token is required"
-        res = requests.post(
-            DATABASE_URL_TMPL.format(database_id=database_id),
-            headers={
-                "Authorization": "Bearer " + self._notion_access_token,
-                "Content-Type": "application/json",
-                "Notion-Version": "2022-06-28",
-            },
-            json=query_dict,
-        )
-
-        data = res.json()
 
         database_content = []
-        if "results" not in data or data["results"] is None:
+        next_cursor = None
+        has_more = True
+
+        while has_more:
+            current_query = query_dict.copy()
+            if next_cursor:
+                current_query["start_cursor"] = next_cursor
+
+            res = requests.post(
+                DATABASE_URL_TMPL.format(database_id=database_id),
+                headers={
+                    "Authorization": "Bearer " + self._notion_access_token,
+                    "Content-Type": "application/json",
+                    "Notion-Version": "2022-06-28",
+                },
+                json=current_query,
+            )
+
+            response_data = res.json()
+
+            if "results" not in response_data or response_data["results"] is None:
+                break
+
+            for result in response_data["results"]:
+                properties = result["properties"]
+                data = {}
+                value: Any
+                for property_name, property_value in properties.items():
+                    type = property_value["type"]
+                    if type == "multi_select":
+                        value = []
+                        multi_select_list = property_value[type]
+                        for multi_select in multi_select_list:
+                            value.append(multi_select["name"])
+                    elif type in {"rich_text", "title"}:
+                        if len(property_value[type]) > 0:
+                            value = property_value[type][0]["plain_text"]
+                        else:
+                            value = ""
+                    elif type in {"select", "status"}:
+                        if property_value[type]:
+                            value = property_value[type]["name"]
+                        else:
+                            value = ""
+                    else:
+                        value = property_value[type]
+                    data[property_name] = value
+                row_dict = {k: v for k, v in data.items() if v}
+                row_content = ""
+                for key, value in row_dict.items():
+                    if isinstance(value, dict):
+                        value_dict = {k: v for k, v in value.items() if v}
+                        value_content = "".join(f"{k}:{v} " for k, v in value_dict.items())
+                        row_content = row_content + f"{key}:{value_content}\n"
+                    else:
+                        row_content = row_content + f"{key}:{value}\n"
+                database_content.append(row_content)
+
+            has_more = response_data.get("has_more", False)
+            next_cursor = response_data.get("next_cursor")
+
+        if not database_content:
             return []
-        for result in data["results"]:
-            properties = result["properties"]
-            data = {}
-            value: Any
-            for property_name, property_value in properties.items():
-                type = property_value["type"]
-                if type == "multi_select":
-                    value = []
-                    multi_select_list = property_value[type]
-                    for multi_select in multi_select_list:
-                        value.append(multi_select["name"])
-                elif type in {"rich_text", "title"}:
-                    if len(property_value[type]) > 0:
-                        value = property_value[type][0]["plain_text"]
-                    else:
-                        value = ""
-                elif type in {"select", "status"}:
-                    if property_value[type]:
-                        value = property_value[type]["name"]
-                    else:
-                        value = ""
-                else:
-                    value = property_value[type]
-                data[property_name] = value
-            row_dict = {k: v for k, v in data.items() if v}
-            row_content = ""
-            for key, value in row_dict.items():
-                if isinstance(value, dict):
-                    value_dict = {k: v for k, v in value.items() if v}
-                    value_content = "".join(f"{k}:{v} " for k, v in value_dict.items())
-                    row_content = row_content + f"{key}:{value_content}\n"
-                else:
-                    row_content = row_content + f"{key}:{value}\n"
-            database_content.append(row_content)
 
         return [Document(page_content="\n".join(database_content))]
 

api/core/workflow/nodes/agent/agent_node.py

@@ -214,7 +214,7 @@ class AgentNode(ToolNode):
                         )
                         if tool_runtime.entity.description:
                             tool_runtime.entity.description.llm = (
-                                extra.get("descrption", "") or tool_runtime.entity.description.llm
+                                extra.get("description", "") or tool_runtime.entity.description.llm
                             )
                         for tool_runtime_params in tool_runtime.entity.parameters:
                             tool_runtime_params.form = (

docker/.env.example

@@ -1067,7 +1067,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets
 # Plugin oss bucket
 PLUGIN_STORAGE_OSS_BUCKET=
 # Plugin oss s3 credentials
-PLUGIN_S3_USE_AWS=
+PLUGIN_S3_USE_AWS=false
 PLUGIN_S3_USE_AWS_MANAGED_IAM=false
 PLUGIN_S3_ENDPOINT=
 PLUGIN_S3_USE_PATH_STYLE=false

docker/docker-compose.middleware.yaml

@@ -104,7 +104,7 @@ services:
       PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
       PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
       PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
-      S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-}
+      S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false}
       S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
       S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
       S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}

docker/docker-compose.yaml

@@ -467,7 +467,7 @@ x-shared-env: &shared-api-worker-env
   PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
   PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
   PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
-  PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-}
+  PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false}
   PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
   PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
   PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}

docker/middleware.env.example

@@ -133,7 +133,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets
 PLUGIN_STORAGE_OSS_BUCKET=
 # Plugin oss s3 credentials
 PLUGIN_S3_USE_AWS_MANAGED_IAM=false
-PLUGIN_S3_USE_AWS=
+PLUGIN_S3_USE_AWS=false
 PLUGIN_S3_ENDPOINT=
 PLUGIN_S3_USE_PATH_STYLE=false
 PLUGIN_AWS_ACCESS_KEY=

web/app/components/app/configuration/debug/index.tsx

@@ -156,12 +156,11 @@ const Debug: FC<IDebug> = ({
     }
     let hasEmptyInput = ''
     const requiredVars = modelConfig.configs.prompt_variables.filter(({ key, name, required, type }) => {
-      if (type !== 'string' && type !== 'paragraph' && type !== 'select')
+      if (type !== 'string' && type !== 'paragraph' && type !== 'select' && type !== 'number')
         return false
       const res = (!key || !key.trim()) || (!name || !name.trim()) || (required || required === undefined || required === null)
       return res
     }) // compatible with old version
-    // debugger
     requiredVars.forEach(({ key, name }) => {
       if (hasEmptyInput)
         return

web/app/components/base/markdown/markdown-utils.ts

@@ -28,7 +28,7 @@ export const preprocessLaTeX = (content: string) => {
 }
 
 export const preprocessThinkTag = (content: string) => {
-  const thinkOpenTagRegex = /<think>\n/g
+  const thinkOpenTagRegex = /(<think>\n)+/g
   const thinkCloseTagRegex = /\n<\/think>/g
   return flow([
     (str: string) => str.replace(thinkOpenTagRegex, '<details data-think=true>\n'),

web/app/components/tools/edit-custom-collection-modal/test-api.tsx

@@ -31,10 +31,13 @@ const TestApi: FC<Props> = ({
   const language = getLanguage(locale)
   const [credentialsModalShow, setCredentialsModalShow] = useState(false)
   const [tempCredential, setTempCredential] = React.useState<Credential>(customCollection.credentials)
+  const [testing, setTesting] = useState(false)
   const [result, setResult] = useState<string>('')
   const { operation_id: toolName, parameters } = tool
   const [parametersValue, setParametersValue] = useState<Record<string, string>>({})
   const handleTest = async () => {
+    if (testing) return
+    setTesting(true)
     // clone test schema
     const credentials = JSON.parse(JSON.stringify(tempCredential)) as Credential
     if (credentials.auth_type === AuthType.none) {
@@ -52,6 +55,7 @@ const TestApi: FC<Props> = ({
     }
     const res = await testAPIAvailable(data) as any
     setResult(res.error || res.result)
+    setTesting(false)
   }
 
   return (
@@ -107,7 +111,7 @@ const TestApi: FC<Props> = ({
               </div>
 
             </div>
-            <Button variant='primary' className=' mt-4 h-10 w-full' onClick={handleTest}>{t('tools.test.title')}</Button>
+            <Button variant='primary' className=' mt-4 h-10 w-full' loading={testing} disabled={testing} onClick={handleTest}>{t('tools.test.title')}</Button>
             <div className='mt-6'>
               <div className='flex items-center space-x-3'>
                 <div className='system-xs-semibold text-text-tertiary'>{t('tools.test.testResult')}</div>

web/app/components/workflow/nodes/_base/components/agent-strategy.tsx

@@ -118,6 +118,7 @@ export const AgentStrategy = memo((props: AgentStrategyProps) => {
             title={<>
               {renderI18nObject(def.label)} {def.required && <span className='text-red-500'>*</span>}
             </>}
+            key={def.variable}
             tooltip={def.tooltip && renderI18nObject(def.tooltip)}
             inline
           >

web/app/components/workflow/nodes/_base/components/editor/code-editor/index.tsx

@@ -140,6 +140,7 @@ const CodeEditor: FC<Props> = ({
         language={languageMap[language] || 'javascript'}
         theme={isMounted ? theme : 'default-theme'} // sometimes not load the default theme
         value={outPutValue}
+        loading={<span className='text-text-primary'>Loading...</span>}
         onChange={handleEditorChange}
         // https://microsoft.github.io/monaco-editor/typedoc/interfaces/editor.IEditorOptions.html
         options={{

web/app/components/workflow/nodes/agent/node.tsx

@@ -54,9 +54,9 @@ const AgentNode: FC<NodeProps<AgentNodeType>> = (props) => {
         const field = param.name
         const value = inputs.agent_parameters?.[field]?.value
         if (value) {
-          (value as unknown as any[]).forEach((item) => {
+          (value as unknown as any[]).forEach((item, idx) => {
             tools.push({
-              id: `${param.name}-${i}`,
+              id: `${param.name}-${idx}`,
               providerName: item.provider_name,
             })
           })

web/i18n/es-ES/app.ts

@@ -207,6 +207,7 @@ const translation = {
     modelNotSupportedTip: 'El modelo actual no admite esta función y se degrada automáticamente a inyección de comandos.',
     structuredTip: 'Las Salidas Estructuradas son una función que garantiza que el modelo siempre generará respuestas que se ajusten a su esquema JSON proporcionado.',
     modelNotSupported: 'Modelo no soportado',
+    structured: 'sistemático',
   },
   accessItemsDescription: {
     anyone: 'Cualquiera puede acceder a la aplicación web.',

web/i18n/es-ES/dataset.ts

@@ -202,6 +202,7 @@ const translation = {
       builtInDescription: 'Los metadatos integrados se extraen y generan automáticamente. Deben estar habilitados antes de su uso y no se pueden editar.',
       name: 'Nombre',
       description: 'Puedes gestionar todos los metadatos en este conocimiento aquí. Las modificaciones se sincronizarán en todos los documentos.',
+      disabled: 'desactivar',
     },
     documentMetadata: {
       technicalParameters: 'Parámetros técnicos',

web/i18n/ko-KR/workflow.ts

@@ -188,6 +188,7 @@ const translation = {
     nodeResize: '노드 크기 조정됨',
     nodeDragStop: '노드가 이동했습니다.',
     edgeDelete: '노드가 연결이 끊어졌습니다.',
+    nodeTitleChange: '노드 제목이 변경됨',
   },
   errorMsg: {
     fieldRequired: '{{field}}가 필요합니다',

web/i18n/ro-RO/workflow.ts

@@ -188,6 +188,7 @@ const translation = {
     nodeDescriptionChange: 'Descrierea nodului a fost modificată',
     edgeDelete: 'Nod deconectat',
     nodeAdd: 'Nod adăugat',
+    nodeDragStop: 'Nod mutat',
   },
   errorMsg: {
     fieldRequired: '{{field}} este obligatoriu',

web/i18n/sl-SI/workflow.ts

@@ -112,6 +112,7 @@ const translation = {
     pointerMode: 'Način s kazalcem',
     autoSaved: 'Samodejno shranjeno',
     configure: 'Konfiguriraj',
+    inRunMode: 'V načinu izvajanja',
   },
   env: {
     modal: {
@@ -185,6 +186,7 @@ const translation = {
     clearHistory: 'Počisti zgodovino',
     hintText: 'Vaša dejanja urejanja so sledena v zgodovini sprememb, ki se hrani na vaši napravi za čas trajanja te seje. Ta zgodovina bo izbrisana, ko zapustite urejevalnik.',
     placeholder: 'Še niste spremenili ničesar.',
+    stepForward_one: '{{count}} korak naprej',
   },
   errorMsg: {
     fields: {
@@ -836,6 +838,7 @@ const translation = {
           upload_file_id: 'Naložite ID datoteke',
           title: 'datoteke, ki jih je ustvaril agent',
           url: 'URL slike',
+          transfer_method: 'Metoda prenosa. Vrednost je remote_url ali local_file.',
         },
         json: 'agent generiran json',
         text: 'vsebina, ki jo je ustvaril agent',

web/types/app.ts

@@ -90,6 +90,7 @@ export type TextTypeFormItem = {
   variable: string
   required: boolean
   max_length: number
+  hide: boolean
 }
 
 export type SelectTypeFormItem = {
@@ -98,6 +99,7 @@ export type SelectTypeFormItem = {
   variable: string
   required: boolean
   options: string[]
+  hide: boolean
 }
 
 export type ParagraphTypeFormItem = {
@@ -105,6 +107,7 @@ export type ParagraphTypeFormItem = {
   label: string
   variable: string
   required: boolean
+  hide: boolean
 }
 /**
  * User Input Form Item

web/utils/model-config.ts

@@ -40,6 +40,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         max_length: content.max_length,
         options: [],
         is_context_var,
+        hide: content.hide,
       })
     }
     else if (type === 'number') {
@@ -49,6 +50,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         required: content.required,
         type,
         options: [],
+        hide: content.hide,
       })
     }
     else if (type === 'select') {
@@ -59,6 +61,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         type: 'select',
         options: content.options,
         is_context_var,
+        hide: content.hide,
       })
     }
     else if (type === 'file') {
@@ -73,6 +76,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
           allowed_file_upload_methods: content.allowed_file_upload_methods,
           number_limits: 1,
         },
+        hide: content.hide,
       })
     }
     else if (type === 'file-list') {
@@ -87,6 +91,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
           allowed_file_upload_methods: content.allowed_file_upload_methods,
           number_limits: content.max_length,
         },
+        hide: content.hide,
       })
     }
     else {
@@ -100,6 +105,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         icon: content.icon,
         icon_background: content.icon_background,
         is_context_var,
+        hide: content.hide,
       })
     }
   })
@@ -119,6 +125,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required !== false, // default true
           max_length: item.max_length,
           default: '',
+          hide: item.hide,
         },
       } as any)
       return
@@ -130,6 +137,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           variable: item.key,
           required: item.required !== false, // default true
           default: '',
+          hide: item.hide,
         },
       } as any)
     }
@@ -141,6 +149,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required !== false, // default true
           options: item.options,
           default: '',
+          hide: item.hide,
         },
       } as any)
     }
@@ -155,6 +164,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required,
           icon: item.icon,
           icon_background: item.icon_background,
+          hide: item.hide,
         },
       } as any)
     }