Merge branch 'feat/rag-pipeline' into deploy/rag-dev

api/core/rag/extractor/notion_extractor.py

@@ -79,55 +79,71 @@ class NotionExtractor(BaseExtractor):
     def _get_notion_database_data(self, database_id: str, query_dict: dict[str, Any] = {}) -> list[Document]:
         """Get all the pages from a Notion database."""
         assert self._notion_access_token is not None, "Notion access token is required"
-        res = requests.post(
-            DATABASE_URL_TMPL.format(database_id=database_id),
-            headers={
-                "Authorization": "Bearer " + self._notion_access_token,
-                "Content-Type": "application/json",
-                "Notion-Version": "2022-06-28",
-            },
-            json=query_dict,
-        )
-
-        data = res.json()
 
         database_content = []
-        if "results" not in data or data["results"] is None:
+        next_cursor = None
+        has_more = True
+
+        while has_more:
+            current_query = query_dict.copy()
+            if next_cursor:
+                current_query["start_cursor"] = next_cursor
+
+            res = requests.post(
+                DATABASE_URL_TMPL.format(database_id=database_id),
+                headers={
+                    "Authorization": "Bearer " + self._notion_access_token,
+                    "Content-Type": "application/json",
+                    "Notion-Version": "2022-06-28",
+                },
+                json=current_query,
+            )
+
+            response_data = res.json()
+
+            if "results" not in response_data or response_data["results"] is None:
+                break
+
+            for result in response_data["results"]:
+                properties = result["properties"]
+                data = {}
+                value: Any
+                for property_name, property_value in properties.items():
+                    type = property_value["type"]
+                    if type == "multi_select":
+                        value = []
+                        multi_select_list = property_value[type]
+                        for multi_select in multi_select_list:
+                            value.append(multi_select["name"])
+                    elif type in {"rich_text", "title"}:
+                        if len(property_value[type]) > 0:
+                            value = property_value[type][0]["plain_text"]
+                        else:
+                            value = ""
+                    elif type in {"select", "status"}:
+                        if property_value[type]:
+                            value = property_value[type]["name"]
+                        else:
+                            value = ""
+                    else:
+                        value = property_value[type]
+                    data[property_name] = value
+                row_dict = {k: v for k, v in data.items() if v}
+                row_content = ""
+                for key, value in row_dict.items():
+                    if isinstance(value, dict):
+                        value_dict = {k: v for k, v in value.items() if v}
+                        value_content = "".join(f"{k}:{v} " for k, v in value_dict.items())
+                        row_content = row_content + f"{key}:{value_content}\n"
+                    else:
+                        row_content = row_content + f"{key}:{value}\n"
+                database_content.append(row_content)
+
+            has_more = response_data.get("has_more", False)
+            next_cursor = response_data.get("next_cursor")
+
+        if not database_content:
             return []
-        for result in data["results"]:
-            properties = result["properties"]
-            data = {}
-            value: Any
-            for property_name, property_value in properties.items():
-                type = property_value["type"]
-                if type == "multi_select":
-                    value = []
-                    multi_select_list = property_value[type]
-                    for multi_select in multi_select_list:
-                        value.append(multi_select["name"])
-                elif type in {"rich_text", "title"}:
-                    if len(property_value[type]) > 0:
-                        value = property_value[type][0]["plain_text"]
-                    else:
-                        value = ""
-                elif type in {"select", "status"}:
-                    if property_value[type]:
-                        value = property_value[type]["name"]
-                    else:
-                        value = ""
-                else:
-                    value = property_value[type]
-                data[property_name] = value
-            row_dict = {k: v for k, v in data.items() if v}
-            row_content = ""
-            for key, value in row_dict.items():
-                if isinstance(value, dict):
-                    value_dict = {k: v for k, v in value.items() if v}
-                    value_content = "".join(f"{k}:{v} " for k, v in value_dict.items())
-                    row_content = row_content + f"{key}:{value_content}\n"
-                else:
-                    row_content = row_content + f"{key}:{value}\n"
-            database_content.append(row_content)
 
         return [Document(page_content="\n".join(database_content))]
 

docker/.env.example

@@ -1067,7 +1067,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets
 # Plugin oss bucket
 PLUGIN_STORAGE_OSS_BUCKET=
 # Plugin oss s3 credentials
-PLUGIN_S3_USE_AWS=
+PLUGIN_S3_USE_AWS=false
 PLUGIN_S3_USE_AWS_MANAGED_IAM=false
 PLUGIN_S3_ENDPOINT=
 PLUGIN_S3_USE_PATH_STYLE=false

docker/docker-compose.middleware.yaml

@@ -103,7 +103,7 @@ services:
       PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
       PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
       PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
-      S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-}
+      S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false}
       S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
       S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
       S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}

docker/docker-compose.yaml

@@ -467,7 +467,7 @@ x-shared-env: &shared-api-worker-env
   PLUGIN_PACKAGE_CACHE_PATH: ${PLUGIN_PACKAGE_CACHE_PATH:-plugin_packages}
   PLUGIN_MEDIA_CACHE_PATH: ${PLUGIN_MEDIA_CACHE_PATH:-assets}
   PLUGIN_STORAGE_OSS_BUCKET: ${PLUGIN_STORAGE_OSS_BUCKET:-}
-  PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-}
+  PLUGIN_S3_USE_AWS: ${PLUGIN_S3_USE_AWS:-false}
   PLUGIN_S3_USE_AWS_MANAGED_IAM: ${PLUGIN_S3_USE_AWS_MANAGED_IAM:-false}
   PLUGIN_S3_ENDPOINT: ${PLUGIN_S3_ENDPOINT:-}
   PLUGIN_S3_USE_PATH_STYLE: ${PLUGIN_S3_USE_PATH_STYLE:-false}

docker/middleware.env.example

@@ -133,7 +133,7 @@ PLUGIN_MEDIA_CACHE_PATH=assets
 PLUGIN_STORAGE_OSS_BUCKET=
 # Plugin oss s3 credentials
 PLUGIN_S3_USE_AWS_MANAGED_IAM=false
-PLUGIN_S3_USE_AWS=
+PLUGIN_S3_USE_AWS=false
 PLUGIN_S3_ENDPOINT=
 PLUGIN_S3_USE_PATH_STYLE=false
 PLUGIN_AWS_ACCESS_KEY=

web/app/components/app/configuration/debug/index.tsx

@@ -156,12 +156,11 @@ const Debug: FC<IDebug> = ({
     }
     let hasEmptyInput = ''
     const requiredVars = modelConfig.configs.prompt_variables.filter(({ key, name, required, type }) => {
-      if (type !== 'string' && type !== 'paragraph' && type !== 'select')
+      if (type !== 'string' && type !== 'paragraph' && type !== 'select' && type !== 'number')
         return false
       const res = (!key || !key.trim()) || (!name || !name.trim()) || (required || required === undefined || required === null)
       return res
     }) // compatible with old version
-    // debugger
     requiredVars.forEach(({ key, name }) => {
       if (hasEmptyInput)
         return

web/app/components/base/markdown/markdown-utils.ts

@@ -28,7 +28,7 @@ export const preprocessLaTeX = (content: string) => {
 }
 
 export const preprocessThinkTag = (content: string) => {
-  const thinkOpenTagRegex = /<think>\n/g
+  const thinkOpenTagRegex = /(<think>\n)+/g
   const thinkCloseTagRegex = /\n<\/think>/g
   return flow([
     (str: string) => str.replace(thinkOpenTagRegex, '<details data-think=true>\n'),

web/app/components/rag-pipeline/components/input-field/dialog-wrapper.tsx

@@ -1,47 +1,58 @@
-import { Fragment, useCallback } from 'react'
+import { useCallback } from 'react'
 import type { ReactNode } from 'react'
 import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react'
 import cn from '@/utils/classnames'
 
 type DialogWrapperProps = {
+  dialogClassName?: string
   className?: string
   panelWrapperClassName?: string
+  outerWrapperClassName?: string
   children: ReactNode
   show: boolean
   onClose?: () => void
 }
 
 const DialogWrapper = ({
+  dialogClassName,
   className,
   panelWrapperClassName,
+  outerWrapperClassName,
   children,
   show,
   onClose,
 }: DialogWrapperProps) => {
   const close = useCallback(() => onClose?.(), [onClose])
   return (
-    <Transition appear show={show} as={Fragment}>
-      <Dialog as='div' className='relative z-[1000000]' onClose={close}>
+    <Transition appear show={show}>
+      <Dialog
+        as='div'
+        className={cn('relative z-40', dialogClassName)}
+        onClose={close}
+      >
         <TransitionChild>
-          <div className={cn(
-            'fixed inset-0 bg-black/25',
-            'data-[closed]:opacity-0',
-            'data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-            'data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-          )} />
+          <div
+            className={cn(
+              'fixed inset-0 bg-black/25',
+              'data-[closed]:opacity-0',
+              'data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
+              'data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
+            )}
+          />
         </TransitionChild>
 
-        <div className='fixed inset-0'>
+        <div className={cn('fixed inset-0', outerWrapperClassName)}>
           <div className={cn('flex min-h-full flex-col items-end justify-center pb-1 pt-[116px]', panelWrapperClassName)}>
             <TransitionChild>
-              <DialogPanel className={cn(
-                'relative flex w-[420px] grow flex-col overflow-hidden border-components-panel-border bg-components-panel-bg-alt p-0 shadow-xl shadow-shadow-shadow-5 transition-all',
-                'rounded-l-2xl border-y-[0.5px] border-l-[0.5px]',
-                'data-[closed]:scale-95  data-[closed]:opacity-0',
-                'data-[enter]:scale-100 data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-                'data-[leave]:scale-95 data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-                className,
-              )}>
+              <DialogPanel
+                className={cn(
+                  'relative flex w-[420px] flex-col overflow-hidden border-components-panel-border bg-components-panel-bg-alt p-0 shadow-xl shadow-shadow-shadow-5 transition-all',
+                  'data-[closed]:scale-95  data-[closed]:opacity-0',
+                  'data-[enter]:scale-100 data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
+                  'data-[leave]:scale-95 data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
+                  className,
+                )}
+              >
                 {children}
               </DialogPanel>
             </TransitionChild>

web/app/components/rag-pipeline/components/input-field/editor/dialog-wrapper.tsx

@@ -1,54 +0,0 @@
-import { Fragment, useCallback } from 'react'
-import type { ReactNode } from 'react'
-import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react'
-import cn from '@/utils/classnames'
-
-type DialogWrapperProps = {
-  className?: string
-  panelWrapperClassName?: string
-  children: ReactNode
-  show: boolean
-  onClose?: () => void
-}
-
-const DialogWrapper = ({
-  className,
-  panelWrapperClassName,
-  children,
-  show,
-  onClose,
-}: DialogWrapperProps) => {
-  const close = useCallback(() => onClose?.(), [onClose])
-  return (
-    <Transition appear show={show} as={Fragment}>
-      <Dialog as='div' className='relative z-[1000001]' onClose={close}>
-        <TransitionChild>
-          <div className={cn(
-            'fixed inset-0 bg-black/25',
-            'data-[closed]:opacity-0',
-            'data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-            'data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-          )} />
-        </TransitionChild>
-
-        <div className='fixed inset-0 overflow-y-auto'>
-          <div className={cn('flex min-h-full flex-col items-end justify-start pb-1 pt-[116px]', panelWrapperClassName)}>
-            <TransitionChild>
-              <DialogPanel className={cn(
-                'relative flex w-[400px] flex-col overflow-hidden rounded-2xl border-[0.5px] border-components-panel-border bg-components-panel-bg p-0 shadow-xl shadow-shadow-shadow-9 transition-all',
-                'data-[closed]:scale-95 data-[closed]:opacity-0',
-                'data-[enter]:scale-100 data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-                'data-[leave]:scale-95 data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-                className,
-              )}>
-                {children}
-              </DialogPanel>
-            </TransitionChild>
-          </div>
-        </div>
-      </Dialog>
-    </Transition >
-  )
-}
-
-export default DialogWrapper

web/app/components/rag-pipeline/components/input-field/editor/index.tsx

@@ -1,5 +1,5 @@
 import { RiCloseLine } from '@remixicon/react'
-import DialogWrapper from './dialog-wrapper'
+import DialogWrapper from '../dialog-wrapper'
 import InputFieldForm from './form'
 import { convertFormDataToINputField, convertToInputFieldFormData } from './utils'
 import { useCallback } from 'react'
@@ -33,7 +33,9 @@ const InputFieldEditor = ({
     <DialogWrapper
       show={show}
       onClose={onClose}
-      panelWrapperClassName='pr-[424px]'
+      outerWrapperClassName='overflow-y-auto'
+      panelWrapperClassName='pr-[424px] justify-start'
+      className='w-[400px] rounded-2xl border-[0.5px] bg-components-panel-bg shadow-shadow-shadow-9'
     >
       <div className='system-xl-semibold flex items-center pb-1 pl-4 pr-11 pt-3.5 text-text-primary'>
         {initialData ? t('datasetPipeline.inputFieldPanel.editInputField') : t('datasetPipeline.inputFieldPanel.addInputField')}

web/app/components/rag-pipeline/components/input-field/index.tsx

@@ -94,6 +94,7 @@ const InputFieldDialog = ({
       <DialogWrapper
         show={!!showInputFieldDialog}
         onClose={closePanel}
+        className='grow rounded-l-2xl border-y-[0.5px] border-l-[0.5px]'
       >
         <div className='flex grow flex-col'>
           <div className='flex items-center p-4 pb-0'>

web/app/components/rag-pipeline/components/input-field/preview/dialog-wrapper.tsx

@@ -1,54 +0,0 @@
-import { Fragment, useCallback } from 'react'
-import type { ReactNode } from 'react'
-import { Dialog, DialogPanel, Transition, TransitionChild } from '@headlessui/react'
-import cn from '@/utils/classnames'
-
-type DialogWrapperProps = {
-  className?: string
-  panelWrapperClassName?: string
-  children: ReactNode
-  show: boolean
-  onClose?: () => void
-}
-
-const DialogWrapper = ({
-  className,
-  panelWrapperClassName,
-  children,
-  show,
-  onClose,
-}: DialogWrapperProps) => {
-  const close = useCallback(() => onClose?.(), [onClose])
-  return (
-    <Transition appear show={show} as={Fragment}>
-      <Dialog as='div' className='relative z-[1000001]' onClose={close}>
-        <TransitionChild>
-          <div className={cn(
-            'fixed inset-0 bg-black/25',
-            'data-[closed]:opacity-0',
-            'data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-            'data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-          )} />
-        </TransitionChild>
-
-        <div className='fixed inset-0'>
-          <div className={cn('flex min-h-full flex-col items-end justify-start pb-1 pt-[116px]', panelWrapperClassName)}>
-            <TransitionChild>
-              <DialogPanel className={cn(
-                'relative flex w-[480px] grow flex-col overflow-hidden rounded-2xl border-y-[0.5px] border-l-[0.5px] border-components-panel-border bg-components-panel-bg p-0 shadow-xl shadow-shadow-shadow-5 transition-all',
-                'data-[closed]:scale-95 data-[closed]:opacity-0',
-                'data-[enter]:scale-100 data-[enter]:opacity-100 data-[enter]:duration-300 data-[enter]:ease-out',
-                'data-[leave]:scale-95 data-[leave]:opacity-0 data-[leave]:duration-200 data-[leave]:ease-in',
-                className,
-              )}>
-                {children}
-              </DialogPanel>
-            </TransitionChild>
-          </div>
-        </div>
-      </Dialog>
-    </Transition >
-  )
-}
-
-export default DialogWrapper

web/app/components/rag-pipeline/components/input-field/preview/index.tsx

@@ -1,6 +1,6 @@
 import { useState } from 'react'
 import { RiCloseLine } from '@remixicon/react'
-import DialogWrapper from './dialog-wrapper'
+import DialogWrapper from '../dialog-wrapper'
 import { useTranslation } from 'react-i18next'
 import Badge from '@/app/components/base/badge'
 import DataSource from './data-source'
@@ -24,7 +24,8 @@ const PreviewPanel = ({
     <DialogWrapper
       show={show}
       onClose={onClose}
-      panelWrapperClassName='pr-[424px]'
+      panelWrapperClassName='pr-[424px] justify-start'
+      className='w-[480px] grow rounded-2xl border-[0.5px] bg-components-panel-bg'
     >
       <div className='flex items-center gap-x-2 px-4 pt-1'>
         <div className='grow py-1'>

web/types/app.ts

@@ -90,6 +90,7 @@ export type TextTypeFormItem = {
   variable: string
   required: boolean
   max_length: number
+  hide: boolean
 }
 
 export type SelectTypeFormItem = {
@@ -98,6 +99,7 @@ export type SelectTypeFormItem = {
   variable: string
   required: boolean
   options: string[]
+  hide: boolean
 }
 
 export type ParagraphTypeFormItem = {
@@ -105,6 +107,7 @@ export type ParagraphTypeFormItem = {
   label: string
   variable: string
   required: boolean
+  hide: boolean
 }
 /**
  * User Input Form Item

web/utils/model-config.ts

@@ -40,6 +40,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         max_length: content.max_length,
         options: [],
         is_context_var,
+        hide: content.hide,
       })
     }
     else if (type === 'number') {
@@ -49,6 +50,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         required: content.required,
         type,
         options: [],
+        hide: content.hide,
       })
     }
     else if (type === 'select') {
@@ -59,6 +61,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         type: 'select',
         options: content.options,
         is_context_var,
+        hide: content.hide,
       })
     }
     else if (type === 'file') {
@@ -73,6 +76,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
           allowed_file_upload_methods: content.allowed_file_upload_methods,
           number_limits: 1,
         },
+        hide: content.hide,
       })
     }
     else if (type === 'file-list') {
@@ -87,6 +91,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
           allowed_file_upload_methods: content.allowed_file_upload_methods,
           number_limits: content.max_length,
         },
+        hide: content.hide,
       })
     }
     else {
@@ -100,6 +105,7 @@ export const userInputsFormToPromptVariables = (useInputs: UserInputFormItem[] |
         icon: content.icon,
         icon_background: content.icon_background,
         is_context_var,
+        hide: content.hide,
       })
     }
   })
@@ -119,6 +125,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required !== false, // default true
           max_length: item.max_length,
           default: '',
+          hide: item.hide,
         },
       } as any)
       return
@@ -130,6 +137,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           variable: item.key,
           required: item.required !== false, // default true
           default: '',
+          hide: item.hide,
         },
       } as any)
     }
@@ -141,6 +149,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required !== false, // default true
           options: item.options,
           default: '',
+          hide: item.hide,
         },
       } as any)
     }
@@ -155,6 +164,7 @@ export const promptVariablesToUserInputsForm = (promptVariables: PromptVariable[
           required: item.required,
           icon: item.icon,
           icon_background: item.icon_background,
+          hide: item.hide,
         },
       } as any)
     }