Merge branch 'feat/rag-pipeline' of https://github.com/langgenius/dify into feat/rag-pipeline

web/app/components/tools/types.ts

@@ -51,6 +51,7 @@ export type Collection = {
   labels: string[]
   plugin_id?: string
   letter?: string
+  is_authorized?: boolean
 }
 
 export type ToolParameter = {

web/app/components/workflow/block-selector/types.ts

@@ -81,4 +81,5 @@ export type DataSourceItem = {
       parameters: any[]
     }[]
   }
+  is_authorized: boolean
 }

web/app/components/workflow/block-selector/utils.ts

@@ -4,6 +4,7 @@ import type { DataSourceItem } from './types'
 export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
   return {
     id: dataSourceItem.plugin_id,
+    provider: dataSourceItem.provider,
     name: dataSourceItem.declaration.identity.name,
     author: dataSourceItem.declaration.identity.author,
     description: dataSourceItem.declaration.identity.description,
@@ -11,8 +12,8 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
     label: dataSourceItem.declaration.identity.label,
     type: dataSourceItem.declaration.provider_type,
     team_credentials: {},
-    is_team_authorization: false,
     allow_delete: true,
+    is_authorized: dataSourceItem.is_authorized,
     labels: dataSourceItem.declaration.identity.tags || [],
     plugin_id: dataSourceItem.plugin_id,
     tools: dataSourceItem.declaration.datasources.map((datasource) => {
@@ -26,5 +27,6 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
         output_schema: {},
       } as Tool
     }),
+    credentialsSchema: dataSourceItem.declaration.credentials_schema || [],
   }
 }

web/app/components/workflow/nodes/data-source/components/config-credential.tsx

@@ -0,0 +1,137 @@
+'use client'
+import type { FC } from 'react'
+import {
+  memo,
+  useCallback,
+  useMemo,
+  useState,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import { addDefaultValue, toolCredentialToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
+import cn from '@/utils/classnames'
+import Drawer from '@/app/components/base/drawer-plus'
+import Button from '@/app/components/base/button'
+import Toast from '@/app/components/base/toast'
+import Loading from '@/app/components/base/loading'
+import Form from '@/app/components/header/account-setting/model-provider-page/model-modal/Form'
+import { LinkExternal02 } from '@/app/components/base/icons/src/vender/line/general'
+import { useLanguage } from '@/app/components/header/account-setting/model-provider-page/hooks'
+import { noop } from 'lodash-es'
+import { useDataSourceCredentials } from '@/service/use-pipeline'
+import type { ToolCredential } from '@/app/components/tools/types'
+
+type Props = {
+  dataSourceItem: any
+  onCancel: () => void
+  onSaved: (value: Record<string, any>) => void
+  isHideRemoveBtn?: boolean
+  onRemove?: () => void
+  isSaving?: boolean
+}
+
+const ConfigCredential: FC<Props> = ({
+  dataSourceItem,
+  onCancel,
+  onSaved,
+  isHideRemoveBtn,
+  onRemove = noop,
+  isSaving,
+}) => {
+  const { t } = useTranslation()
+  const language = useLanguage()
+  const {
+    provider,
+    plugin_id,
+    credentialsSchema = [],
+    is_authorized,
+  } = dataSourceItem
+  const transformedCredentialsSchema = useMemo(() => {
+    return toolCredentialToFormSchemas(credentialsSchema)
+  }, [credentialsSchema])
+  const [isLoading, setIsLoading] = useState(false)
+  const [tempCredential, setTempCredential] = useState<any>({})
+  const handleUpdateCredentials = useCallback((credentialValue: ToolCredential[]) => {
+    const defaultCredentials = addDefaultValue(credentialValue, transformedCredentialsSchema)
+    setTempCredential(defaultCredentials)
+  }, [transformedCredentialsSchema])
+  useDataSourceCredentials(provider, plugin_id, handleUpdateCredentials)
+
+  const handleSave = async () => {
+    for (const field of transformedCredentialsSchema) {
+      if (field.required && !tempCredential[field.name]) {
+        Toast.notify({ type: 'error', message: t('common.errorMsg.fieldRequired', { field: field.label[language] || field.label.en_US }) })
+        return
+      }
+    }
+    setIsLoading(true)
+    try {
+      await onSaved(tempCredential)
+      setIsLoading(false)
+    }
+    finally {
+      setIsLoading(false)
+    }
+  }
+
+  return (
+    <Drawer
+      isShow
+      onHide={onCancel}
+      title={t('tools.auth.setupModalTitle') as string}
+      titleDescription={t('tools.auth.setupModalTitleDescription') as string}
+      panelClassName='mt-[64px] mb-2 !w-[420px] border-components-panel-border'
+      maxWidthClassName='!max-w-[420px]'
+      height='calc(100vh - 64px)'
+      contentClassName='!bg-components-panel-bg'
+      headerClassName='!border-b-divider-subtle'
+      body={
+
+        <div className='h-full px-6 py-3'>
+          {!transformedCredentialsSchema.length
+            ? <Loading type='app' />
+            : (
+              <>
+                <Form
+                  value={tempCredential}
+                  onChange={(v) => {
+                    setTempCredential(v)
+                  }}
+                  formSchemas={transformedCredentialsSchema as any}
+                  isEditMode={true}
+                  showOnVariableMap={{}}
+                  validating={false}
+                  inputClassName='!bg-components-input-bg-normal'
+                  fieldMoreInfo={item => item.url
+                    ? (<a
+                      href={item.url}
+                      target='_blank' rel='noopener noreferrer'
+                      className='inline-flex items-center text-xs text-text-accent'
+                    >
+                      {t('tools.howToGet')}
+                      <LinkExternal02 className='ml-1 h-3 w-3' />
+                    </a>)
+                    : null}
+                />
+                <div className={cn((is_authorized && !isHideRemoveBtn) ? 'justify-between' : 'justify-end', 'mt-2 flex ')} >
+                  {
+                    (is_authorized && !isHideRemoveBtn) && (
+                      <Button onClick={onRemove}>{t('common.operation.remove')}</Button>
+                    )
+                  }
+                  < div className='flex space-x-2'>
+                    <Button onClick={onCancel}>{t('common.operation.cancel')}</Button>
+                    <Button loading={isLoading || isSaving} disabled={isLoading || isSaving} variant='primary' onClick={handleSave}>{t('common.operation.save')}</Button>
+                  </div>
+                </div>
+              </>
+            )
+          }
+
+        </div >
+      }
+      isShowMask={true}
+      clickOutsideNotOpen={false}
+    />
+  )
+}
+export default memo(ConfigCredential)

web/app/components/workflow/nodes/data-source/hooks/use-config.ts

@@ -1,12 +1,18 @@
 import { useCallback } from 'react'
 import { useStoreApi } from 'reactflow'
+import { useTranslation } from 'react-i18next'
 import { useNodeDataUpdate } from '@/app/components/workflow/hooks'
-import type { InputVar } from '@/models/pipeline'
+import type {
-import type { DataSourceNodeType } from '../types'
+  DataSourceNodeType,
+  ToolVarInputs,
+} from '../types'
+import { useToastContext } from '@/app/components/base/toast'
 
 export const useConfig = (id: string) => {
   const store = useStoreApi()
   const { handleNodeDataUpdateWithSyncDraft } = useNodeDataUpdate()
+  const { notify } = useToastContext()
+  const { t } = useTranslation()
 
   const getNodeData = useCallback(() => {
     const { getNodes } = store.getState()
@@ -29,16 +35,16 @@ export const useConfig = (id: string) => {
     })
   }, [handleNodeDataUpdate, getNodeData])
 
-  const handleInputFieldVariablesChange = useCallback((variables: InputVar[]) => {
+  const handleParametersChange = useCallback((datasource_parameters: ToolVarInputs) => {
     const nodeData = getNodeData()
     handleNodeDataUpdate({
       ...nodeData?.data,
-      variables,
+      datasource_parameters,
     })
   }, [handleNodeDataUpdate, getNodeData])
 
   return {
     handleFileExtensionsChange,
-    handleInputFieldVariablesChange,
+    handleParametersChange,
   }
 }

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -1,29 +1,124 @@
 import type { FC } from 'react'
+import {
+  useCallback,
+  useMemo,
+  useState,
+} from 'react'
 import { useTranslation } from 'react-i18next'
 import { memo } from 'react'
+import { useBoolean } from 'ahooks'
 import type { DataSourceNodeType } from './types'
 import type { NodePanelProps } from '@/app/components/workflow/types'
-import { BoxGroupField } from '@/app/components/workflow/nodes/_base/components/layout'
+import {
+  BoxGroupField,
+  Group,
+  GroupField,
+} from '@/app/components/workflow/nodes/_base/components/layout'
 import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/components/output-vars'
 import TagInput from '@/app/components/base/tag-input'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import { useConfig } from './hooks/use-config'
 import { OUTPUT_VARIABLES_MAP } from './constants'
+import { useStore } from '@/app/components/workflow/store'
+import Button from '@/app/components/base/button'
+import ConfigCredential from './components/config-credential'
+import InputVarList from '@/app/components/workflow/nodes/tool/components/input-var-list'
+import { toolParametersToFormSchemas } from '@/app/components/tools/utils/to-form-schema'
+import type { Var } from '@/app/components/workflow/types'
+import { VarType } from '@/app/components/workflow/types'
+import { useToastContext } from '@/app/components/base/toast'
+import { useUpdateDataSourceCredentials } from '@/service/use-pipeline'
 
 const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const { t } = useTranslation()
+  const { notify } = useToastContext()
   const { nodesReadOnly } = useNodesReadOnly()
+  const dataSourceList = useStore(s => s.dataSourceList)
   const {
     provider_type,
+    provider_id,
     fileExtensions = [],
+    datasource_parameters,
   } = data
   const {
     handleFileExtensionsChange,
+    handleParametersChange,
   } = useConfig(id)
   const isLocalFile = provider_type === 'local_file'
+  const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === provider_id)
+  const isAuthorized = !!currentDataSource?.is_authorized
+  const [showAuthModal, {
+    setTrue: openAuthModal,
+    setFalse: hideAuthModal,
+  }] = useBoolean(false)
+  const currentDataSourceItem: any = currentDataSource?.tools.find(tool => tool.name === data.datasource_name)
+  const formSchemas = useMemo(() => {
+    return currentDataSourceItem ? toolParametersToFormSchemas(currentDataSourceItem.parameters) : []
+  }, [currentDataSourceItem])
+  const [currVarIndex, setCurrVarIndex] = useState(-1)
+  const currVarType = formSchemas[currVarIndex]?._type
+  const handleOnVarOpen = useCallback((index: number) => {
+    setCurrVarIndex(index)
+  }, [])
+
+  const filterVar = useCallback((varPayload: Var) => {
+    if (currVarType)
+      return varPayload.type === currVarType
+
+    return varPayload.type !== VarType.arrayFile
+  }, [currVarType])
+
+  const { mutateAsync } = useUpdateDataSourceCredentials()
+  const handleAuth = useCallback(async (value: any) => {
+    await mutateAsync({
+      provider: currentDataSourceItem?.provider,
+      pluginId: currentDataSourceItem?.plugin_id,
+      credentials: value,
+    })
+
+    notify({
+      type: 'success',
+      message: t('common.api.actionSuccess'),
+    })
+    hideAuthModal()
+  }, [currentDataSourceItem, mutateAsync, notify, t, hideAuthModal])
 
   return (
     <div >
+      {
+        !isAuthorized && !showAuthModal && (
+          <Group>
+            <Button
+              variant='primary'
+              className='w-full'
+              onClick={openAuthModal}
+              disabled={nodesReadOnly}
+            >
+              {t('workflow.nodes.tool.authorize')}
+            </Button>
+          </Group>
+        )
+      }
+      {
+        isAuthorized && (
+          <GroupField
+            groupProps={{
+              withBorderBottom: true,
+            }}
+          >
+            <InputVarList
+              readOnly={nodesReadOnly}
+              nodeId={id}
+              schema={formSchemas as any}
+              filterVar={filterVar}
+              value={datasource_parameters}
+              onChange={handleParametersChange}
+              isSupportConstantValue
+              onOpen={handleOnVarOpen}
+            />
+          </GroupField>
+        )
+      }
       {
         isLocalFile && (
           <BoxGroupField
@@ -70,6 +165,16 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
           )
         }
       </OutputVars>
+      {
+        showAuthModal && (
+          <ConfigCredential
+            dataSourceItem={currentDataSource!}
+            onCancel={hideAuthModal}
+            onSaved={handleAuth}
+            isHideRemoveBtn
+          />
+        )
+      }
     </div>
   )
 }

web/service/use-pipeline.ts

@@ -1,5 +1,5 @@
 import type { MutationOptions } from '@tanstack/react-query'
-import { useMutation, useQuery } from '@tanstack/react-query'
+import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'
 import { del, get, patch, post } from './base'
 import type {
   DeleteTemplateResponse,
@@ -23,6 +23,7 @@ import type {
   UpdateTemplateInfoResponse,
 } from '@/models/pipeline'
 import type { DataSourceItem } from '@/app/components/workflow/block-selector/types'
+import type { ToolCredential } from '@/app/components/tools/types'
 
 const NAME_SPACE = 'pipeline'
 
@@ -206,3 +207,39 @@ export const useRunPublishedPipeline = (
     ...mutationOptions,
   })
 }
+
+export const useDataSourceCredentials = (provider: string, pluginId: string, onSuccess: (value: ToolCredential[]) => void) => {
+  return useQuery<ToolCredential[]>({
+    queryKey: [NAME_SPACE, 'datasource-credentials', provider, pluginId],
+    queryFn: async () => {
+      const result = await get<ToolCredential[]>(`/auth/datasource/provider/${provider}/plugin/${pluginId}`)
+      onSuccess(result)
+      return result
+    },
+    enabled: !!provider && !!pluginId,
+    retry: 2,
+  })
+}
+
+export const useUpdateDataSourceCredentials = (
+) => {
+  const queryClient = useQueryClient()
+  return useMutation({
+    mutationKey: [NAME_SPACE, 'update-datasource-credentials'],
+    mutationFn: ({
+      provider,
+      pluginId,
+      credentials,
+    }: { provider: string; pluginId: string; credentials: Record<string, any>; }) => {
+      return post(`/auth/datasource/provider/${provider}/plugin/${pluginId}`, {
+        body: {
+          credentials,
+        },
+      }).then(() => {
+        queryClient.invalidateQueries({
+          queryKey: [NAME_SPACE, 'datasource'],
+        })
+      })
+    },
+  })
+}