datasource

web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts

@@ -23,6 +23,7 @@ export const useNodesSyncDraft = () => {
       edges,
       transform,
     } = store.getState()
+    const nodes = getNodes()
     const [x, y, zoom] = transform
     const {
       pipelineId,
@@ -32,8 +33,6 @@ export const useNodesSyncDraft = () => {
     } = workflowStore.getState()
 
     if (pipelineId) {
-      const nodes = getNodes()
-
       const producedNodes = produce(nodes, (draft) => {
         draft.forEach((node) => {
           Object.keys(node.data).forEach((key) => {

web/app/components/workflow/block-selector/data-sources.tsx

@@ -32,7 +32,7 @@ const DataSources = ({
   const wrapElemRef = useRef<HTMLDivElement>(null)
   const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => {
     onSelect(BlockEnum.DataSource, toolDefaultValue && {
-      provider_id: toolDefaultValue?.provider_id,
+      plugin_id: toolDefaultValue?.provider_id,
       provider_type: toolDefaultValue?.provider_type,
       provider_name: toolDefaultValue?.provider_name,
       datasource_name: toolDefaultValue?.tool_name,

web/app/components/workflow/block-selector/types.ts

@@ -36,7 +36,7 @@ export type ToolDefaultValue = {
 }
 
 export type DataSourceDefaultValue = {
-  provider_id: string
+  plugin_id: string
   provider_type: string
   provider_name: string
   datasource_name: string

web/app/components/workflow/block-selector/utils.ts

@@ -5,7 +5,7 @@ export const transformDataSourceToTool = (dataSourceItem: DataSourceItem) => {
   return {
     id: dataSourceItem.plugin_id,
     provider: dataSourceItem.provider,
-    name: dataSourceItem.declaration.identity.name,
+    name: dataSourceItem.provider,
     author: dataSourceItem.declaration.identity.author,
     description: dataSourceItem.declaration.identity.description,
     icon: dataSourceItem.declaration.identity.icon,

web/app/components/workflow/index.tsx

@@ -375,4 +375,4 @@ const WorkflowWithDefaultContext = ({
   )
 }
 
-export default memo(WorkflowWithDefaultContext)
+export default WorkflowWithDefaultContext

web/app/components/workflow/nodes/data-source/constants.ts

@@ -23,13 +23,16 @@ export const DEFAULT_FILE_EXTENSIONS_IN_LOCAL_FILE_DATA_SOURCE = [
   'html',
 ]
 
-export const OUTPUT_VARIABLES_MAP = {
+export const COMMON_OUTPUT = [
-  datasource_type: {
+  {
     name: 'datasource_type',
     type: VarType.string,
     description: 'local_file, online_document, website_crawl',
   },
-  file: {
+]
+
+export const FILE_OUTPUT = [
+  {
     name: 'file',
     type: VarType.file,
     description: 'file',
@@ -76,4 +79,27 @@ export const OUTPUT_VARIABLES_MAP = {
       },
     ],
   },
-}
+]
+
+export const WEBSITE_OUTPUT = [
+  {
+    name: 'source_url',
+    type: VarType.string,
+    description: 'The URL of the crawled website',
+  },
+  {
+    name: 'content',
+    type: VarType.string,
+    description: 'The content of the crawled website',
+  },
+  {
+    name: 'title',
+    type: VarType.string,
+    description: 'The title of the crawled website',
+  },
+  {
+    name: 'description',
+    type: VarType.string,
+    description: 'The description of the crawled website',
+  },
+]

web/app/components/workflow/nodes/data-source/default.ts

@@ -1,8 +1,13 @@
 import type { NodeDefault } from '../../types'
 import type { DataSourceNodeType } from './types'
+import { DataSourceClassification } from './types'
 import { genNodeMetaData } from '@/app/components/workflow/utils'
 import { BlockEnum } from '@/app/components/workflow/types'
-import { OUTPUT_VARIABLES_MAP } from './constants'
+import {
+  COMMON_OUTPUT,
+  FILE_OUTPUT,
+  WEBSITE_OUTPUT,
+} from './constants'
 
 const metaData = genNodeMetaData({
   sort: -1,
@@ -24,20 +29,18 @@ const nodeDefault: NodeDefault<DataSourceNodeType> = {
     const {
       provider_type,
     } = payload
-    const isLocalFile = provider_type === 'local_file'
+    const isLocalFile = provider_type === DataSourceClassification.file
+    const isWebsiteCrawl = provider_type === DataSourceClassification.website
     return [
-      {
+      ...COMMON_OUTPUT.map(item => ({ variable: item.name, type: item.type })),
-        variable: OUTPUT_VARIABLES_MAP.datasource_type.name,
-        type: OUTPUT_VARIABLES_MAP.datasource_type.type,
-      },
       ...(
         isLocalFile
-          ? [
+          ? FILE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
-              {
+          : []
-                variable: OUTPUT_VARIABLES_MAP.file.name,
+      ),
-                type: OUTPUT_VARIABLES_MAP.file.type,
+      ...(
-              },
+        isWebsiteCrawl
-          ]
+          ? WEBSITE_OUTPUT.map(item => ({ variable: item.name, type: item.type }))
           : []
       ),
       ...ragVars,

web/app/components/workflow/nodes/data-source/panel.tsx

@@ -8,6 +8,7 @@ import { useTranslation } from 'react-i18next'
 import { memo } from 'react'
 import { useBoolean } from 'ahooks'
 import type { DataSourceNodeType } from './types'
+import { DataSourceClassification } from './types'
 import type { NodePanelProps } from '@/app/components/workflow/types'
 import {
   BoxGroupField,
@@ -17,7 +18,11 @@ import OutputVars, { VarItem } from '@/app/components/workflow/nodes/_base/compo
 import TagInput from '@/app/components/base/tag-input'
 import { useNodesReadOnly } from '@/app/components/workflow/hooks'
 import { useConfig } from './hooks/use-config'
-import { OUTPUT_VARIABLES_MAP } from './constants'
+import {
+  COMMON_OUTPUT,
+  FILE_OUTPUT,
+  WEBSITE_OUTPUT,
+} from './constants'
 import { useStore } from '@/app/components/workflow/store'
 import Button from '@/app/components/base/button'
 import ConfigCredential from './components/config-credential'
@@ -35,7 +40,7 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
   const dataSourceList = useStore(s => s.dataSourceList)
   const {
     provider_type,
-    provider_id,
+    plugin_id,
     fileExtensions = [],
     datasource_parameters,
   } = data
@@ -43,8 +48,9 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
     handleFileExtensionsChange,
     handleParametersChange,
   } = useConfig(id)
-  const isLocalFile = provider_type === 'local_file'
+  const isLocalFile = provider_type === DataSourceClassification.file
-  const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === provider_id)
+  const isWebsiteCrawl = provider_type === DataSourceClassification.website
+  const currentDataSource = dataSourceList?.find(ds => ds.plugin_id === plugin_id)
   const isAuthorized = !!currentDataSource?.is_authorized
   const [showAuthModal, {
     setTrue: openAuthModal,
@@ -150,24 +156,37 @@ const Panel: FC<NodePanelProps<DataSourceNodeType>> = ({ id, data }) => {
         )
       }
       <OutputVars>
-        <VarItem
-          name={OUTPUT_VARIABLES_MAP.datasource_type.name}
-          type={OUTPUT_VARIABLES_MAP.datasource_type.type}
-          description={OUTPUT_VARIABLES_MAP.datasource_type.description}
-        />
         {
-          isLocalFile && (
+          COMMON_OUTPUT.map(item => (
             <VarItem
-              name={OUTPUT_VARIABLES_MAP.file.name}
+              name={item.name}
-              type={OUTPUT_VARIABLES_MAP.file.type}
+              type={item.type}
-              description={OUTPUT_VARIABLES_MAP.file.description}
+              description={item.description}
-              subItems={OUTPUT_VARIABLES_MAP.file.subItems.map(item => ({
+            />
+          ))
+        }
+        {
+          isLocalFile && FILE_OUTPUT.map(item => (
+            <VarItem
+              name={item.name}
+              type={item.type}
+              description={item.description}
+              subItems={item.subItems.map(item => ({
                 name: item.name,
                 type: item.type,
                 description: item.description,
               }))}
             />
-          )
+          ))
+        }
+        {
+          isWebsiteCrawl && WEBSITE_OUTPUT.map(item => (
+            <VarItem
+              name={item.name}
+              type={item.type}
+              description={item.description}
+            />
+          ))
         }
       </OutputVars>
       {

web/app/components/workflow/nodes/data-source/types.ts

@@ -6,6 +6,11 @@ export enum VarType {
   mixed = 'mixed',
 }
 
+export enum DataSourceClassification {
+  file = 'local_file',
+  website = 'website_crawl',
+}
+
 export type ToolVarInputs = Record<string, {
   type: VarType
   value?: string | ValueSelector | any
@@ -13,7 +18,7 @@ export type ToolVarInputs = Record<string, {
 
 export type DataSourceNodeType = CommonNodeType & {
   fileExtensions?: string[]
-  provider_id: string
+  plugin_id: string
   provider_type: string
   provider_name: string
   datasource_name: string

web/app/components/workflow/types.ts

@@ -100,7 +100,7 @@ export type CommonNodeType<T = {}> = {
   retry_config?: WorkflowRetryConfig
   default_value?: DefaultValueForm[]
 } & T & Partial<Pick<ToolDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'tool_name'>>
-  & Partial<Pick<DataSourceDefaultValue, 'provider_id' | 'provider_type' | 'provider_name' | 'datasource_name'>>
+  & Partial<Pick<DataSourceDefaultValue, 'plugin_id' | 'provider_type' | 'provider_name' | 'datasource_name'>>
 
 export type CommonEdgeType = {
   _hovering?: boolean