diff --git a/web/app/components/rag-pipeline/components/input-field/editor/index.tsx b/web/app/components/rag-pipeline/components/input-field/editor/index.tsx
index c6b5bd131a..5641400e3e 100644
--- a/web/app/components/rag-pipeline/components/input-field/editor/index.tsx
+++ b/web/app/components/rag-pipeline/components/input-field/editor/index.tsx
@@ -48,7 +48,7 @@ const InputFieldEditor = ({
options,
placeholder,
unit,
- default: defaultValue,
+ default_value: defaultValue,
allowed_file_upload_methods: allowedFileUploadMethods,
allowed_file_types: allowedTypesAndExtensions.allowedFileTypes,
allowed_file_extensions: allowedTypesAndExtensions.allowedFileExtensions,
diff --git a/web/app/components/rag-pipeline/components/input-field/editor/utils.ts b/web/app/components/rag-pipeline/components/input-field/editor/utils.ts
index 655626550a..f9c2d143dd 100644
--- a/web/app/components/rag-pipeline/components/input-field/editor/utils.ts
+++ b/web/app/components/rag-pipeline/components/input-field/editor/utils.ts
@@ -14,7 +14,7 @@ export const convertToInputFieldFormData = (data?: InputVar): FormData => {
label,
variable,
max_length,
- 'default': defaultValue,
+ default_value,
required,
tooltips,
options,
@@ -30,7 +30,7 @@ export const convertToInputFieldFormData = (data?: InputVar): FormData => {
label,
variable,
maxLength: max_length,
- default: defaultValue,
+ default: default_value,
required,
tooltips,
options,
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
index 7b9b10a8e7..a5f6a769ce 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
@@ -1,4 +1,4 @@
-import { useCallback } from 'react'
+import { useCallback, useEffect } from 'react'
import { useDataSourceOptions } from '../hooks'
import OptionCard from './option-card'
import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
@@ -9,9 +9,8 @@ import { DataSourceProvider } from '@/models/common'
import type { Datasource } from '../types'
type DataSourceOptionsProps = {
- dataSources: Datasource[]
dataSourceNodeId: string
- onSelect: (option: string) => void
+ onSelect: (option: Datasource) => void
}
const DATA_SOURCE_ICONS = {
@@ -23,15 +22,23 @@ const DATA_SOURCE_ICONS = {
}
const DataSourceOptions = ({
- dataSources,
dataSourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
- const options = useDataSourceOptions(dataSources)
+ const { dataSources, options } = useDataSourceOptions()
const handelSelect = useCallback((value: string) => {
- onSelect(value)
- }, [onSelect])
+ const selectedOption = dataSources.find(option => option.nodeId === value)
+ if (!selectedOption)
+ return
+ onSelect(selectedOption)
+ }, [dataSources, onSelect])
+
+ useEffect(() => {
+ if (options.length > 0)
+ handelSelect(options[0].value)
+ // eslint-disable-next-line react-hooks/exhaustive-deps
+ }, [])
return (
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
index 2a9e764a9a..81121bbfe4 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
@@ -1,21 +1,24 @@
import { useMemo } from 'react'
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
import { useStore } from '@/app/components/workflow/store'
-import { InputVarType } from '@/app/components/workflow/types'
import { usePipelineProcessingParams } from '@/service/use-pipeline'
+import { PipelineInputVarType } from '@/models/pipeline'
-type PartialInputVarType = InputVarType.textInput | InputVarType.number | InputVarType.select | InputVarType.checkbox
+type PartialInputVarType = PipelineInputVarType.textInput | PipelineInputVarType.number | PipelineInputVarType.select | PipelineInputVarType.checkbox
const VAR_TYPE_MAP: Record
= {
- [InputVarType.textInput]: BaseFieldType.textInput,
- [InputVarType.number]: BaseFieldType.numberInput,
- [InputVarType.select]: BaseFieldType.select,
- [InputVarType.checkbox]: BaseFieldType.checkbox,
+ [PipelineInputVarType.textInput]: BaseFieldType.textInput,
+ [PipelineInputVarType.number]: BaseFieldType.numberInput,
+ [PipelineInputVarType.select]: BaseFieldType.select,
+ [PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
}
-export const useConfigurations = () => {
+export const useConfigurations = (datasourceNodeId: string) => {
const pipelineId = useStore(state => state.pipelineId)
- const { data: paramsConfig } = usePipelineProcessingParams(pipelineId!)
+ const { data: paramsConfig } = usePipelineProcessingParams({
+ pipeline_id: pipelineId!,
+ node_id: datasourceNodeId,
+ })
const initialData = useMemo(() => {
const variables = paramsConfig?.variables || []
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
index 33d788dc58..a3a8494c94 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
@@ -6,15 +6,17 @@ import type { FormType } from '@/app/components/base/form'
import { useCallback } from 'react'
type DocumentProcessingProps = {
+ dataSourceNodeId: string
onProcess: (data: Record) => void
onBack: () => void
}
const DocumentProcessing = ({
+ dataSourceNodeId,
onProcess,
onBack,
}: DocumentProcessingProps) => {
- const { initialData, configurations } = useConfigurations()
+ const { initialData, configurations } = useConfigurations(dataSourceNodeId)
const schema = generateZodSchema(configurations)
const renderCustomActions = useCallback((form: FormType) => (
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
index 4d7cbbcd9b..e99463ef47 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
+++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
@@ -3,6 +3,10 @@ import type { DataSourceOption, Datasource } from './types'
import { TestRunStep } from './types'
import { DataSourceType } from '@/models/datasets'
import { DataSourceProvider } from '@/models/common'
+import { useNodes } from 'reactflow'
+import { BlockEnum } from '@/app/components/workflow/types'
+import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
+import { useMemo } from 'react'
export const useTestRunSteps = () => {
const { t } = useTranslation()
@@ -19,45 +23,78 @@ export const useTestRunSteps = () => {
return steps
}
-export const useDataSourceOptions = (dataSources: Datasource[]) => {
+export const useDataSourceOptions = () => {
const { t } = useTranslation()
- const options: DataSourceOption[] = []
- dataSources.forEach((source) => {
- if (source.type === DataSourceType.FILE) {
- options.push({
- label: t('datasetPipeline.testRun.dataSource.localFiles'),
- value: source.nodeId,
- type: DataSourceType.FILE,
- })
- }
- if (source.type === DataSourceType.NOTION) {
- options.push({
- label: 'Notion',
- value: source.nodeId,
- type: DataSourceType.NOTION,
- })
- }
- if (source.type === DataSourceProvider.fireCrawl) {
- options.push({
- label: 'Firecrawl',
- value: source.nodeId,
- type: DataSourceProvider.fireCrawl,
- })
- }
- if (source.type === DataSourceProvider.jinaReader) {
- options.push({
- label: 'Jina Reader',
- value: source.nodeId,
- type: DataSourceProvider.jinaReader,
- })
- }
- if (source.type === DataSourceProvider.waterCrawl) {
- options.push({
- label: 'Water Crawl',
- value: source.nodeId,
- type: DataSourceProvider.waterCrawl,
- })
- }
- })
- return options
+ const nodes = useNodes()
+ const dataSources: Datasource[] = useMemo(() => {
+ const dataSourceNodes = nodes.filter(node => node.data.type === BlockEnum.DataSource)
+ return dataSourceNodes.map((node) => {
+ let type: DataSourceType | DataSourceProvider = DataSourceType.FILE
+ switch (node.data.tool_name) {
+ case 'file_upload':
+ type = DataSourceType.FILE
+ break
+ case 'search_notion':
+ type = DataSourceType.NOTION
+ break
+ case 'firecrawl':
+ type = DataSourceProvider.fireCrawl
+ break
+ case 'jina_reader':
+ type = DataSourceProvider.jinaReader
+ break
+ case 'water_crawl':
+ type = DataSourceProvider.waterCrawl
+ break
+ }
+ return {
+ nodeId: node.id,
+ type,
+ config: {},
+ }
+ })
+ }, [nodes])
+
+ const options = useMemo(() => {
+ const options: DataSourceOption[] = []
+ dataSources.forEach((source) => {
+ if (source.type === DataSourceType.FILE) {
+ options.push({
+ label: t('datasetPipeline.testRun.dataSource.localFiles'),
+ value: source.nodeId,
+ type: DataSourceType.FILE,
+ })
+ }
+ if (source.type === DataSourceType.NOTION) {
+ options.push({
+ label: 'Notion',
+ value: source.nodeId,
+ type: DataSourceType.NOTION,
+ })
+ }
+ if (source.type === DataSourceProvider.fireCrawl) {
+ options.push({
+ label: 'Firecrawl',
+ value: source.nodeId,
+ type: DataSourceProvider.fireCrawl,
+ })
+ }
+ if (source.type === DataSourceProvider.jinaReader) {
+ options.push({
+ label: 'Jina Reader',
+ value: source.nodeId,
+ type: DataSourceProvider.jinaReader,
+ })
+ }
+ if (source.type === DataSourceProvider.waterCrawl) {
+ options.push({
+ label: 'Water Crawl',
+ value: source.nodeId,
+ type: DataSourceProvider.waterCrawl,
+ })
+ }
+ })
+ return options
+ }, [dataSources, t])
+ return { dataSources, options }
}
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
index 0f11006ec0..17a48af315 100644
--- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
+++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx
@@ -26,11 +26,7 @@ const TestRunPanel = () => {
const { t } = useTranslation()
const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
const [currentStep, setCurrentStep] = useState(1)
- const [datasource, setDatasource] = useState({
- nodeId: '1',
- type: DataSourceType.FILE,
- config: {},
- })
+ const [datasource, setDatasource] = useState()
const [fileList, setFiles] = useState([])
const [notionPages, setNotionPages] = useState([])
const [websitePages, setWebsitePages] = useState([])
@@ -41,28 +37,6 @@ const TestRunPanel = () => {
const enableBilling = useProviderContextSelector(state => state.enableBilling)
const steps = useTestRunSteps()
- // TODO: replace with real data sources from API
- const dataSources = useMemo(() => [{
- nodeId: '1',
- type: DataSourceType.FILE,
- config: {},
- }, {
- nodeId: '2',
- type: DataSourceType.NOTION,
- config: {},
- }, {
- nodeId: '3',
- type: DataSourceProvider.fireCrawl,
- config: {},
- }, {
- nodeId: '4',
- type: DataSourceProvider.jinaReader,
- config: {},
- }, {
- nodeId: '5',
- type: DataSourceProvider.waterCrawl,
- config: {},
- }], [])
const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
@@ -77,6 +51,7 @@ const TestRunPanel = () => {
}, [fileList, isShowVectorSpaceFull])
const nextBtnDisabled = useMemo(() => {
+ if (!datasource) return false
if (datasource.type === DataSourceType.FILE)
return nextDisabled
if (datasource.type === DataSourceType.NOTION)
@@ -92,13 +67,6 @@ const TestRunPanel = () => {
setShowDebugAndPreviewPanel(false)
}
- const handleDataSourceSelect = useCallback((option: string) => {
- const dataSource = dataSources.find(dataSource => dataSource.nodeId === option)
- if (!dataSource)
- return
- setDatasource(dataSource)
- }, [dataSources])
-
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
const newList = produce(list, (draft) => {
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
@@ -129,6 +97,8 @@ const TestRunPanel = () => {
const { handleRun } = usePipelineRun()
const handleProcess = useCallback((data: Record) => {
+ if (!datasource)
+ return
const datasourceInfo: Record = {}
if (datasource.type === DataSourceType.FILE)
datasourceInfo.fileId = fileList.map(file => file.fileID)
@@ -176,11 +146,10 @@ const TestRunPanel = () => {
<>
- {datasource.type === DataSourceType.FILE && (
+ {datasource?.type === DataSourceType.FILE && (
{
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
- {datasource.type === DataSourceType.NOTION && (
+ {datasource?.type === DataSourceType.NOTION && (
)}
- {datasource.type === DataSourceProvider.fireCrawl && (
+ {datasource?.type === DataSourceProvider.fireCrawl && (
{
onCrawlOptionsChange={setCrawlOptions}
/>
)}
- {datasource.type === DataSourceProvider.jinaReader && (
+ {datasource?.type === DataSourceProvider.jinaReader && (
{
onCrawlOptionsChange={setCrawlOptions}
/>
)}
- {datasource.type === DataSourceProvider.waterCrawl && (
+ {datasource?.type === DataSourceProvider.waterCrawl && (
{
{
currentStep === 2 && (
diff --git a/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts b/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts
index 7bd4697a1d..0c8f1d51c9 100644
--- a/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts
+++ b/web/app/components/rag-pipeline/hooks/use-pipeline-init.ts
@@ -81,7 +81,7 @@ export const usePipelineInit = () => {
useEffect(() => {
handleGetInitialWorkflowData()
- // eslint-disable-next-line react-hooks/exhaustive-deps
+ // eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
return {
diff --git a/web/app/components/workflow/nodes/data-source/types.ts b/web/app/components/workflow/nodes/data-source/types.ts
index 3a8c06ab4c..969ef15647 100644
--- a/web/app/components/workflow/nodes/data-source/types.ts
+++ b/web/app/components/workflow/nodes/data-source/types.ts
@@ -1,3 +1,6 @@
import type { CommonNodeType } from '@/app/components/workflow/types'
+import type { RAGPipelineVariables } from '@/models/pipeline'
-export type DataSourceNodeType = CommonNodeType
+export type DataSourceNodeType = CommonNodeType & {
+ variables: RAGPipelineVariables
+}
diff --git a/web/models/pipeline.ts b/web/models/pipeline.ts
index 4578fe59c4..7f94ab0ab2 100644
--- a/web/models/pipeline.ts
+++ b/web/models/pipeline.ts
@@ -113,7 +113,7 @@ export type RAGPipelineVariable = {
label: string
variable: string
max_length?: number
- default?: string
+ default_value?: string
placeholder?: string
unit?: string
required: boolean
@@ -125,9 +125,40 @@ export type RAGPipelineVariable = {
}
export type InputVar = Omit
+export type RAGPipelineVariables = RAGPipelineVariable[]
-export type PipelineProcessingParamsResponse = {
- variables: RAGPipelineVariable[]
+export type PipelineProcessingParamsRequest = {
+ pipeline_id: string
+ node_id: string
}
-export type RAGPipelineVariables = RAGPipelineVariable[]
+export type PipelineProcessingParamsResponse = {
+ variables: RAGPipelineVariables
+}
+
+export type PipelineDatasourceNodeRunRequest = {
+ pipeline_id: string
+ node_id: string
+ inputs: Record
+}
+
+export type PipelineDatasourceNodeRunResponse = {
+ id: string
+ inputs: Record
+ process_data: Record
+ outputs: Record
+ status: string
+ error?: string
+ elapsed_time: number
+ execution_metadata: {
+ total_tokens: number
+ total_price: number
+ currency?: string
+ }
+ extras: {
+ icon: string | object
+ }
+ created_at: string
+ created_by: string
+ finished_at: string
+}
diff --git a/web/service/use-pipeline.ts b/web/service/use-pipeline.ts
index 471405abdf..8e9d7e879c 100644
--- a/web/service/use-pipeline.ts
+++ b/web/service/use-pipeline.ts
@@ -8,6 +8,8 @@ import type {
ImportPipelineDSLRequest,
ImportPipelineDSLResponse,
PipelineCheckDependenciesResponse,
+ PipelineDatasourceNodeRunRequest,
+ PipelineProcessingParamsRequest,
PipelineProcessingParamsResponse,
PipelineTemplateByIdResponse,
PipelineTemplateListParams,
@@ -95,7 +97,7 @@ export const useImportPipelineDSLConfirm = (
return useMutation({
mutationKey: [NAME_SPACE, 'dsl-import-confirm'],
mutationFn: (importId: string) => {
- return post(`/rag/pipeline/imports/${importId}/confirm`)
+ return post(`/rag/pipelines/imports/${importId}/confirm`)
},
...mutationOptions,
})
@@ -113,12 +115,29 @@ export const useCheckPipelineDependencies = (
})
}
+export const useDatasourceNodeRun = () => {
+ return useMutation({
+ mutationKey: [NAME_SPACE, 'datasource-node-run'],
+ mutationFn: (request: PipelineDatasourceNodeRunRequest) => {
+ const { pipeline_id, node_id, ...rest } = request
+ return post(`/rag/pipelines/${pipeline_id}/workflows/published/nodes/${node_id}/run`, {
+ body: rest,
+ })
+ },
+ })
+}
+
// Get the config of shared input fields
-export const usePipelineProcessingParams = (pipelineId: string) => {
+export const usePipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
+ const { pipeline_id, node_id } = params
return useQuery({
- queryKey: [NAME_SPACE, 'pipeline-processing-params', pipelineId],
+ queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
queryFn: () => {
- return get(`/rag/pipeline/${pipelineId}/workflows/processing/parameters`)
+ return get(`/rag/pipelines/${pipeline_id}/workflows/processing/parameters`, {
+ params: {
+ node_id,
+ },
+ })
},
})
}