opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-04-14 16:08:01 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: update data source handling and improve processing parameters integration

Browse Source
This commit is contained in:
twwu 2025-06-05 10:24:25 +08:00
parent 4c2cc98ebc
commit 842136959b
9 changed files with 54 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -274,7 +274,6 @@ const CreateFormPipeline = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={[]} // todo: replace with actual variables if needed
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
@ -284,6 +283,7 @@ const CreateFormPipeline = () => {
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
usingPublished
/>
)}
{isShowVectorSpaceFull && (

17
web/app/components/rag-pipeline/components/input-field/index.tsx
View File

@ -56,16 +56,17 @@ const InputFieldDialog = ({
const { doSyncWorkflowDraft } = useNodesSyncDraft()
useUnmount(() => {
doSyncWorkflowDraft()
})
const { run: syncWorkflowDraft } = useDebounceFn(() => {
doSyncWorkflowDraft()
const { run: syncWorkflowDraft, cancel: cancelSyncWorkflowDraft } = useDebounceFn(async () => {
await doSyncWorkflowDraft()
}, {
wait: 500,
})
useUnmount(() => {
cancelSyncWorkflowDraft()
doSyncWorkflowDraft()
})
const datasourceNodeDataMap = useMemo(() => {
const datasourceNodeDataMap: Record<string, DataSourceNodeType> = {}
const datasourceNodes: Node<DataSourceNodeType>[] = nodes.filter(node => node.data.type === BlockEnum.DataSource)
@ -76,7 +77,7 @@ const InputFieldDialog = ({
return datasourceNodeDataMap
}, [nodes])
const updateInputFields = useCallback((key: string, value: InputVar[]) => {
const updateInputFields = useCallback(async (key: string, value: InputVar[]) => {
inputFieldsMap.current[key] = value
const newRagPipelineVariables: RAGPipelineVariables = []
Object.keys(inputFieldsMap.current).forEach((key) => {
@ -89,7 +90,7 @@ const InputFieldDialog = ({
})
})
setRagPipelineVariables?.(newRagPipelineVariables)
syncWorkflowDraft()
await syncWorkflowDraft()
}, [setRagPipelineVariables, syncWorkflowDraft])
const closePanel = useCallback(() => {

15
web/app/components/rag-pipeline/components/input-field/preview/data-source.tsx
View File

@ -3,17 +3,24 @@ import { useTranslation } from 'react-i18next'
import DataSourceOptions from '../../panel/test-run/data-source-options'
import Form from './form'
import type { Datasource } from '../../panel/test-run/types'
import { useStore } from '@/app/components/workflow/store'
import { useDraftPipelinePreProcessingParams } from '@/service/use-pipeline'
type DatasourceProps = {
onSelect: (dataSource: Datasource) => void
datasourceNodeId: string
dataSourceNodeId: string
}
const DataSource = ({
onSelect: setDatasource,
datasourceNodeId,
dataSourceNodeId,
}: DatasourceProps) => {
const { t } = useTranslation()
const pipelineId = useStore(state => state.pipelineId)
const { data: paramsConfig } = useDraftPipelinePreProcessingParams({
pipeline_id: pipelineId!,
node_id: dataSourceNodeId,
}, !!pipelineId && !!dataSourceNodeId)
return (
<div className='flex flex-col'>
@ -23,10 +30,10 @@ const DataSource = ({
<div className='px-4 py-2'>
<DataSourceOptions
onSelect={setDatasource}
datasourceNodeId={datasourceNodeId}
dataSourceNodeId={dataSourceNodeId}
/>
</div>
<Form variables={[]} />
<Form variables={paramsConfig?.variables || []} />
</div>
)
}

4
web/app/components/rag-pipeline/components/input-field/preview/index.tsx
View File

@ -40,13 +40,15 @@ const PreviewPanel = ({
<RiCloseLine className='size-4 text-text-tertiary' />
</button>
</div>
{/* Data source form Preview */}
<DataSource
onSelect={setDatasource}
datasourceNodeId={datasource?.nodeId || ''}
dataSourceNodeId={datasource?.nodeId || ''}
/>
<div className='px-4 py-2'>
<Divider type='horizontal' className='bg-divider-subtle' />
</div>
{/* Process documents form Preview */}
<ProcessDocuments dataSourceNodeId={datasource?.nodeId || ''} />
</DialogWrapper>
)

6
web/app/components/rag-pipeline/components/panel/test-run/data-source-options/index.tsx
View File

@ -4,12 +4,12 @@ import OptionCard from './option-card'
import type { Datasource } from '../types'
type DataSourceOptionsProps = {
datasourceNodeId: string
dataSourceNodeId: string
onSelect: (option: Datasource) => void
}
const DataSourceOptions = ({
datasourceNodeId,
dataSourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
const { datasources, options } = useDatasourceOptions()
@ -34,7 +34,7 @@ const DataSourceOptions = ({
key={option.value}
label={option.label}
nodeData={option.data}
selected={datasourceNodeId === option.value}
selected={dataSourceNodeId === option.value}
onClick={handelSelect.bind(null, option.value)}
/>
))}

25
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
View File

@ -1,5 +1,5 @@
'use client'
import React, { useCallback, useEffect, useState } from 'react'
import React, { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResultItem } from '@/models/datasets'
import Header from '@/app/components/datasets/create/website/base/header'
@ -7,15 +7,17 @@ import Options from './options'
import Crawling from './crawling'
import ErrorMessage from './error-message'
import CrawledResult from './crawled-result'
import type { RAGPipelineVariables } from '@/models/pipeline'
import { useDatasourceNodeRun } from '@/service/use-pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import {
useDatasourceNodeRun,
useDraftPipelinePreProcessingParams,
usePublishedPipelineProcessingParams,
} from '@/service/use-pipeline'
import { useStore } from '@/app/components/workflow/store'
const I18N_PREFIX = 'datasetCreation.stepOne.website'
type CrawlerProps = {
nodeId: string
variables: RAGPipelineVariables
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
@ -25,6 +27,7 @@ type CrawlerProps = {
docLink: string
}
onPreview?: (payload: CrawlResultItem) => void
usingPublished?: boolean
}
enum Step {
@ -35,17 +38,23 @@ enum Step {
const Crawler = ({
nodeId,
variables,
checkedCrawlResult,
headerInfo,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
usingPublished = false,
}: CrawlerProps) => {
const { t } = useTranslation()
const [step, setStep] = useState<Step>(Step.init)
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const pipelineId = useStore(s => s.pipelineId)
const usePreProcessingParams = useRef(usingPublished ? usePublishedPipelineProcessingParams : useDraftPipelinePreProcessingParams)
const { data: paramsConfig } = usePreProcessingParams.current({
pipeline_id: pipelineId!,
node_id: nodeId,
}, !!pipelineId && !!nodeId)
useEffect(() => {
if (step !== Step.init)
@ -95,7 +104,7 @@ const Crawler = ({
/>
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
<Options
variables={variables}
variables={paramsConfig?.variables || []}
isRunning={isRunning}
controlFoldOptions={controlFoldOptions}
onSubmit={(value) => {

7
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx
View File

@ -1,12 +1,10 @@
'use client'
import React from 'react'
import type { CrawlResultItem } from '@/models/datasets'
import type { RAGPipelineVariables } from '@/models/pipeline'
import Crawler from './base/crawler'
type WebsiteCrawlProps = {
nodeId: string
variables: RAGPipelineVariables
checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void
@ -16,26 +14,27 @@ type WebsiteCrawlProps = {
docLink: string
}
onPreview?: (payload: CrawlResultItem) => void
usingPublished?: boolean
}
const WebsiteCrawl = ({
nodeId,
variables,
checkedCrawlResult,
headerInfo,
onCheckedCrawlResultChange,
onJobIdChange,
onPreview,
usingPublished,
}: WebsiteCrawlProps) => {
return (
<Crawler
nodeId={nodeId}
variables={variables}
checkedCrawlResult={checkedCrawlResult}
headerInfo={headerInfo}
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange}
onPreview={onPreview}
usingPublished={usingPublished}
/>
)
}

3
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -117,7 +117,7 @@ const TestRunPanel = () => {
<>
<div className='flex flex-col gap-y-4 px-4 py-2'>
<DataSourceOptions
datasourceNodeId={datasource?.nodeId || ''}
dataSourceNodeId={datasource?.nodeId || ''}
onSelect={setDatasource}
/>
{datasource?.type === DatasourceType.localFile && (
@ -139,7 +139,6 @@ const TestRunPanel = () => {
{datasource?.type === DatasourceType.websiteCrawl && (
<WebsiteCrawl
nodeId={datasource?.nodeId || ''}
variables={[]} // todo: replace with actual variables if needed
checkedCrawlResult={websitePages}
headerInfo={{
title: datasource.description,

10
web/service/use-pipeline.ts
View File

@ -141,7 +141,7 @@ export const useDatasourceNodeRun = (
export const useDraftPipelineProcessingParams = (params: PipelineProcessingParamsRequest, enabled = true) => {
const { pipeline_id, node_id } = params
return useQuery<PipelineProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id],
queryFn: () => {
return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/processing/parameters`, {
params: {
@ -157,7 +157,7 @@ export const useDraftPipelineProcessingParams = (params: PipelineProcessingParam
export const usePublishedPipelineProcessingParams = (params: PipelineProcessingParamsRequest) => {
const { pipeline_id, node_id } = params
return useQuery<PipelineProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id, node_id],
queryFn: () => {
return get<PipelineProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
params: {
@ -165,6 +165,7 @@ export const usePublishedPipelineProcessingParams = (params: PipelineProcessingP
},
})
},
staleTime: 0,
})
}
@ -254,7 +255,7 @@ export const useUpdateDataSourceCredentials = (
export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
const { pipeline_id, node_id } = params
return useQuery<PipelinePreProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id],
queryFn: () => {
return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/draft/pre-processing/parameters`, {
params: {
@ -270,7 +271,7 @@ export const useDraftPipelinePreProcessingParams = (params: PipelinePreProcessin
export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProcessingParamsRequest, enabled = true) => {
const { pipeline_id, node_id } = params
return useQuery<PipelinePreProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'pipeline-processing-params', pipeline_id],
queryKey: [NAME_SPACE, 'pipeline-pre-processing-params', pipeline_id, node_id],
queryFn: () => {
return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
params: {
@ -278,6 +279,7 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce
},
})
},
staleTime: 0,
enabled,
})
}