opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/rag-pipeline' into deploy/rag-dev

This commit is contained in:
twwu 2025-06-17 16:30:37 +08:00
parent 2db0b19044 796797d12b
commit 879ac940dd
11 changed files with 76 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
web/app/components/datasets/documents/create-from-pipeline/process-documents/actions.tsx
View File

@ -5,11 +5,13 @@ import { RiArrowLeftLine } from '@remixicon/react'
type ActionsProps = {
onBack: () => void
runDisabled?: boolean
onProcess: () => void
}
const Actions = ({
onBack,
runDisabled,
onProcess,
}: ActionsProps) => {
const { t } = useTranslation()
@ -26,6 +28,7 @@ const Actions = ({
</Button>
<Button
variant='primary'
disabled={runDisabled}
onClick={onProcess}
>
{t('datasetPipeline.operations.saveAndProcess')}

15
web/app/components/datasets/documents/create-from-pipeline/process-documents/hooks.ts
View File

@ -1,22 +1,12 @@
import { useMemo } from 'react'
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
import { usePublishedPipelineProcessingParams } from '@/service/use-pipeline'
import { PipelineInputVarType } from '@/models/pipeline'
import { VAR_TYPE_MAP } from '@/models/pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
[PipelineInputVarType.textInput]: BaseFieldType.textInput,
[PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
[PipelineInputVarType.select]: BaseFieldType.select,
[PipelineInputVarType.singleFile]: BaseFieldType.file,
[PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
[PipelineInputVarType.number]: BaseFieldType.numberInput,
[PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
}
export const useConfigurations = (datasourceNodeId: string) => {
const pipelineId = useDatasetDetailContextWithSelector(state => state.dataset?.pipeline_id)
const { data: paramsConfig } = usePublishedPipelineProcessingParams({
const { data: paramsConfig, isFetching: isFetchingParams } = usePublishedPipelineProcessingParams({
pipeline_id: pipelineId!,
node_id: datasourceNodeId,
})
@ -61,6 +51,7 @@ export const useConfigurations = (datasourceNodeId: string) => {
}, [paramsConfig])
return {
isFetchingParams,
initialData,
configurations,
}

7
web/app/components/datasets/documents/create-from-pipeline/process-documents/index.tsx
View File

@ -1,3 +1,4 @@
import React from 'react'
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
import { useConfigurations } from './hooks'
import Form from './form'
@ -20,7 +21,7 @@ const ProcessDocuments = ({
onBack,
ref,
}: ProcessDocumentsProps) => {
const { initialData, configurations } = useConfigurations(dataSourceNodeId)
const { isFetchingParams, initialData, configurations } = useConfigurations(dataSourceNodeId)
const schema = generateZodSchema(configurations)
return (
@ -33,9 +34,9 @@ const ProcessDocuments = ({
onSubmit={onSubmit}
onPreview={onPreview}
/>
<Actions onBack={onBack} onProcess={onProcess} />
<Actions runDisabled={isFetchingParams} onBack={onBack} onProcess={onProcess} />
</div>
)
}
export default ProcessDocuments
export default React.memo(ProcessDocuments)

26
web/app/components/datasets/documents/detail/index.tsx
View File

@ -1,7 +1,7 @@
'use client'
import type { FC } from 'react'
import React, { useMemo, useState } from 'react'
import { createContext, useContext, useContextSelector } from 'use-context-selector'
import { createContext, useContextSelector } from 'use-context-selector'
import { useTranslation } from 'react-i18next'
import { useRouter } from 'next/navigation'
import { RiArrowLeftLine, RiLayoutLeft2Line, RiLayoutRight2Line } from '@remixicon/react'
@ -17,9 +17,9 @@ import style from './style.module.css'
import cn from '@/utils/classnames'
import Divider from '@/app/components/base/divider'
import Loading from '@/app/components/base/loading'
import { ToastContext } from '@/app/components/base/toast'
import Toast from '@/app/components/base/toast'
import type { ChunkingMode, ParentMode, ProcessMode } from '@/models/datasets'
import { useDatasetDetailContext } from '@/context/dataset-detail'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import FloatRightContainer from '@/app/components/base/float-right-container'
import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
import { useCheckSegmentBatchImportProgress, useChildSegmentListKey, useSegmentBatchImport, useSegmentListKey } from '@/service/knowledge/use-segment'
@ -83,8 +83,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
const media = useBreakpoints()
const isMobile = media === MediaType.mobile
const { notify } = useContext(ToastContext)
const { dataset } = useDatasetDetailContext()
const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
const embeddingAvailable = !!dataset?.embedding_available
const [showMetadata, setShowMetadata] = useState(!isMobile)
const [newSegmentModalVisible, setNewSegmentModalVisible] = useState(false)
@ -103,10 +102,10 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
if (res.job_status === ProcessStatus.WAITING || res.job_status === ProcessStatus.PROCESSING)
setTimeout(() => checkProcess(res.job_id), 2500)
if (res.job_status === ProcessStatus.ERROR)
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}` })
},
onError: (e) => {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
},
})
}
@ -124,7 +123,7 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
checkProcess(res.job_id)
},
onError: (e) => {
notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
Toast.notify({ type: 'error', message: `${t('datasetDocuments.list.batchModal.runError')}${'message' in e ? `: ${e.message}` : ''}` })
},
})
}
@ -171,12 +170,12 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
}
const mode = useMemo(() => {
return documentDetail?.document_process_rule?.mode
}, [documentDetail?.document_process_rule])
return documentDetail?.document_process_rule?.mode || documentDetail?.dataset_process_rule?.mode
}, [documentDetail?.document_process_rule?.mode, documentDetail?.dataset_process_rule?.mode])
const parentMode = useMemo(() => {
return documentDetail?.document_process_rule?.rules?.parent_mode
}, [documentDetail?.document_process_rule])
return documentDetail?.document_process_rule?.rules?.parent_mode || documentDetail?.dataset_process_rule?.rules?.parent_mode || 'paragraph'
}, [documentDetail?.document_process_rule?.rules?.parent_mode, documentDetail?.dataset_process_rule?.rules?.parent_mode])
const isFullDocMode = useMemo(() => {
return mode === 'hierarchical' && parentMode === 'full-doc'
@ -260,7 +259,8 @@ const DocumentDetail: FC<Props> = ({ datasetId, documentId }) => {
{isDetailLoading
? <Loading type='app' />
: <div className={cn('flex h-full min-w-0 grow flex-col',
embedding ? '' : isFullDocMode ? 'relative pl-11 pr-11 pt-4' : 'relative pl-5 pr-11 pt-3',
!embedding && isFullDocMode && 'relative pl-11 pr-11 pt-4',
!embedding && !isFullDocMode && 'relative pl-5 pr-11 pt-3',
)}>
{embedding
? <Embedding

11
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
View File

@ -54,7 +54,7 @@ const Crawler = ({
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
const { data: paramsConfig } = usePreProcessingParams.current({
const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
pipeline_id: pipelineId!,
node_id: nodeId,
}, !!pipelineId && !!nodeId)
@ -129,6 +129,10 @@ const Crawler = ({
setCrawlErrorMessage('')
}, [runDatasourceNode, nodeId, pipelineId, onCheckedCrawlResultChange, checkCrawlStatus, t])
const handleSubmit = useCallback((value: Record<string, any>) => {
handleRun(value)
}, [handleRun])
return (
<div className='flex flex-col'>
<Header
@ -139,10 +143,9 @@ const Crawler = ({
<Options
variables={paramsConfig?.variables || []}
isRunning={isRunning}
runDisabled={isFetchingParams}
controlFoldOptions={controlFoldOptions}
onSubmit={(value) => {
handleRun(value)
}}
onSubmit={handleSubmit}
/>
</div>
{!isInit && (

40
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/options/hooks.ts
View File

@ -1,21 +1,22 @@
import type { BaseConfiguration, BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
import { PipelineInputVarType, type RAGPipelineVariables } from '@/models/pipeline'
import type { BaseConfiguration } from '@/app/components/base/form/form-scenarios/base/types'
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
import { type RAGPipelineVariables, VAR_TYPE_MAP } from '@/models/pipeline'
import { useMemo } from 'react'
export const useInitialData = (variables: RAGPipelineVariables) => {
const initialData = useMemo(() => {
const initialData: Record<string, any> = {}
variables.forEach((item) => {
if ([PipelineInputVarType.textInput, PipelineInputVarType.paragraph, PipelineInputVarType.select].includes(item.type))
initialData[item.variable] = item.default_value || ''
if (item.type === PipelineInputVarType.number)
initialData[item.variable] = item.default_value || 0
if ([PipelineInputVarType.singleFile, PipelineInputVarType.multiFiles].includes(item.type))
initialData[item.variable] = []
if (item.type === PipelineInputVarType.checkbox)
initialData[item.variable] = item.default_value || true
})
return initialData
return variables.reduce((acc, item) => {
const type = VAR_TYPE_MAP[item.type]
if ([BaseFieldType.textInput, BaseFieldType.paragraph, BaseFieldType.select].includes(type))
acc[item.variable] = item.default_value ?? ''
if (type === BaseFieldType.numberInput)
acc[item.variable] = item.default_value ?? 0
if (type === BaseFieldType.checkbox)
acc[item.variable] = true
if ([BaseFieldType.file, BaseFieldType.fileList].includes(type))
acc[item.variable] = []
return acc
}, {} as Record<string, any>)
}, [variables])
return initialData
@ -26,21 +27,22 @@ export const useConfigurations = (variables: RAGPipelineVariables) => {
const configurations: BaseConfiguration[] = []
variables.forEach((item) => {
configurations.push({
type: item.type as unknown as BaseFieldType,
type: VAR_TYPE_MAP[item.type],
variable: item.variable,
label: item.label,
required: item.required,
placeholder: item.placeholder,
tooltip: item.tooltips,
maxLength: item.max_length,
options: item.options?.map(option => ({
label: option,
value: option,
})),
maxLength: item.max_length,
showConditions: [],
allowedFileUploadMethods: item.allowed_file_upload_methods,
placeholder: item.placeholder,
tooltip: item.tooltips,
unit: item.unit,
allowedFileTypes: item.allowed_file_types,
allowedFileExtensions: item.allowed_file_extensions,
allowedFileUploadMethods: item.allowed_file_upload_methods,
})
})
return configurations

4
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/options/index.tsx
View File

@ -17,6 +17,7 @@ const I18N_PREFIX = 'datasetCreation.stepOne.website'
type OptionsProps = {
variables: RAGPipelineVariables
isRunning: boolean
runDisabled?: boolean
controlFoldOptions?: number
onSubmit: (data: Record<string, any>) => void
}
@ -24,6 +25,7 @@ type OptionsProps = {
const Options = ({
variables,
isRunning,
runDisabled,
controlFoldOptions,
onSubmit,
}: OptionsProps) => {
@ -90,7 +92,7 @@ const Options = ({
<Button
variant='primary'
onClick={form.handleSubmit}
disabled={isRunning}
disabled={runDisabled || isRunning}
loading={isRunning}
className='shrink-0 gap-x-0.5'
spinnerClassName='!ml-0'

4
web/app/components/rag-pipeline/components/panel/test-run/document-processing/actions.tsx
View File

@ -7,11 +7,13 @@ import { WorkflowRunningStatus } from '@/app/components/workflow/types'
type ActionsProps = {
formParams: CustomActionsProps
runDisabled?: boolean
onBack: () => void
}
const Actions = ({
formParams,
runDisabled,
onBack,
}: ActionsProps) => {
const { t } = useTranslation()
@ -32,7 +34,7 @@ const Actions = ({
onClick={() => {
form.handleSubmit()
}}
disabled={isSubmitting || !canSubmit || isRunning}
disabled={runDisabled || isSubmitting || !canSubmit || isRunning}
loading={isSubmitting || isRunning}
>
{t('datasetPipeline.operations.process')}

15
web/app/components/rag-pipeline/components/panel/test-run/document-processing/hooks.ts
View File

@ -3,21 +3,11 @@ import type { BaseConfiguration } from '@/app/components/base/form/form-scenario
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
import { useStore } from '@/app/components/workflow/store'
import { useDraftPipelineProcessingParams } from '@/service/use-pipeline'
import { PipelineInputVarType } from '@/models/pipeline'
const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
[PipelineInputVarType.textInput]: BaseFieldType.textInput,
[PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
[PipelineInputVarType.select]: BaseFieldType.select,
[PipelineInputVarType.singleFile]: BaseFieldType.file,
[PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
[PipelineInputVarType.number]: BaseFieldType.numberInput,
[PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
}
import { VAR_TYPE_MAP } from '@/models/pipeline'
export const useConfigurations = (datasourceNodeId: string) => {
const pipelineId = useStore(state => state.pipelineId)
const { data: paramsConfig } = useDraftPipelineProcessingParams({
const { data: paramsConfig, isFetching: isFetchingParams } = useDraftPipelineProcessingParams({
pipeline_id: pipelineId!,
node_id: datasourceNodeId,
})
@ -62,6 +52,7 @@ export const useConfigurations = (datasourceNodeId: string) => {
}, [paramsConfig])
return {
isFetchingParams,
initialData,
configurations,
}

10
web/app/components/rag-pipeline/components/panel/test-run/document-processing/index.tsx
View File

@ -1,8 +1,8 @@
import React, { useCallback } from 'react'
import { generateZodSchema } from '@/app/components/base/form/form-scenarios/base/utils'
import { useConfigurations } from './hooks'
import Options from './options'
import Actions from './actions'
import { useCallback } from 'react'
import type { CustomActionsProps } from '@/app/components/base/form/components/form/actions'
type DocumentProcessingProps = {
@ -16,12 +16,12 @@ const DocumentProcessing = ({
onProcess,
onBack,
}: DocumentProcessingProps) => {
const { initialData, configurations } = useConfigurations(dataSourceNodeId)
const { isFetchingParams, initialData, configurations } = useConfigurations(dataSourceNodeId)
const schema = generateZodSchema(configurations)
const renderCustomActions = useCallback((props: CustomActionsProps) => (
<Actions formParams={props} onBack={onBack} />
), [onBack])
<Actions runDisabled={isFetchingParams} formParams={props} onBack={onBack} />
), [isFetchingParams, onBack])
return (
<Options
@ -34,4 +34,4 @@ const DocumentProcessing = ({
)
}
export default DocumentProcessing
export default React.memo(DocumentProcessing)

11
web/models/pipeline.ts
View File

@ -5,6 +5,7 @@ import type { Dependency } from '@/app/components/plugins/types'
import type { AppIconSelection } from '@/app/components/base/app-icon-picker'
import type { Viewport } from 'reactflow'
import type { TransferMethod } from '@/types/app'
import { BaseFieldType } from '@/app/components/base/form/form-scenarios/base/types'
export enum DatasourceType {
localFile = 'local_file',
@ -119,6 +120,16 @@ export enum PipelineInputVarType {
checkbox = 'checkbox',
}
export const VAR_TYPE_MAP: Record<PipelineInputVarType, BaseFieldType> = {
[PipelineInputVarType.textInput]: BaseFieldType.textInput,
[PipelineInputVarType.paragraph]: BaseFieldType.paragraph,
[PipelineInputVarType.select]: BaseFieldType.select,
[PipelineInputVarType.singleFile]: BaseFieldType.file,
[PipelineInputVarType.multiFiles]: BaseFieldType.fileList,
[PipelineInputVarType.number]: BaseFieldType.numberInput,
[PipelineInputVarType.checkbox]: BaseFieldType.checkbox,
}
export type RAGPipelineVariable = {
belong_to_node_id: string // indicates belong to which node or 'shared'
type: PipelineInputVarType