opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feat/rag-pipeline' into deploy/rag-dev

This commit is contained in:
twwu 2025-06-06 17:27:14 +08:00
parent 2322496552 3e2f12b065
commit b8813e199f
15 changed files with 73 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
web/app/components/base/form/form-scenarios/input-field/field.tsx
View File

@ -50,6 +50,7 @@ const InputField = ({
return (
<form.AppField
name={variable}
listeners={listeners}
children={field => (
<field.TextField
label={label}

7
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -162,11 +162,16 @@ export const useWebsiteCrawl = () => {
setCurrentWebsite(undefined)
}, [])
const updataCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
setWebsitePages(checkedCrawlResult)
previewWebsitePage.current = checkedCrawlResult[0]
}, [])
return {
websitePages,
websiteCrawlJobId,
previewWebsitePage,
setWebsitePages,
updataCheckedCrawlResultChange,
setWebsiteCrawlJobId,
currentWebsite,
updateCurrentWebsite,

27
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -72,9 +72,9 @@ const CreateFormPipeline = () => {
} = useOnlineDocuments()
const {
websitePages,
websiteCrawlJobId,
// websiteCrawlJobId, // todo: Add status query
previewWebsitePage,
setWebsitePages,
updataCheckedCrawlResultChange,
setWebsiteCrawlJobId,
currentWebsite,
updateCurrentWebsite,
@ -124,13 +124,8 @@ const CreateFormPipeline = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = {
job_id: websiteCrawlJobId,
result: previewWebsitePage.current,
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl)
datasourceInfoList.push(previewWebsitePage.current)
await runPublishedPipeline({
pipeline_id: pipelineId!,
inputs: data,
@ -143,7 +138,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource)
@ -176,11 +171,9 @@ const CreateFormPipeline = () => {
})
}
if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = {
job_id: websiteCrawlJobId,
result: websitePages,
}
datasourceInfoList.push(documentInfo)
websitePages.forEach((websitePage) => {
datasourceInfoList.push(websitePage)
})
}
await runPublishedPipeline({
pipeline_id: pipelineId!,
@ -196,7 +189,7 @@ const CreateFormPipeline = () => {
handleNextStep()
},
})
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websitePages])
const onClickProcess = useCallback(() => {
isPreview.current = false
@ -285,7 +278,7 @@ const CreateFormPipeline = () => {
docLink: datasource.docLink || '',
}}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onCheckedCrawlResultChange={updataCheckedCrawlResultChange}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>

26
web/app/components/rag-pipeline/components/input-field/editor/form/hooks.ts
View File

@ -64,11 +64,12 @@ export const useHiddenFieldNames = (type: PipelineInputVarType) => {
}
export const useConfigurations = (props: {
getFieldValue: (fieldName: DeepKeys<FormData>) => any,
setFieldValue: (fieldName: DeepKeys<FormData>, value: any) => void,
supportFile: boolean
}) => {
const { t } = useTranslation()
const { setFieldValue, supportFile } = props
const { getFieldValue, setFieldValue, supportFile } = props
const handleTypeChange = useCallback((type: PipelineInputVarType) => {
if ([PipelineInputVarType.singleFile, PipelineInputVarType.multiFiles].includes(type)) {
@ -84,6 +85,17 @@ export const useConfigurations = (props: {
setFieldValue('maxLength', DEFAULT_VALUE_MAX_LEN)
}, [setFieldValue])
const handleVariableNameBlur = useCallback((value: string) => {
if (!value)
return
setFieldValue('label', value)
}, [setFieldValue])
const handleDisplayNameBlur = useCallback((value: string) => {
if (!value)
setFieldValue('label', getFieldValue('variable'))
}, [getFieldValue, setFieldValue])
const initialConfigurations = useMemo((): InputFieldConfiguration[] => {
return [{
type: InputFieldType.inputTypeSelect,
@ -101,13 +113,19 @@ export const useConfigurations = (props: {
variable: 'variable',
placeholder: t('appDebug.variableConfig.inputPlaceholder'),
required: true,
listeners: {
onBlur: ({ value }) => handleVariableNameBlur(value),
},
showConditions: [],
}, {
type: InputFieldType.textInput,
label: t('appDebug.variableConfig.labelName'),
label: t('appDebug.variableConfig.displayName'),
variable: 'label',
placeholder: t('appDebug.variableConfig.inputPlaceholder'),
required: true,
required: false,
listeners: {
onBlur: ({ value }) => handleDisplayNameBlur(value),
},
showConditions: [],
}, {
type: InputFieldType.numberInput,
@ -155,7 +173,7 @@ export const useConfigurations = (props: {
required: true,
showConditions: [],
}]
}, [handleTypeChange, supportFile, t])
}, [t, supportFile, handleTypeChange, handleVariableNameBlur, handleDisplayNameBlur])
return initialConfigurations
}

5
web/app/components/rag-pipeline/components/input-field/editor/form/initial-fields.tsx
View File

@ -16,11 +16,16 @@ const InitialFields = ({
render: function Render({
form,
}) {
const getFieldValue = useCallback((fieldName: string) => {
return form.getFieldValue(fieldName)
}, [form])
const setFieldValue = useCallback((fieldName: string, value: any) => {
form.setFieldValue(fieldName, value)
}, [form])
const initialConfigurations = useConfigurations({
getFieldValue,
setFieldValue,
supportFile,
})

3
web/app/components/rag-pipeline/components/input-field/editor/index.tsx
View File

@ -27,8 +27,7 @@ const InputFieldEditor = ({
const handleSubmit = useCallback((value: FormData, moreInfo?: MoreInfo) => {
const inputFieldData = convertFormDataToINputField(value)
onSubmit(inputFieldData, moreInfo)
onClose()
}, [onSubmit, onClose])
}, [onSubmit])
return (
<DialogWrapper

18
web/app/components/rag-pipeline/components/input-field/field-list/hooks.ts
View File

@ -10,6 +10,7 @@ import type { MoreInfo, ValueSelector } from '@/app/components/workflow/types'
import { ChangeType } from '@/app/components/workflow/types'
import { useWorkflow } from '@/app/components/workflow/hooks'
import { useBoolean } from 'ahooks'
import Toast from '@/app/components/base/toast'
export const useFieldList = (
initialInputFields: InputVar[],
@ -51,8 +52,9 @@ export const useFieldList = (
setEditingField(inputFieldsRef.current[index])
setShowInputFieldEditor(true)
}, [])
const handleCancelInputFieldEditor = useCallback(() => {
const handleCloseInputFieldEditor = useCallback(() => {
setShowInputFieldEditor(false)
editingFieldIndex.current = -1
setEditingField(undefined)
}, [])
@ -77,6 +79,15 @@ export const useFieldList = (
}, [removedIndex, handleInputFieldsChange, removeUsedVarInNodes, removedVar, hideRemoveVarConfirm])
const handleSubmitField = useCallback((data: InputVar, moreInfo?: MoreInfo) => {
const isDuplicate = inputFieldsRef.current.some(field =>
field.variable === data.variable && field.variable !== editingField?.variable)
if (isDuplicate) {
Toast.notify({
type: 'error',
message: 'Variable name already exists. Please choose a different name.',
})
return
}
const newInputFields = produce(inputFieldsRef.current, (draft) => {
const currentIndex = editingFieldIndex.current
if (currentIndex === -1) {
@ -89,7 +100,8 @@ export const useFieldList = (
// Update variable name in nodes if it has changed
if (moreInfo?.type === ChangeType.changeVarName)
handleOutVarRenameChange(nodeId, [nodeId, moreInfo.payload?.beforeKey || ''], [nodeId, moreInfo.payload?.afterKey || ''])
}, [handleInputFieldsChange, handleOutVarRenameChange, nodeId])
handleCloseInputFieldEditor()
}, [editingField?.variable, handleCloseInputFieldEditor, handleInputFieldsChange, handleOutVarRenameChange, nodeId])
return {
inputFields,
@ -100,7 +112,7 @@ export const useFieldList = (
editingField,
showInputFieldEditor,
handleOpenInputFieldEditor,
handleCancelInputFieldEditor,
handleCloseInputFieldEditor,
isShowRemoveVarConfirm,
hideRemoveVarConfirm,
onRemoveVarConfirm,

4
web/app/components/rag-pipeline/components/input-field/field-list/index.tsx
View File

@ -34,7 +34,7 @@ const FieldList = ({
handleSubmitField,
handleListSortChange,
handleRemoveField,
handleCancelInputFieldEditor,
handleCloseInputFieldEditor,
handleOpenInputFieldEditor,
showInputFieldEditor,
editingField,
@ -69,7 +69,7 @@ const FieldList = ({
show={showInputFieldEditor}
initialData={editingField}
onSubmit={handleSubmitField}
onClose={handleCancelInputFieldEditor}
onClose={handleCloseInputFieldEditor}
/>
)}
<RemoveEffectVarConfirm

2
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawled-result.tsx
View File

@ -62,7 +62,7 @@ const CrawledResult = ({
time: usedTime.toFixed(1),
})}
</div>
<div className='rounded-xl border border-components-panel-border bg-components-panel-bg'>
<div className='overflow-hidden rounded-xl border border-components-panel-border bg-components-panel-bg'>
<div className='flex items-center px-4 py-2'>
<CheckboxWithLabel
isChecked={isCheckAll}

4
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx
View File

@ -11,7 +11,7 @@ import {
useDraftDatasourceNodeRun,
useDraftPipelinePreProcessingParams,
usePublishedDatasourceNodeRun,
usePublishedPipelineProcessingParams,
usePublishedPipelinePreProcessingParams,
} from '@/service/use-pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { DatasourceType } from '@/models/pipeline'
@ -52,7 +52,7 @@ const Crawler = ({
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelineProcessingParams : useDraftPipelinePreProcessingParams)
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
const { data: paramsConfig } = usePreProcessingParams.current({
pipeline_id: pipelineId!,
node_id: nodeId,

1
web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/options/index.tsx
View File

@ -90,6 +90,7 @@ const Options = ({
<Button
variant='primary'
onClick={form.handleSubmit}
disabled={isRunning}
loading={isRunning}
className='shrink-0 gap-x-0.5'
spinnerClassName='!ml-0'

13
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -40,7 +40,7 @@ const TestRunPanel = () => {
} = useOnlineDocuments()
const {
websitePages,
websiteCrawlJobId,
// websiteCrawlJobId, // todo: Add status query
setWebsitePages,
setWebsiteCrawlJobId,
} = useWebsiteCrawl()
@ -90,20 +90,15 @@ const TestRunPanel = () => {
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl) {
const documentInfo = {
job_id: websiteCrawlJobId,
result: [websitePages[0]],
}
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.websiteCrawl)
datasourceInfoList.push(websitePages[0])
handleRun({
inputs: data,
start_node_id: datasource.nodeId,
datasource_type: datasource.type,
datasource_info_list: datasourceInfoList,
})
}, [datasource, fileList, handleRun, onlineDocuments, websiteCrawlJobId, websitePages])
}, [datasource, fileList, handleRun, onlineDocuments, websitePages])
return (
<div

1
web/i18n/en-US/app-debug.ts
View File

@ -365,6 +365,7 @@ const translation = {
'apiBasedVar': 'API-based Variable',
'varName': 'Variable Name',
'labelName': 'Label Name',
'displayName': 'Display Name',
'inputPlaceholder': 'Please input',
'content': 'Content',
'required': 'Required',

1
web/i18n/zh-Hans/app-debug.ts
View File

@ -360,6 +360,7 @@ const translation = {
'varName': '变量名称',
'inputPlaceholder': '请输入',
'labelName': '显示名称',
'displayName': '显示名称',
'required': '必填',
'hide': '隐藏',
'placeholder': '占位符',

2
web/service/use-pipeline.ts
View File

@ -288,7 +288,7 @@ export const usePublishedPipelinePreProcessingParams = (params: PipelinePreProce
return useQuery<PipelinePreProcessingParamsResponse>({
queryKey: [NAME_SPACE, 'published-pipeline-pre-processing-params', pipeline_id, node_id],
queryFn: () => {
return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/processing/parameters`, {
return get<PipelinePreProcessingParamsResponse>(`/rag/pipelines/${pipeline_id}/workflows/published/pre-processing/parameters`, {
params: {
node_id,
},