mirror of https://github.com/langgenius/dify.git
refactor: Refactor OnlineDocuments component and remove OnlineDocumentSelector
This commit is contained in:
twwu
parent
5248fcca56
commit
93eabef58a
|
|
@ -249,10 +249,11 @@ const CreateFormPipeline = () => {
|
|||
/>
|
||||
{datasourceType === DatasourceType.localFile && (
|
||||
<LocalFile
|
||||
files={fileList}
|
||||
fileList={fileList}
|
||||
allowedExtensions={datasource!.nodeData.fileExtensions || []}
|
||||
updateFile={updateFile}
|
||||
updateFileList={updateFileList}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
onPreview={updateCurrentFile}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
/>
|
||||
|
|
@ -261,8 +262,8 @@ const CreateFormPipeline = () => {
|
|||
<OnlineDocuments
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
onlineDocuments={onlineDocuments}
|
||||
updateOnlineDocuments={updateOnlineDocuments}
|
||||
pageIdList={onlineDocuments.map(doc => doc.page_id)}
|
||||
onSelect={updateOnlineDocuments}
|
||||
canPreview
|
||||
onPreview={updateCurrentPage}
|
||||
/>
|
||||
|
|
|
|||
|
|
@ -1,338 +0,0 @@
|
|||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import SimplePieChart from '@/app/components/base/simple-pie-chart'
|
||||
import { upload } from '@/service/base'
|
||||
import I18n from '@/context/i18n'
|
||||
import { LanguagesSupported } from '@/i18n/language'
|
||||
import { IS_CE_EDITION } from '@/config'
|
||||
import { Theme } from '@/types/app'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { useFileUploadConfig } from '@/service/use-common'
|
||||
|
||||
const FILES_NUMBER_LIMIT = 20
|
||||
|
||||
export type FileUploaderProps = {
|
||||
fileList: FileItem[]
|
||||
allowedExtensions: string[]
|
||||
prepareFileList: (files: FileItem[]) => void
|
||||
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
onFileListUpdate?: (files: FileItem[]) => void
|
||||
onPreview?: (file: File) => void
|
||||
notSupportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const FileUploader = ({
|
||||
fileList,
|
||||
allowedExtensions,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
onPreview,
|
||||
notSupportBatchUpload,
|
||||
}: FileUploaderProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { locale } = useContext(I18n)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
const dropRef = useRef<HTMLDivElement>(null)
|
||||
const dragRef = useRef<HTMLDivElement>(null)
|
||||
const fileUploader = useRef<HTMLInputElement>(null)
|
||||
const hideUpload = notSupportBatchUpload && fileList.length > 0
|
||||
|
||||
const { data: fileUploadConfigResponse } = useFileUploadConfig()
|
||||
const supportTypesShowNames = useMemo(() => {
|
||||
const extensionMap: { [key: string]: string } = {
|
||||
md: 'markdown',
|
||||
pptx: 'pptx',
|
||||
htm: 'html',
|
||||
xlsx: 'xlsx',
|
||||
docx: 'docx',
|
||||
}
|
||||
|
||||
return allowedExtensions
|
||||
.map(item => extensionMap[item] || item) // map to standardized extension
|
||||
.map(item => item.toLowerCase()) // convert to lower case
|
||||
.filter((item, index, self) => self.indexOf(item) === index) // remove duplicates
|
||||
.map(item => item.toUpperCase()) // convert to upper case
|
||||
.join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
|
||||
}, [locale, allowedExtensions])
|
||||
const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`)
|
||||
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
|
||||
file_size_limit: 15,
|
||||
batch_count_limit: 5,
|
||||
}, [fileUploadConfigResponse])
|
||||
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
|
||||
// utils
|
||||
const getFileType = (currentFile: File) => {
|
||||
if (!currentFile)
|
||||
return ''
|
||||
|
||||
const arr = currentFile.name.split('.')
|
||||
return arr[arr.length - 1]
|
||||
}
|
||||
|
||||
const getFileSize = (size: number) => {
|
||||
if (size / 1024 < 10)
|
||||
return `${(size / 1024).toFixed(2)}KB`
|
||||
|
||||
return `${(size / 1024 / 1024).toFixed(2)}MB`
|
||||
}
|
||||
|
||||
const isValid = useCallback((file: File) => {
|
||||
const { size } = file
|
||||
const ext = `.${getFileType(file)}`
|
||||
const isValidType = ACCEPTS.includes(ext.toLowerCase())
|
||||
if (!isValidType)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') })
|
||||
|
||||
const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024
|
||||
if (!isValidSize)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) })
|
||||
|
||||
return isValidType && isValidSize
|
||||
}, [fileUploadConfig, notify, t, ACCEPTS])
|
||||
|
||||
const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', fileItem.file)
|
||||
const onProgress = (e: ProgressEvent) => {
|
||||
if (e.lengthComputable) {
|
||||
const percent = Math.floor(e.loaded / e.total * 100)
|
||||
onFileUpdate(fileItem, percent, fileListRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
return upload({
|
||||
xhr: new XMLHttpRequest(),
|
||||
data: formData,
|
||||
onprogress: onProgress,
|
||||
}, false, undefined, '?source=datasets')
|
||||
.then((res: File) => {
|
||||
const completeFile = {
|
||||
fileID: fileItem.fileID,
|
||||
file: res,
|
||||
progress: -1,
|
||||
}
|
||||
const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
|
||||
fileListRef.current[index] = completeFile
|
||||
onFileUpdate(completeFile, 100, fileListRef.current)
|
||||
return Promise.resolve({ ...completeFile })
|
||||
})
|
||||
.catch((e) => {
|
||||
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
|
||||
onFileUpdate(fileItem, -2, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [fileListRef, notify, onFileUpdate, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
return Promise.all(bFiles.map(fileUpload))
|
||||
}, [fileUpload])
|
||||
|
||||
const uploadMultipleFiles = useCallback(async (files: FileItem[]) => {
|
||||
const batchCountLimit = fileUploadConfig.batch_count_limit
|
||||
const length = files.length
|
||||
let start = 0
|
||||
let end = 0
|
||||
|
||||
while (start < length) {
|
||||
if (start + batchCountLimit > length)
|
||||
end = length
|
||||
else
|
||||
end = start + batchCountLimit
|
||||
const bFiles = files.slice(start, end)
|
||||
await uploadBatchFiles(bFiles)
|
||||
start = end
|
||||
}
|
||||
}, [fileUploadConfig, uploadBatchFiles])
|
||||
|
||||
const initialUpload = useCallback((files: File[]) => {
|
||||
if (!files.length)
|
||||
return false
|
||||
|
||||
if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
|
||||
return false
|
||||
}
|
||||
|
||||
const preparedFiles = files.map((file, index) => ({
|
||||
fileID: `file${index}-${Date.now()}`,
|
||||
file,
|
||||
progress: -1,
|
||||
}))
|
||||
const newFiles = [...fileListRef.current, ...preparedFiles]
|
||||
prepareFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
|
||||
const handleDragEnter = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target !== dragRef.current && setDragging(true)
|
||||
}
|
||||
const handleDragOver = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
}
|
||||
const handleDragLeave = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target === dragRef.current && setDragging(false)
|
||||
}
|
||||
|
||||
const handleDrop = useCallback((e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
setDragging(false)
|
||||
if (!e.dataTransfer)
|
||||
return
|
||||
|
||||
let files = [...e.dataTransfer.files] as File[]
|
||||
if (notSupportBatchUpload)
|
||||
files = files.slice(0, 1)
|
||||
|
||||
const validFiles = files.filter(isValid)
|
||||
initialUpload(validFiles)
|
||||
}, [initialUpload, isValid, notSupportBatchUpload])
|
||||
|
||||
const selectHandle = () => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.click()
|
||||
}
|
||||
|
||||
const removeFile = (fileID: string) => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.value = ''
|
||||
|
||||
fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
|
||||
onFileListUpdate?.([...fileListRef.current])
|
||||
}
|
||||
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = [...(e.target.files ?? [])] as File[]
|
||||
initialUpload(files.filter(isValid))
|
||||
}, [isValid, initialUpload])
|
||||
|
||||
const { theme } = useTheme()
|
||||
const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme])
|
||||
|
||||
useEffect(() => {
|
||||
const dropElement = dropRef.current
|
||||
dropElement?.addEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.addEventListener('dragover', handleDragOver)
|
||||
dropElement?.addEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.addEventListener('drop', handleDrop)
|
||||
return () => {
|
||||
dropElement?.removeEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.removeEventListener('dragover', handleDragOver)
|
||||
dropElement?.removeEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.removeEventListener('drop', handleDrop)
|
||||
}
|
||||
}, [handleDrop])
|
||||
|
||||
return (
|
||||
<div className='flex flex-col'>
|
||||
{!hideUpload && (
|
||||
<input
|
||||
ref={fileUploader}
|
||||
id='fileUploader'
|
||||
className='hidden'
|
||||
type='file'
|
||||
multiple={!notSupportBatchUpload}
|
||||
accept={ACCEPTS.join(',')}
|
||||
onChange={fileChangeHandle}
|
||||
/>
|
||||
)}
|
||||
{!hideUpload && (
|
||||
<div
|
||||
ref={dropRef}
|
||||
className={cn(
|
||||
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs leading-4 text-text-tertiary',
|
||||
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
|
||||
)}>
|
||||
<div className='flex min-h-5 items-center justify-center text-sm leading-4 text-text-secondary'>
|
||||
<RiUploadCloud2Line className='mr-2 size-5' />
|
||||
|
||||
<span>
|
||||
{t('datasetCreation.stepOne.uploader.button')}
|
||||
{allowedExtensions.length > 0 && (
|
||||
<label className='ml-1 cursor-pointer text-text-accent' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<div>{t('datasetCreation.stepOne.uploader.tip', {
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
batchCount: fileUploadConfig.batch_count_limit,
|
||||
})}</div>
|
||||
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
|
||||
</div>
|
||||
)}
|
||||
{fileList.length > 0 && (
|
||||
<div className='mt-1 flex flex-col gap-y-1'>
|
||||
{fileList.map((fileItem, index) => {
|
||||
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
|
||||
const isError = fileItem.progress === -2
|
||||
return (
|
||||
<div
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
)}
|
||||
>
|
||||
<div className='flex w-12 shrink-0 items-center justify-center'>
|
||||
<DocumentFileIcon
|
||||
className='size-6 shrink-0'
|
||||
name={fileItem.file.name}
|
||||
extension={getFileType(fileItem.file)}
|
||||
/>
|
||||
</div>
|
||||
<div className='flex shrink grow flex-col gap-0.5'>
|
||||
<div className='flex w-full'>
|
||||
<div className='w-0 grow truncate text-xs text-text-secondary'>{fileItem.file.name}</div>
|
||||
</div>
|
||||
<div className='w-full truncate text-2xs leading-3 text-text-tertiary'>
|
||||
<span className='uppercase'>{getFileType(fileItem.file)}</span>
|
||||
<span className='px-1 text-text-quaternary'>·</span>
|
||||
<span>{getFileSize(fileItem.file.size)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className='flex w-16 shrink-0 items-center justify-end gap-1 pr-3'>
|
||||
{isUploading && (
|
||||
<SimplePieChart percentage={fileItem.progress} stroke={chartColor} fill={chartColor} animationDuration={0} />
|
||||
)}
|
||||
{
|
||||
isError && (
|
||||
<RiErrorWarningFill className='size-4 text-text-destructive' />
|
||||
)
|
||||
}
|
||||
<span className='flex h-6 w-6 cursor-pointer items-center justify-center' onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
removeFile(fileItem.fileID)
|
||||
}}>
|
||||
<RiDeleteBinLine className='size-4 text-text-tertiary' />
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default FileUploader
|
||||
|
|
@ -1,33 +1,337 @@
|
|||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import { useContext } from 'use-context-selector'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import FileUploader from './file-uploader'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import SimplePieChart from '@/app/components/base/simple-pie-chart'
|
||||
import { upload } from '@/service/base'
|
||||
import I18n from '@/context/i18n'
|
||||
import { LanguagesSupported } from '@/i18n/language'
|
||||
import { IS_CE_EDITION } from '@/config'
|
||||
import { Theme } from '@/types/app'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { useFileUploadConfig } from '@/service/use-common'
|
||||
|
||||
type LocalFileProps = {
|
||||
files: FileItem[]
|
||||
const FILES_NUMBER_LIMIT = 20
|
||||
|
||||
export type LocalFileProps = {
|
||||
fileList: FileItem[]
|
||||
allowedExtensions: string[]
|
||||
updateFileList: (files: FileItem[]) => void
|
||||
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
prepareFileList: (files: FileItem[]) => void
|
||||
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
onFileListUpdate?: (files: FileItem[]) => void
|
||||
onPreview?: (file: File) => void
|
||||
notSupportBatchUpload: boolean
|
||||
notSupportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const LocalFile = ({
|
||||
files,
|
||||
fileList,
|
||||
allowedExtensions,
|
||||
updateFileList,
|
||||
updateFile,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
onPreview,
|
||||
notSupportBatchUpload,
|
||||
}: LocalFileProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { locale } = useContext(I18n)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
const dropRef = useRef<HTMLDivElement>(null)
|
||||
const dragRef = useRef<HTMLDivElement>(null)
|
||||
const fileUploader = useRef<HTMLInputElement>(null)
|
||||
const hideUpload = notSupportBatchUpload && fileList.length > 0
|
||||
|
||||
const { data: fileUploadConfigResponse } = useFileUploadConfig()
|
||||
const supportTypesShowNames = useMemo(() => {
|
||||
const extensionMap: { [key: string]: string } = {
|
||||
md: 'markdown',
|
||||
pptx: 'pptx',
|
||||
htm: 'html',
|
||||
xlsx: 'xlsx',
|
||||
docx: 'docx',
|
||||
}
|
||||
|
||||
return allowedExtensions
|
||||
.map(item => extensionMap[item] || item) // map to standardized extension
|
||||
.map(item => item.toLowerCase()) // convert to lower case
|
||||
.filter((item, index, self) => self.indexOf(item) === index) // remove duplicates
|
||||
.map(item => item.toUpperCase()) // convert to upper case
|
||||
.join(locale !== LanguagesSupported[1] ? ', ' : '、 ')
|
||||
}, [locale, allowedExtensions])
|
||||
const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`)
|
||||
const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? {
|
||||
file_size_limit: 15,
|
||||
batch_count_limit: 5,
|
||||
}, [fileUploadConfigResponse])
|
||||
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
|
||||
// utils
|
||||
const getFileType = (currentFile: File) => {
|
||||
if (!currentFile)
|
||||
return ''
|
||||
|
||||
const arr = currentFile.name.split('.')
|
||||
return arr[arr.length - 1]
|
||||
}
|
||||
|
||||
const getFileSize = (size: number) => {
|
||||
if (size / 1024 < 10)
|
||||
return `${(size / 1024).toFixed(2)}KB`
|
||||
|
||||
return `${(size / 1024 / 1024).toFixed(2)}MB`
|
||||
}
|
||||
|
||||
const isValid = useCallback((file: File) => {
|
||||
const { size } = file
|
||||
const ext = `.${getFileType(file)}`
|
||||
const isValidType = ACCEPTS.includes(ext.toLowerCase())
|
||||
if (!isValidType)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') })
|
||||
|
||||
const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024
|
||||
if (!isValidSize)
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) })
|
||||
|
||||
return isValidType && isValidSize
|
||||
}, [fileUploadConfig, notify, t, ACCEPTS])
|
||||
|
||||
const fileUpload = useCallback(async (fileItem: FileItem): Promise<FileItem> => {
|
||||
const formData = new FormData()
|
||||
formData.append('file', fileItem.file)
|
||||
const onProgress = (e: ProgressEvent) => {
|
||||
if (e.lengthComputable) {
|
||||
const percent = Math.floor(e.loaded / e.total * 100)
|
||||
onFileUpdate(fileItem, percent, fileListRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
return upload({
|
||||
xhr: new XMLHttpRequest(),
|
||||
data: formData,
|
||||
onprogress: onProgress,
|
||||
}, false, undefined, '?source=datasets')
|
||||
.then((res: File) => {
|
||||
const completeFile = {
|
||||
fileID: fileItem.fileID,
|
||||
file: res,
|
||||
progress: -1,
|
||||
}
|
||||
const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
|
||||
fileListRef.current[index] = completeFile
|
||||
onFileUpdate(completeFile, 100, fileListRef.current)
|
||||
return Promise.resolve({ ...completeFile })
|
||||
})
|
||||
.catch((e) => {
|
||||
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
|
||||
onFileUpdate(fileItem, -2, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [fileListRef, notify, onFileUpdate, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
return Promise.all(bFiles.map(fileUpload))
|
||||
}, [fileUpload])
|
||||
|
||||
const uploadMultipleFiles = useCallback(async (files: FileItem[]) => {
|
||||
const batchCountLimit = fileUploadConfig.batch_count_limit
|
||||
const length = files.length
|
||||
let start = 0
|
||||
let end = 0
|
||||
|
||||
while (start < length) {
|
||||
if (start + batchCountLimit > length)
|
||||
end = length
|
||||
else
|
||||
end = start + batchCountLimit
|
||||
const bFiles = files.slice(start, end)
|
||||
await uploadBatchFiles(bFiles)
|
||||
start = end
|
||||
}
|
||||
}, [fileUploadConfig, uploadBatchFiles])
|
||||
|
||||
const initialUpload = useCallback((files: File[]) => {
|
||||
if (!files.length)
|
||||
return false
|
||||
|
||||
if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) {
|
||||
notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) })
|
||||
return false
|
||||
}
|
||||
|
||||
const preparedFiles = files.map((file, index) => ({
|
||||
fileID: `file${index}-${Date.now()}`,
|
||||
file,
|
||||
progress: -1,
|
||||
}))
|
||||
const newFiles = [...fileListRef.current, ...preparedFiles]
|
||||
prepareFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
|
||||
const handleDragEnter = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target !== dragRef.current && setDragging(true)
|
||||
}
|
||||
const handleDragOver = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
}
|
||||
const handleDragLeave = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
e.target === dragRef.current && setDragging(false)
|
||||
}
|
||||
|
||||
const handleDrop = useCallback((e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
e.stopPropagation()
|
||||
setDragging(false)
|
||||
if (!e.dataTransfer)
|
||||
return
|
||||
|
||||
let files = [...e.dataTransfer.files] as File[]
|
||||
if (notSupportBatchUpload)
|
||||
files = files.slice(0, 1)
|
||||
|
||||
const validFiles = files.filter(isValid)
|
||||
initialUpload(validFiles)
|
||||
}, [initialUpload, isValid, notSupportBatchUpload])
|
||||
|
||||
const selectHandle = () => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.click()
|
||||
}
|
||||
|
||||
const removeFile = (fileID: string) => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.value = ''
|
||||
|
||||
fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
|
||||
onFileListUpdate?.([...fileListRef.current])
|
||||
}
|
||||
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = [...(e.target.files ?? [])] as File[]
|
||||
initialUpload(files.filter(isValid))
|
||||
}, [isValid, initialUpload])
|
||||
|
||||
const { theme } = useTheme()
|
||||
const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme])
|
||||
|
||||
useEffect(() => {
|
||||
const dropElement = dropRef.current
|
||||
dropElement?.addEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.addEventListener('dragover', handleDragOver)
|
||||
dropElement?.addEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.addEventListener('drop', handleDrop)
|
||||
return () => {
|
||||
dropElement?.removeEventListener('dragenter', handleDragEnter)
|
||||
dropElement?.removeEventListener('dragover', handleDragOver)
|
||||
dropElement?.removeEventListener('dragleave', handleDragLeave)
|
||||
dropElement?.removeEventListener('drop', handleDrop)
|
||||
}
|
||||
}, [handleDrop])
|
||||
|
||||
return (
|
||||
<FileUploader
|
||||
fileList={files}
|
||||
allowedExtensions={allowedExtensions}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
onPreview={onPreview}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
/>
|
||||
<div className='flex flex-col'>
|
||||
{!hideUpload && (
|
||||
<input
|
||||
ref={fileUploader}
|
||||
id='fileUploader'
|
||||
className='hidden'
|
||||
type='file'
|
||||
multiple={!notSupportBatchUpload}
|
||||
accept={ACCEPTS.join(',')}
|
||||
onChange={fileChangeHandle}
|
||||
/>
|
||||
)}
|
||||
{!hideUpload && (
|
||||
<div
|
||||
ref={dropRef}
|
||||
className={cn(
|
||||
'relative box-border flex min-h-20 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-components-dropzone-border bg-components-dropzone-bg px-4 py-3 text-xs leading-4 text-text-tertiary',
|
||||
dragging && 'border-components-dropzone-border-accent bg-components-dropzone-bg-accent',
|
||||
)}>
|
||||
<div className='flex min-h-5 items-center justify-center text-sm leading-4 text-text-secondary'>
|
||||
<RiUploadCloud2Line className='mr-2 size-5' />
|
||||
|
||||
<span>
|
||||
{t('datasetCreation.stepOne.uploader.button')}
|
||||
{allowedExtensions.length > 0 && (
|
||||
<label className='ml-1 cursor-pointer text-text-accent' onClick={selectHandle}>{t('datasetCreation.stepOne.uploader.browse')}</label>
|
||||
)}
|
||||
</span>
|
||||
</div>
|
||||
<div>{t('datasetCreation.stepOne.uploader.tip', {
|
||||
size: fileUploadConfig.file_size_limit,
|
||||
supportTypes: supportTypesShowNames,
|
||||
batchCount: fileUploadConfig.batch_count_limit,
|
||||
})}</div>
|
||||
{dragging && <div ref={dragRef} className='absolute left-0 top-0 h-full w-full' />}
|
||||
</div>
|
||||
)}
|
||||
{fileList.length > 0 && (
|
||||
<div className='mt-1 flex flex-col gap-y-1'>
|
||||
{fileList.map((fileItem, index) => {
|
||||
const isUploading = fileItem.progress >= 0 && fileItem.progress < 100
|
||||
const isError = fileItem.progress === -2
|
||||
return (
|
||||
<div
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
)}
|
||||
>
|
||||
<div className='flex w-12 shrink-0 items-center justify-center'>
|
||||
<DocumentFileIcon
|
||||
className='size-6 shrink-0'
|
||||
name={fileItem.file.name}
|
||||
extension={getFileType(fileItem.file)}
|
||||
/>
|
||||
</div>
|
||||
<div className='flex shrink grow flex-col gap-0.5'>
|
||||
<div className='flex w-full'>
|
||||
<div className='w-0 grow truncate text-xs text-text-secondary'>{fileItem.file.name}</div>
|
||||
</div>
|
||||
<div className='w-full truncate text-2xs leading-3 text-text-tertiary'>
|
||||
<span className='uppercase'>{getFileType(fileItem.file)}</span>
|
||||
<span className='px-1 text-text-quaternary'>·</span>
|
||||
<span>{getFileSize(fileItem.file.size)}</span>
|
||||
</div>
|
||||
</div>
|
||||
<div className='flex w-16 shrink-0 items-center justify-end gap-1 pr-3'>
|
||||
{isUploading && (
|
||||
<SimplePieChart percentage={fileItem.progress} stroke={chartColor} fill={chartColor} animationDuration={0} />
|
||||
)}
|
||||
{
|
||||
isError && (
|
||||
<RiErrorWarningFill className='size-4 text-text-destructive' />
|
||||
)
|
||||
}
|
||||
<span className='flex h-6 w-6 cursor-pointer items-center justify-center' onClick={(e) => {
|
||||
e.stopPropagation()
|
||||
removeFile(fileItem.fileID)
|
||||
}}>
|
||||
<RiDeleteBinLine className='size-4 text-text-tertiary' />
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,36 +1,166 @@
|
|||
import type { NotionPage } from '@/models/common'
|
||||
import OnlineDocumentSelector from './online-document-selector'
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
|
||||
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
|
||||
import PageSelector from '@/app/components/base/notion-page-selector/page-selector'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
type OnlineDocumentsProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
onlineDocuments: NotionPage[]
|
||||
updateOnlineDocuments: (value: NotionPage[]) => void
|
||||
pageIdList?: string[]
|
||||
onSelect: (selectedPages: NotionPage[]) => void
|
||||
canPreview?: boolean
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPage: NotionPage) => void
|
||||
isInPipeline?: boolean
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
}
|
||||
|
||||
const OnlineDocuments = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
onlineDocuments,
|
||||
updateOnlineDocuments,
|
||||
canPreview = false,
|
||||
pageIdList,
|
||||
onSelect,
|
||||
canPreview,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
nodeId,
|
||||
nodeData,
|
||||
}: OnlineDocumentsProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDocuments = useCallback(async () => {
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: {},
|
||||
datasource_type: DatasourceType.onlineDocument,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
|
||||
},
|
||||
onError: (message: string) => {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message,
|
||||
})
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL])
|
||||
|
||||
useEffect(() => {
|
||||
getOnlineDocuments()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const firstWorkspaceId = documentsData[0]?.workspace_id
|
||||
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
|
||||
|
||||
const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
|
||||
const selectedPagesId = new Set<string>()
|
||||
const boundPagesId = new Set<string>()
|
||||
const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
if (page.is_bound) {
|
||||
selectedPagesId.add(page.page_id)
|
||||
boundPagesId.add(page.page_id)
|
||||
}
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
}, {})
|
||||
return [pagesMap, selectedPagesId, boundPagesId]
|
||||
}, [documentsData])
|
||||
const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(pageIdList || [])]
|
||||
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
|
||||
|
||||
const handleSearchValueChange = useCallback((value: string) => {
|
||||
setSearchValue(value)
|
||||
}, [])
|
||||
const handleSelectWorkspace = useCallback((workspaceId: string) => {
|
||||
setCurrentWorkspaceId(workspaceId)
|
||||
}, [])
|
||||
const handleSelectPages = (newSelectedPagesId: Set<string>) => {
|
||||
const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId])
|
||||
|
||||
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
|
||||
onSelect(selectedPages)
|
||||
}
|
||||
const handlePreviewPage = (previewPageId: string) => {
|
||||
if (onPreview)
|
||||
onPreview(PagesMapAndSelectedPagesId[0][previewPageId])
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
setCurrentWorkspaceId(firstWorkspaceId)
|
||||
}, [firstWorkspaceId])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
if (!documentsData?.length)
|
||||
return null
|
||||
|
||||
return (
|
||||
<OnlineDocumentSelector
|
||||
nodeId={nodeId}
|
||||
nodeData={nodeData}
|
||||
value={onlineDocuments.map(page => page.page_id)}
|
||||
onSelect={updateOnlineDocuments}
|
||||
canPreview={canPreview}
|
||||
onPreview={onPreview}
|
||||
isInPipeline={isInPipeline}
|
||||
/>
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
isInPipeline={isInPipeline}
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
|
||||
<div className='flex grow items-center gap-x-1'>
|
||||
<WorkspaceSelector
|
||||
value={currentWorkspaceId || firstWorkspaceId}
|
||||
items={documentsData}
|
||||
onSelect={handleSelectWorkspace}
|
||||
/>
|
||||
</div>
|
||||
<SearchInput
|
||||
value={searchValue}
|
||||
onChange={handleSearchValueChange}
|
||||
/>
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-b-xl'>
|
||||
<PageSelector
|
||||
value={selectedPagesId}
|
||||
disabledValue={PagesMapAndSelectedPagesId[2]}
|
||||
searchValue={searchValue}
|
||||
list={currentWorkspace?.pages || []}
|
||||
pagesMap={PagesMapAndSelectedPagesId[0]}
|
||||
onSelect={handleSelectPages}
|
||||
canPreview={canPreview}
|
||||
previewPageId={previewPageId}
|
||||
onPreview={handlePreviewPage}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,167 +0,0 @@
|
|||
import { useCallback, useEffect, useMemo, useState } from 'react'
|
||||
import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
|
||||
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
|
||||
import PageSelector from '@/app/components/base/notion-page-selector/page-selector'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
type OnlineDocumentSelectorProps = {
|
||||
value?: string[]
|
||||
onSelect: (selectedPages: NotionPage[]) => void
|
||||
canPreview?: boolean
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPage: NotionPage) => void
|
||||
isInPipeline?: boolean
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
}
|
||||
|
||||
const OnlineDocumentSelector = ({
|
||||
value,
|
||||
onSelect,
|
||||
canPreview,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
nodeId,
|
||||
nodeData,
|
||||
}: OnlineDocumentSelectorProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDocuments = useCallback(async () => {
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: {},
|
||||
datasource_type: DatasourceType.onlineDocument,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
setDocumentsData(documentsData.data as DataSourceNotionWorkspace[])
|
||||
},
|
||||
onError: (message: string) => {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message,
|
||||
})
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL])
|
||||
|
||||
useEffect(() => {
|
||||
getOnlineDocuments()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const firstWorkspaceId = documentsData[0]?.workspace_id
|
||||
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
|
||||
|
||||
const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
|
||||
const selectedPagesId = new Set<string>()
|
||||
const boundPagesId = new Set<string>()
|
||||
const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
if (page.is_bound) {
|
||||
selectedPagesId.add(page.page_id)
|
||||
boundPagesId.add(page.page_id)
|
||||
}
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
}, {})
|
||||
return [pagesMap, selectedPagesId, boundPagesId]
|
||||
}, [documentsData])
|
||||
const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])]
|
||||
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
|
||||
|
||||
const handleSearchValueChange = useCallback((value: string) => {
|
||||
setSearchValue(value)
|
||||
}, [])
|
||||
const handleSelectWorkspace = useCallback((workspaceId: string) => {
|
||||
setCurrentWorkspaceId(workspaceId)
|
||||
}, [])
|
||||
const handleSelectPages = (newSelectedPagesId: Set<string>) => {
|
||||
const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId])
|
||||
|
||||
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
|
||||
onSelect(selectedPages)
|
||||
}
|
||||
const handlePreviewPage = (previewPageId: string) => {
|
||||
if (onPreview)
|
||||
onPreview(PagesMapAndSelectedPagesId[0][previewPageId])
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
setCurrentWorkspaceId(firstWorkspaceId)
|
||||
}, [firstWorkspaceId])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
if (!documentsData?.length)
|
||||
return null
|
||||
|
||||
return (
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
isInPipeline={isInPipeline}
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<div className='flex h-12 items-center gap-x-2 rounded-t-xl border-b border-b-divider-regular bg-components-panel-bg p-2'>
|
||||
<div className='flex grow items-center gap-x-1'>
|
||||
<WorkspaceSelector
|
||||
value={currentWorkspaceId || firstWorkspaceId}
|
||||
items={documentsData}
|
||||
onSelect={handleSelectWorkspace}
|
||||
/>
|
||||
</div>
|
||||
<SearchInput
|
||||
value={searchValue}
|
||||
onChange={handleSearchValueChange}
|
||||
/>
|
||||
</div>
|
||||
<div className='overflow-hidden rounded-b-xl'>
|
||||
<PageSelector
|
||||
value={selectedPagesId}
|
||||
disabledValue={PagesMapAndSelectedPagesId[2]}
|
||||
searchValue={searchValue}
|
||||
list={currentWorkspace?.pages || []}
|
||||
pagesMap={PagesMapAndSelectedPagesId[0]}
|
||||
onSelect={handleSelectPages}
|
||||
canPreview={canPreview}
|
||||
previewPageId={previewPageId}
|
||||
onPreview={handlePreviewPage}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default OnlineDocumentSelector
|
||||
|
|
@ -8,6 +8,7 @@ import NotionIcon from '@/app/components/base/notion-icon'
|
|||
import cn from '@/utils/classnames'
|
||||
import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common'
|
||||
|
||||
// todo: refactor this component to use the new OnlineDocumentSelector component
|
||||
type PageSelectorProps = {
|
||||
value: Set<string>
|
||||
disabledValue: Set<string>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
import React from 'react'
|
||||
import Button from '@/app/components/base/button'
|
||||
import Divider from '@/app/components/base/divider'
|
||||
import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react'
|
||||
|
||||
type HeaderProps = {
|
||||
onClickConfiguration?: () => void
|
||||
docTitle: string
|
||||
docLink: string
|
||||
}
|
||||
|
||||
const Header = ({
|
||||
onClickConfiguration,
|
||||
docTitle,
|
||||
docLink,
|
||||
}: HeaderProps) => {
|
||||
return (
|
||||
<div className='flex items-center gap-x-2'>
|
||||
<div className='flex shrink-0 grow items-center gap-x-1'>
|
||||
<div className='w-20 bg-black'>
|
||||
{/* placeholder */}
|
||||
</div>
|
||||
<Divider type='vertical' className='mx-1 h-3.5' />
|
||||
<Button
|
||||
variant='ghost'
|
||||
size='small'
|
||||
className='px-1'
|
||||
>
|
||||
<RiEqualizer2Line
|
||||
className='size-4'
|
||||
onClick={onClickConfiguration}
|
||||
/>
|
||||
</Button>
|
||||
</div>
|
||||
<a
|
||||
className='system-xs-medium flex items-center gap-x-1 overflow-hidden text-text-accent'
|
||||
href={docLink}
|
||||
target='_blank'
|
||||
rel='noopener noreferrer'
|
||||
>
|
||||
<RiBookOpenLine className='size-3.5 shrink-0' />
|
||||
<span className='grow truncate' title={docTitle}>{docTitle}</span>
|
||||
</a>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default React.memo(Header)
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import Connect from './connect'
|
||||
import Header from './header'
|
||||
|
||||
type OnlineDriveProps = {
|
||||
nodeData: DataSourceNodeType
|
||||
|
|
@ -9,7 +9,12 @@ const OnlineDrive = ({
|
|||
nodeData,
|
||||
}: OnlineDriveProps) => {
|
||||
return (
|
||||
<Connect nodeData={nodeData} />
|
||||
<div className='flex flex-col gap-y-2'>
|
||||
<Header
|
||||
docTitle='Online Drive Docs'
|
||||
docLink='https://docs.dify.ai/'
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,180 +0,0 @@
|
|||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import Options from './options'
|
||||
import Crawling from './crawling'
|
||||
import ErrorMessage from './error-message'
|
||||
import CrawledResult from './crawled-result'
|
||||
import {
|
||||
useDraftPipelinePreProcessingParams,
|
||||
usePublishedPipelinePreProcessingParams,
|
||||
} from '@/service/use-pipeline'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import type {
|
||||
DataSourceNodeCompletedResponse,
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export type CrawlerProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
crawlResult: CrawlResult | undefined
|
||||
setCrawlResult: (payload: CrawlResult) => void
|
||||
step: CrawlStep
|
||||
setStep: (step: CrawlStep) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
previewIndex?: number
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
isInPipeline?: boolean
|
||||
}
|
||||
|
||||
const Crawler = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
step,
|
||||
setStep,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
previewIndex,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
}: CrawlerProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const [totalNum, setTotalNum] = useState(0)
|
||||
const [crawledNum, setCrawledNum] = useState(0)
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
|
||||
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
|
||||
const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
|
||||
pipeline_id: pipelineId!,
|
||||
node_id: nodeId,
|
||||
}, !!pipelineId && !!nodeId)
|
||||
|
||||
useEffect(() => {
|
||||
if (step !== CrawlStep.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
|
||||
const isInit = step === CrawlStep.init
|
||||
const isCrawlFinished = step === CrawlStep.finished
|
||||
const isRunning = step === CrawlStep.running
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const handleRun = useCallback(async (value: Record<string, any>) => {
|
||||
setStep(CrawlStep.running)
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: value,
|
||||
datasource_type: DatasourceType.websiteCrawl,
|
||||
response_mode: 'streaming',
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => {
|
||||
setTotalNum(data.total ?? 0)
|
||||
setCrawledNum(data.completed ?? 0)
|
||||
},
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => {
|
||||
const { data: crawlData, time_consuming } = data
|
||||
const crawlResultData = {
|
||||
data: crawlData.map((item: any) => {
|
||||
const { content, ...rest } = item
|
||||
return {
|
||||
markdown: content || '',
|
||||
...rest,
|
||||
} as CrawlResultItem
|
||||
}),
|
||||
time_consuming: time_consuming ?? 0,
|
||||
}
|
||||
setCrawlResult(crawlResultData)
|
||||
onCheckedCrawlResultChange(crawlData || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
onError: (message: string) => {
|
||||
setCrawlErrorMessage(message || t(`${I18N_PREFIX}.unknownError`))
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t])
|
||||
|
||||
const handleSubmit = useCallback((value: Record<string, any>) => {
|
||||
handleRun(value)
|
||||
}, [handleRun])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
return (
|
||||
<div className='flex flex-col'>
|
||||
<Header
|
||||
isInPipeline
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<Options
|
||||
variables={paramsConfig?.variables || []}
|
||||
isRunning={isRunning}
|
||||
runDisabled={isFetchingParams}
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
onSubmit={handleSubmit}
|
||||
/>
|
||||
</div>
|
||||
{!isInit && (
|
||||
<div className='relative flex flex-col'>
|
||||
{isRunning && (
|
||||
<Crawling
|
||||
crawledNum={crawledNum}
|
||||
totalNum={totalNum}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage
|
||||
className='mt-2'
|
||||
title={t(`${I18N_PREFIX}.exceptionErrorTitle`)}
|
||||
errorMsg={crawlErrorMessage}
|
||||
/>
|
||||
)}
|
||||
{isCrawlFinished && !showError && (
|
||||
<CrawledResult
|
||||
className='mt-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={onPreview}
|
||||
supportMultipleChoice={false} // only support single choice in test run
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default React.memo(Crawler)
|
||||
|
|
@ -1,9 +1,41 @@
|
|||
'use client'
|
||||
import React from 'react'
|
||||
import type { CrawlerProps } from './base/crawler'
|
||||
import Crawler from './base/crawler'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import Options from './base/options'
|
||||
import Crawling from './base/crawling'
|
||||
import ErrorMessage from './base/error-message'
|
||||
import CrawledResult from './base/crawled-result'
|
||||
import {
|
||||
useDraftPipelinePreProcessingParams,
|
||||
usePublishedPipelinePreProcessingParams,
|
||||
} from '@/service/use-pipeline'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { ssePost } from '@/service/base'
|
||||
import type {
|
||||
DataSourceNodeCompletedResponse,
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
|
||||
type WebsiteCrawlProps = CrawlerProps
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export type WebsiteCrawlProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
crawlResult: CrawlResult | undefined
|
||||
setCrawlResult: (payload: CrawlResult) => void
|
||||
step: CrawlStep
|
||||
setStep: (step: CrawlStep) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
previewIndex?: number
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
isInPipeline?: boolean
|
||||
}
|
||||
|
||||
const WebsiteCrawl = ({
|
||||
nodeId,
|
||||
|
|
@ -16,22 +48,133 @@ const WebsiteCrawl = ({
|
|||
onCheckedCrawlResultChange,
|
||||
previewIndex,
|
||||
onPreview,
|
||||
isInPipeline,
|
||||
isInPipeline = false,
|
||||
}: WebsiteCrawlProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const [totalNum, setTotalNum] = useState(0)
|
||||
const [crawledNum, setCrawledNum] = useState(0)
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
|
||||
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
|
||||
const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
|
||||
pipeline_id: pipelineId!,
|
||||
node_id: nodeId,
|
||||
}, !!pipelineId && !!nodeId)
|
||||
|
||||
useEffect(() => {
|
||||
if (step !== CrawlStep.init)
|
||||
setControlFoldOptions(Date.now())
|
||||
}, [step])
|
||||
|
||||
const isInit = step === CrawlStep.init
|
||||
const isCrawlFinished = step === CrawlStep.finished
|
||||
const isRunning = step === CrawlStep.running
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const handleRun = useCallback(async (value: Record<string, any>) => {
|
||||
setStep(CrawlStep.running)
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: value,
|
||||
datasource_type: DatasourceType.websiteCrawl,
|
||||
response_mode: 'streaming',
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => {
|
||||
setTotalNum(data.total ?? 0)
|
||||
setCrawledNum(data.completed ?? 0)
|
||||
},
|
||||
onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => {
|
||||
const { data: crawlData, time_consuming } = data
|
||||
const crawlResultData = {
|
||||
data: crawlData.map((item: any) => {
|
||||
const { content, ...rest } = item
|
||||
return {
|
||||
markdown: content || '',
|
||||
...rest,
|
||||
} as CrawlResultItem
|
||||
}),
|
||||
time_consuming: time_consuming ?? 0,
|
||||
}
|
||||
setCrawlResult(crawlResultData)
|
||||
onCheckedCrawlResultChange(crawlData || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
onError: (message: string) => {
|
||||
setCrawlErrorMessage(message || t(`${I18N_PREFIX}.unknownError`))
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t])
|
||||
|
||||
const handleSubmit = useCallback((value: Record<string, any>) => {
|
||||
handleRun(value)
|
||||
}, [handleRun])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
title: nodeData.title,
|
||||
docTitle: 'How to use?',
|
||||
docLink: 'https://docs.dify.ai',
|
||||
}
|
||||
}, [nodeData])
|
||||
|
||||
return (
|
||||
<Crawler
|
||||
nodeId={nodeId}
|
||||
nodeData={nodeData}
|
||||
crawlResult={crawlResult}
|
||||
setCrawlResult={setCrawlResult}
|
||||
step={step}
|
||||
setStep={setStep}
|
||||
checkedCrawlResult={checkedCrawlResult}
|
||||
onCheckedCrawlResultChange={onCheckedCrawlResultChange}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={onPreview}
|
||||
isInPipeline={isInPipeline}
|
||||
/>
|
||||
<div className='flex flex-col'>
|
||||
<Header
|
||||
isInPipeline
|
||||
{...headerInfo}
|
||||
/>
|
||||
<div className='mt-2 rounded-xl border border-components-panel-border bg-background-default-subtle'>
|
||||
<Options
|
||||
variables={paramsConfig?.variables || []}
|
||||
isRunning={isRunning}
|
||||
runDisabled={isFetchingParams}
|
||||
controlFoldOptions={controlFoldOptions}
|
||||
onSubmit={handleSubmit}
|
||||
/>
|
||||
</div>
|
||||
{!isInit && (
|
||||
<div className='relative flex flex-col'>
|
||||
{isRunning && (
|
||||
<Crawling
|
||||
crawledNum={crawledNum}
|
||||
totalNum={totalNum}
|
||||
/>
|
||||
)}
|
||||
{showError && (
|
||||
<ErrorMessage
|
||||
className='mt-2'
|
||||
title={t(`${I18N_PREFIX}.exceptionErrorTitle`)}
|
||||
errorMsg={crawlErrorMessage}
|
||||
/>
|
||||
)}
|
||||
{isCrawlFinished && !showError && (
|
||||
<CrawledResult
|
||||
className='mt-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={onPreview}
|
||||
supportMultipleChoice={false} // only support single choice in test run
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
export default WebsiteCrawl
|
||||
export default React.memo(WebsiteCrawl)
|
||||
|
|
|
|||
|
|
@ -122,10 +122,11 @@ const TestRunPanel = () => {
|
|||
/>
|
||||
{datasourceType === DatasourceType.localFile && (
|
||||
<LocalFile
|
||||
files={fileList}
|
||||
fileList={fileList}
|
||||
allowedExtensions={datasource!.nodeData.fileExtensions || []}
|
||||
updateFile={updateFile}
|
||||
updateFileList={updateFileList}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
notSupportBatchUpload={false} // only support single file upload in test run
|
||||
/>
|
||||
)}
|
||||
|
|
@ -133,8 +134,8 @@ const TestRunPanel = () => {
|
|||
<OnlineDocuments
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
onlineDocuments={onlineDocuments}
|
||||
updateOnlineDocuments={updateOnlineDocuments}
|
||||
pageIdList={onlineDocuments.map(doc => doc.page_id)}
|
||||
onSelect={updateOnlineDocuments}
|
||||
isInPipeline
|
||||
/>
|
||||
)}
|
||||
|
|
|
|||
Loading…
Reference in New Issue