diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index de763cb937..532029c710 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -249,10 +249,11 @@ const CreateFormPipeline = () => { /> {datasourceType === DatasourceType.localFile && ( @@ -261,8 +262,8 @@ const CreateFormPipeline = () => { doc.page_id)} + onSelect={updateOnlineDocuments} canPreview onPreview={updateCurrentPage} /> diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx deleted file mode 100644 index 0389ca87ed..0000000000 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx +++ /dev/null @@ -1,338 +0,0 @@ -'use client' -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { useContext } from 'use-context-selector' -import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react' -import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon' -import cn from '@/utils/classnames' -import type { CustomFile as File, FileItem } from '@/models/datasets' -import { ToastContext } from '@/app/components/base/toast' -import SimplePieChart from '@/app/components/base/simple-pie-chart' -import { upload } from '@/service/base' -import I18n from '@/context/i18n' -import { LanguagesSupported } from '@/i18n/language' -import { IS_CE_EDITION } from '@/config' -import { Theme } from '@/types/app' -import useTheme from '@/hooks/use-theme' -import { useFileUploadConfig } from '@/service/use-common' - -const FILES_NUMBER_LIMIT = 20 - -export type FileUploaderProps = { - fileList: FileItem[] - allowedExtensions: string[] - prepareFileList: (files: FileItem[]) => void - onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void - onFileListUpdate?: (files: FileItem[]) => void - onPreview?: (file: File) => void - notSupportBatchUpload?: boolean -} - -const FileUploader = ({ - fileList, - allowedExtensions, - prepareFileList, - onFileUpdate, - onFileListUpdate, - onPreview, - notSupportBatchUpload, -}: FileUploaderProps) => { - const { t } = useTranslation() - const { notify } = useContext(ToastContext) - const { locale } = useContext(I18n) - const [dragging, setDragging] = useState(false) - const dropRef = useRef(null) - const dragRef = useRef(null) - const fileUploader = useRef(null) - const hideUpload = notSupportBatchUpload && fileList.length > 0 - - const { data: fileUploadConfigResponse } = useFileUploadConfig() - const supportTypesShowNames = useMemo(() => { - const extensionMap: { [key: string]: string } = { - md: 'markdown', - pptx: 'pptx', - htm: 'html', - xlsx: 'xlsx', - docx: 'docx', - } - - return allowedExtensions - .map(item => extensionMap[item] || item) // map to standardized extension - .map(item => item.toLowerCase()) // convert to lower case - .filter((item, index, self) => self.indexOf(item) === index) // remove duplicates - .map(item => item.toUpperCase()) // convert to upper case - .join(locale !== LanguagesSupported[1] ? ', ' : '、 ') - }, [locale, allowedExtensions]) - const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`) - const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { - file_size_limit: 15, - batch_count_limit: 5, - }, [fileUploadConfigResponse]) - - const fileListRef = useRef([]) - - // utils - const getFileType = (currentFile: File) => { - if (!currentFile) - return '' - - const arr = currentFile.name.split('.') - return arr[arr.length - 1] - } - - const getFileSize = (size: number) => { - if (size / 1024 < 10) - return `${(size / 1024).toFixed(2)}KB` - - return `${(size / 1024 / 1024).toFixed(2)}MB` - } - - const isValid = useCallback((file: File) => { - const { size } = file - const ext = `.${getFileType(file)}` - const isValidType = ACCEPTS.includes(ext.toLowerCase()) - if (!isValidType) - notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') }) - - const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024 - if (!isValidSize) - notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) }) - - return isValidType && isValidSize - }, [fileUploadConfig, notify, t, ACCEPTS]) - - const fileUpload = useCallback(async (fileItem: FileItem): Promise => { - const formData = new FormData() - formData.append('file', fileItem.file) - const onProgress = (e: ProgressEvent) => { - if (e.lengthComputable) { - const percent = Math.floor(e.loaded / e.total * 100) - onFileUpdate(fileItem, percent, fileListRef.current) - } - } - - return upload({ - xhr: new XMLHttpRequest(), - data: formData, - onprogress: onProgress, - }, false, undefined, '?source=datasets') - .then((res: File) => { - const completeFile = { - fileID: fileItem.fileID, - file: res, - progress: -1, - } - const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID) - fileListRef.current[index] = completeFile - onFileUpdate(completeFile, 100, fileListRef.current) - return Promise.resolve({ ...completeFile }) - }) - .catch((e) => { - notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') }) - onFileUpdate(fileItem, -2, fileListRef.current) - return Promise.resolve({ ...fileItem }) - }) - .finally() - }, [fileListRef, notify, onFileUpdate, t]) - - const uploadBatchFiles = useCallback((bFiles: FileItem[]) => { - bFiles.forEach(bf => (bf.progress = 0)) - return Promise.all(bFiles.map(fileUpload)) - }, [fileUpload]) - - const uploadMultipleFiles = useCallback(async (files: FileItem[]) => { - const batchCountLimit = fileUploadConfig.batch_count_limit - const length = files.length - let start = 0 - let end = 0 - - while (start < length) { - if (start + batchCountLimit > length) - end = length - else - end = start + batchCountLimit - const bFiles = files.slice(start, end) - await uploadBatchFiles(bFiles) - start = end - } - }, [fileUploadConfig, uploadBatchFiles]) - - const initialUpload = useCallback((files: File[]) => { - if (!files.length) - return false - - if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) { - notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) }) - return false - } - - const preparedFiles = files.map((file, index) => ({ - fileID: `file${index}-${Date.now()}`, - file, - progress: -1, - })) - const newFiles = [...fileListRef.current, ...preparedFiles] - prepareFileList(newFiles) - fileListRef.current = newFiles - uploadMultipleFiles(preparedFiles) - }, [prepareFileList, uploadMultipleFiles, notify, t, fileList]) - - const handleDragEnter = (e: DragEvent) => { - e.preventDefault() - e.stopPropagation() - e.target !== dragRef.current && setDragging(true) - } - const handleDragOver = (e: DragEvent) => { - e.preventDefault() - e.stopPropagation() - } - const handleDragLeave = (e: DragEvent) => { - e.preventDefault() - e.stopPropagation() - e.target === dragRef.current && setDragging(false) - } - - const handleDrop = useCallback((e: DragEvent) => { - e.preventDefault() - e.stopPropagation() - setDragging(false) - if (!e.dataTransfer) - return - - let files = [...e.dataTransfer.files] as File[] - if (notSupportBatchUpload) - files = files.slice(0, 1) - - const validFiles = files.filter(isValid) - initialUpload(validFiles) - }, [initialUpload, isValid, notSupportBatchUpload]) - - const selectHandle = () => { - if (fileUploader.current) - fileUploader.current.click() - } - - const removeFile = (fileID: string) => { - if (fileUploader.current) - fileUploader.current.value = '' - - fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID) - onFileListUpdate?.([...fileListRef.current]) - } - const fileChangeHandle = useCallback((e: React.ChangeEvent) => { - const files = [...(e.target.files ?? [])] as File[] - initialUpload(files.filter(isValid)) - }, [isValid, initialUpload]) - - const { theme } = useTheme() - const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme]) - - useEffect(() => { - const dropElement = dropRef.current - dropElement?.addEventListener('dragenter', handleDragEnter) - dropElement?.addEventListener('dragover', handleDragOver) - dropElement?.addEventListener('dragleave', handleDragLeave) - dropElement?.addEventListener('drop', handleDrop) - return () => { - dropElement?.removeEventListener('dragenter', handleDragEnter) - dropElement?.removeEventListener('dragover', handleDragOver) - dropElement?.removeEventListener('dragleave', handleDragLeave) - dropElement?.removeEventListener('drop', handleDrop) - } - }, [handleDrop]) - - return ( -
- {!hideUpload && ( - - )} - {!hideUpload && ( -
-
- - - - {t('datasetCreation.stepOne.uploader.button')} - {allowedExtensions.length > 0 && ( - - )} - -
-
{t('datasetCreation.stepOne.uploader.tip', { - size: fileUploadConfig.file_size_limit, - supportTypes: supportTypesShowNames, - batchCount: fileUploadConfig.batch_count_limit, - })}
- {dragging &&
} -
- )} - {fileList.length > 0 && ( -
- {fileList.map((fileItem, index) => { - const isUploading = fileItem.progress >= 0 && fileItem.progress < 100 - const isError = fileItem.progress === -2 - return ( -
fileItem.file?.id && onPreview?.(fileItem.file)} - className={cn( - 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', - isError && 'border-state-destructive-border bg-state-destructive-hover', - )} - > -
- -
-
-
-
{fileItem.file.name}
-
-
- {getFileType(fileItem.file)} - · - {getFileSize(fileItem.file.size)} -
-
-
- {isUploading && ( - - )} - { - isError && ( - - ) - } - { - e.stopPropagation() - removeFile(fileItem.fileID) - }}> - - -
-
- ) - })} -
- )} -
- ) -} - -export default FileUploader diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx index 1ed394a1c9..cfe1883f99 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx @@ -1,33 +1,337 @@ +'use client' +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import { useContext } from 'use-context-selector' +import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react' +import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon' +import cn from '@/utils/classnames' import type { CustomFile as File, FileItem } from '@/models/datasets' -import FileUploader from './file-uploader' +import { ToastContext } from '@/app/components/base/toast' +import SimplePieChart from '@/app/components/base/simple-pie-chart' +import { upload } from '@/service/base' +import I18n from '@/context/i18n' +import { LanguagesSupported } from '@/i18n/language' +import { IS_CE_EDITION } from '@/config' +import { Theme } from '@/types/app' +import useTheme from '@/hooks/use-theme' +import { useFileUploadConfig } from '@/service/use-common' -type LocalFileProps = { - files: FileItem[] +const FILES_NUMBER_LIMIT = 20 + +export type LocalFileProps = { + fileList: FileItem[] allowedExtensions: string[] - updateFileList: (files: FileItem[]) => void - updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void + prepareFileList: (files: FileItem[]) => void + onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void + onFileListUpdate?: (files: FileItem[]) => void onPreview?: (file: File) => void - notSupportBatchUpload: boolean + notSupportBatchUpload?: boolean } const LocalFile = ({ - files, + fileList, allowedExtensions, - updateFileList, - updateFile, + prepareFileList, + onFileUpdate, + onFileListUpdate, onPreview, notSupportBatchUpload, }: LocalFileProps) => { + const { t } = useTranslation() + const { notify } = useContext(ToastContext) + const { locale } = useContext(I18n) + const [dragging, setDragging] = useState(false) + const dropRef = useRef(null) + const dragRef = useRef(null) + const fileUploader = useRef(null) + const hideUpload = notSupportBatchUpload && fileList.length > 0 + + const { data: fileUploadConfigResponse } = useFileUploadConfig() + const supportTypesShowNames = useMemo(() => { + const extensionMap: { [key: string]: string } = { + md: 'markdown', + pptx: 'pptx', + htm: 'html', + xlsx: 'xlsx', + docx: 'docx', + } + + return allowedExtensions + .map(item => extensionMap[item] || item) // map to standardized extension + .map(item => item.toLowerCase()) // convert to lower case + .filter((item, index, self) => self.indexOf(item) === index) // remove duplicates + .map(item => item.toUpperCase()) // convert to upper case + .join(locale !== LanguagesSupported[1] ? ', ' : '、 ') + }, [locale, allowedExtensions]) + const ACCEPTS = allowedExtensions.map((ext: string) => `.${ext}`) + const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { + file_size_limit: 15, + batch_count_limit: 5, + }, [fileUploadConfigResponse]) + + const fileListRef = useRef([]) + + // utils + const getFileType = (currentFile: File) => { + if (!currentFile) + return '' + + const arr = currentFile.name.split('.') + return arr[arr.length - 1] + } + + const getFileSize = (size: number) => { + if (size / 1024 < 10) + return `${(size / 1024).toFixed(2)}KB` + + return `${(size / 1024 / 1024).toFixed(2)}MB` + } + + const isValid = useCallback((file: File) => { + const { size } = file + const ext = `.${getFileType(file)}` + const isValidType = ACCEPTS.includes(ext.toLowerCase()) + if (!isValidType) + notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.typeError') }) + + const isValidSize = size <= fileUploadConfig.file_size_limit * 1024 * 1024 + if (!isValidSize) + notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.size', { size: fileUploadConfig.file_size_limit }) }) + + return isValidType && isValidSize + }, [fileUploadConfig, notify, t, ACCEPTS]) + + const fileUpload = useCallback(async (fileItem: FileItem): Promise => { + const formData = new FormData() + formData.append('file', fileItem.file) + const onProgress = (e: ProgressEvent) => { + if (e.lengthComputable) { + const percent = Math.floor(e.loaded / e.total * 100) + onFileUpdate(fileItem, percent, fileListRef.current) + } + } + + return upload({ + xhr: new XMLHttpRequest(), + data: formData, + onprogress: onProgress, + }, false, undefined, '?source=datasets') + .then((res: File) => { + const completeFile = { + fileID: fileItem.fileID, + file: res, + progress: -1, + } + const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID) + fileListRef.current[index] = completeFile + onFileUpdate(completeFile, 100, fileListRef.current) + return Promise.resolve({ ...completeFile }) + }) + .catch((e) => { + notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') }) + onFileUpdate(fileItem, -2, fileListRef.current) + return Promise.resolve({ ...fileItem }) + }) + .finally() + }, [fileListRef, notify, onFileUpdate, t]) + + const uploadBatchFiles = useCallback((bFiles: FileItem[]) => { + bFiles.forEach(bf => (bf.progress = 0)) + return Promise.all(bFiles.map(fileUpload)) + }, [fileUpload]) + + const uploadMultipleFiles = useCallback(async (files: FileItem[]) => { + const batchCountLimit = fileUploadConfig.batch_count_limit + const length = files.length + let start = 0 + let end = 0 + + while (start < length) { + if (start + batchCountLimit > length) + end = length + else + end = start + batchCountLimit + const bFiles = files.slice(start, end) + await uploadBatchFiles(bFiles) + start = end + } + }, [fileUploadConfig, uploadBatchFiles]) + + const initialUpload = useCallback((files: File[]) => { + if (!files.length) + return false + + if (files.length + fileList.length > FILES_NUMBER_LIMIT && !IS_CE_EDITION) { + notify({ type: 'error', message: t('datasetCreation.stepOne.uploader.validation.filesNumber', { filesNumber: FILES_NUMBER_LIMIT }) }) + return false + } + + const preparedFiles = files.map((file, index) => ({ + fileID: `file${index}-${Date.now()}`, + file, + progress: -1, + })) + const newFiles = [...fileListRef.current, ...preparedFiles] + prepareFileList(newFiles) + fileListRef.current = newFiles + uploadMultipleFiles(preparedFiles) + }, [prepareFileList, uploadMultipleFiles, notify, t, fileList]) + + const handleDragEnter = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + e.target !== dragRef.current && setDragging(true) + } + const handleDragOver = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + } + const handleDragLeave = (e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + e.target === dragRef.current && setDragging(false) + } + + const handleDrop = useCallback((e: DragEvent) => { + e.preventDefault() + e.stopPropagation() + setDragging(false) + if (!e.dataTransfer) + return + + let files = [...e.dataTransfer.files] as File[] + if (notSupportBatchUpload) + files = files.slice(0, 1) + + const validFiles = files.filter(isValid) + initialUpload(validFiles) + }, [initialUpload, isValid, notSupportBatchUpload]) + + const selectHandle = () => { + if (fileUploader.current) + fileUploader.current.click() + } + + const removeFile = (fileID: string) => { + if (fileUploader.current) + fileUploader.current.value = '' + + fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID) + onFileListUpdate?.([...fileListRef.current]) + } + const fileChangeHandle = useCallback((e: React.ChangeEvent) => { + const files = [...(e.target.files ?? [])] as File[] + initialUpload(files.filter(isValid)) + }, [isValid, initialUpload]) + + const { theme } = useTheme() + const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme]) + + useEffect(() => { + const dropElement = dropRef.current + dropElement?.addEventListener('dragenter', handleDragEnter) + dropElement?.addEventListener('dragover', handleDragOver) + dropElement?.addEventListener('dragleave', handleDragLeave) + dropElement?.addEventListener('drop', handleDrop) + return () => { + dropElement?.removeEventListener('dragenter', handleDragEnter) + dropElement?.removeEventListener('dragover', handleDragOver) + dropElement?.removeEventListener('dragleave', handleDragLeave) + dropElement?.removeEventListener('drop', handleDrop) + } + }, [handleDrop]) + return ( - +
+ {!hideUpload && ( + + )} + {!hideUpload && ( +
+
+ + + + {t('datasetCreation.stepOne.uploader.button')} + {allowedExtensions.length > 0 && ( + + )} + +
+
{t('datasetCreation.stepOne.uploader.tip', { + size: fileUploadConfig.file_size_limit, + supportTypes: supportTypesShowNames, + batchCount: fileUploadConfig.batch_count_limit, + })}
+ {dragging &&
} +
+ )} + {fileList.length > 0 && ( +
+ {fileList.map((fileItem, index) => { + const isUploading = fileItem.progress >= 0 && fileItem.progress < 100 + const isError = fileItem.progress === -2 + return ( +
fileItem.file?.id && onPreview?.(fileItem.file)} + className={cn( + 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', + isError && 'border-state-destructive-border bg-state-destructive-hover', + )} + > +
+ +
+
+
+
{fileItem.file.name}
+
+
+ {getFileType(fileItem.file)} + · + {getFileSize(fileItem.file.size)} +
+
+
+ {isUploading && ( + + )} + { + isError && ( + + ) + } + { + e.stopPropagation() + removeFile(fileItem.fileID) + }}> + + +
+
+ ) + })} +
+ )} +
) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx index bb66b9f13c..cc3b9db8e3 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx @@ -1,36 +1,166 @@ -import type { NotionPage } from '@/models/common' -import OnlineDocumentSelector from './online-document-selector' +import { useCallback, useEffect, useMemo, useState } from 'react' +import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' +import SearchInput from '@/app/components/base/notion-page-selector/search-input' +import PageSelector from '@/app/components/base/notion-page-selector/page-selector' +import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' +import Header from '@/app/components/datasets/create/website/base/header' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { DatasourceType } from '@/models/pipeline' +import { ssePost } from '@/service/base' +import Toast from '@/app/components/base/toast' +import type { DataSourceNodeCompletedResponse } from '@/types/pipeline' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' type OnlineDocumentsProps = { - nodeId: string - nodeData: DataSourceNodeType - onlineDocuments: NotionPage[] - updateOnlineDocuments: (value: NotionPage[]) => void + pageIdList?: string[] + onSelect: (selectedPages: NotionPage[]) => void canPreview?: boolean + previewPageId?: string onPreview?: (selectedPage: NotionPage) => void isInPipeline?: boolean + nodeId: string + nodeData: DataSourceNodeType } const OnlineDocuments = ({ - nodeId, - nodeData, - onlineDocuments, - updateOnlineDocuments, - canPreview = false, + pageIdList, + onSelect, + canPreview, + previewPageId, onPreview, isInPipeline = false, + nodeId, + nodeData, }: OnlineDocumentsProps) => { + const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + const [documentsData, setDocumentsData] = useState([]) + const [searchValue, setSearchValue] = useState('') + const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') + + const datasourceNodeRunURL = !isInPipeline + ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` + : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` + + const getOnlineDocuments = useCallback(async () => { + ssePost( + datasourceNodeRunURL, + { + body: { + inputs: {}, + datasource_type: DatasourceType.onlineDocument, + }, + }, + { + onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { + setDocumentsData(documentsData.data as DataSourceNotionWorkspace[]) + }, + onError: (message: string) => { + Toast.notify({ + type: 'error', + message, + }) + }, + }, + ) + }, [datasourceNodeRunURL]) + + useEffect(() => { + getOnlineDocuments() + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + const firstWorkspaceId = documentsData[0]?.workspace_id + const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) + + const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { + const selectedPagesId = new Set() + const boundPagesId = new Set() + const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + next.pages.forEach((page) => { + if (page.is_bound) { + selectedPagesId.add(page.page_id) + boundPagesId.add(page.page_id) + } + prev[page.page_id] = { + ...page, + workspace_id: next.workspace_id, + } + }) + + return prev + }, {}) + return [pagesMap, selectedPagesId, boundPagesId] + }, [documentsData]) + const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(pageIdList || [])] + const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) + + const handleSearchValueChange = useCallback((value: string) => { + setSearchValue(value) + }, []) + const handleSelectWorkspace = useCallback((workspaceId: string) => { + setCurrentWorkspaceId(workspaceId) + }, []) + const handleSelectPages = (newSelectedPagesId: Set) => { + const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId]) + + setSelectedPagesId(new Set(Array.from(newSelectedPagesId))) + onSelect(selectedPages) + } + const handlePreviewPage = (previewPageId: string) => { + if (onPreview) + onPreview(PagesMapAndSelectedPagesId[0][previewPageId]) + } + + useEffect(() => { + setCurrentWorkspaceId(firstWorkspaceId) + }, [firstWorkspaceId]) + + const headerInfo = useMemo(() => { + return { + title: nodeData.title, + docTitle: 'How to use?', + docLink: 'https://docs.dify.ai', + } + }, [nodeData]) + + if (!documentsData?.length) + return null + return ( - page.page_id)} - onSelect={updateOnlineDocuments} - canPreview={canPreview} - onPreview={onPreview} - isInPipeline={isInPipeline} - /> +
+
+
+
+
+ +
+ +
+
+ +
+
+
) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx deleted file mode 100644 index bafe71eb8a..0000000000 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx +++ /dev/null @@ -1,167 +0,0 @@ -import { useCallback, useEffect, useMemo, useState } from 'react' -import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' -import SearchInput from '@/app/components/base/notion-page-selector/search-input' -import PageSelector from '@/app/components/base/notion-page-selector/page-selector' -import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' -import Header from '@/app/components/datasets/create/website/base/header' -import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' -import { DatasourceType } from '@/models/pipeline' -import { ssePost } from '@/service/base' -import Toast from '@/app/components/base/toast' -import type { DataSourceNodeCompletedResponse } from '@/types/pipeline' -import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' - -type OnlineDocumentSelectorProps = { - value?: string[] - onSelect: (selectedPages: NotionPage[]) => void - canPreview?: boolean - previewPageId?: string - onPreview?: (selectedPage: NotionPage) => void - isInPipeline?: boolean - nodeId: string - nodeData: DataSourceNodeType -} - -const OnlineDocumentSelector = ({ - value, - onSelect, - canPreview, - previewPageId, - onPreview, - isInPipeline = false, - nodeId, - nodeData, -}: OnlineDocumentSelectorProps) => { - const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) - const [documentsData, setDocumentsData] = useState([]) - const [searchValue, setSearchValue] = useState('') - const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') - - const datasourceNodeRunURL = !isInPipeline - ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` - : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` - - const getOnlineDocuments = useCallback(async () => { - ssePost( - datasourceNodeRunURL, - { - body: { - inputs: {}, - datasource_type: DatasourceType.onlineDocument, - }, - }, - { - onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { - setDocumentsData(documentsData.data as DataSourceNotionWorkspace[]) - }, - onError: (message: string) => { - Toast.notify({ - type: 'error', - message, - }) - }, - }, - ) - }, [datasourceNodeRunURL]) - - useEffect(() => { - getOnlineDocuments() - // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) - - const firstWorkspaceId = documentsData[0]?.workspace_id - const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) - - const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { - const selectedPagesId = new Set() - const boundPagesId = new Set() - const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { - next.pages.forEach((page) => { - if (page.is_bound) { - selectedPagesId.add(page.page_id) - boundPagesId.add(page.page_id) - } - prev[page.page_id] = { - ...page, - workspace_id: next.workspace_id, - } - }) - - return prev - }, {}) - return [pagesMap, selectedPagesId, boundPagesId] - }, [documentsData]) - const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])] - const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) - - const handleSearchValueChange = useCallback((value: string) => { - setSearchValue(value) - }, []) - const handleSelectWorkspace = useCallback((workspaceId: string) => { - setCurrentWorkspaceId(workspaceId) - }, []) - const handleSelectPages = (newSelectedPagesId: Set) => { - const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId]) - - setSelectedPagesId(new Set(Array.from(newSelectedPagesId))) - onSelect(selectedPages) - } - const handlePreviewPage = (previewPageId: string) => { - if (onPreview) - onPreview(PagesMapAndSelectedPagesId[0][previewPageId]) - } - - useEffect(() => { - setCurrentWorkspaceId(firstWorkspaceId) - }, [firstWorkspaceId]) - - const headerInfo = useMemo(() => { - return { - title: nodeData.title, - docTitle: 'How to use?', - docLink: 'https://docs.dify.ai', - } - }, [nodeData]) - - if (!documentsData?.length) - return null - - return ( -
-
-
-
-
- -
- -
-
- -
-
-
- ) -} - -export default OnlineDocumentSelector diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx index 37710201c7..c696d39c47 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx @@ -8,6 +8,7 @@ import NotionIcon from '@/app/components/base/notion-icon' import cn from '@/utils/classnames' import type { DataSourceNotionPage, DataSourceNotionPageMap } from '@/models/common' +// todo: refactor this component to use the new OnlineDocumentSelector component type PageSelectorProps = { value: Set disabledValue: Set diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/header.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/header.tsx new file mode 100644 index 0000000000..905a2c2492 --- /dev/null +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/header.tsx @@ -0,0 +1,48 @@ +import React from 'react' +import Button from '@/app/components/base/button' +import Divider from '@/app/components/base/divider' +import { RiBookOpenLine, RiEqualizer2Line } from '@remixicon/react' + +type HeaderProps = { + onClickConfiguration?: () => void + docTitle: string + docLink: string +} + +const Header = ({ + onClickConfiguration, + docTitle, + docLink, +}: HeaderProps) => { + return ( +
+
+
+ {/* placeholder */} +
+ + +
+ + + {docTitle} + +
+ ) +} + +export default React.memo(Header) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx index 2d2e10707a..875d276825 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-drive/index.tsx @@ -1,5 +1,5 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -import Connect from './connect' +import Header from './header' type OnlineDriveProps = { nodeData: DataSourceNodeType @@ -9,7 +9,12 @@ const OnlineDrive = ({ nodeData, }: OnlineDriveProps) => { return ( - +
+
+
) } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx deleted file mode 100644 index 2b72742bb0..0000000000 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/base/crawler.tsx +++ /dev/null @@ -1,180 +0,0 @@ -'use client' -import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import { useTranslation } from 'react-i18next' -import type { CrawlResult, CrawlResultItem } from '@/models/datasets' -import { CrawlStep } from '@/models/datasets' -import Header from '@/app/components/datasets/create/website/base/header' -import Options from './options' -import Crawling from './crawling' -import ErrorMessage from './error-message' -import CrawledResult from './crawled-result' -import { - useDraftPipelinePreProcessingParams, - usePublishedPipelinePreProcessingParams, -} from '@/service/use-pipeline' -import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' -import { DatasourceType } from '@/models/pipeline' -import { ssePost } from '@/service/base' -import type { - DataSourceNodeCompletedResponse, - DataSourceNodeProcessingResponse, -} from '@/types/pipeline' -import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' - -const I18N_PREFIX = 'datasetCreation.stepOne.website' - -export type CrawlerProps = { - nodeId: string - nodeData: DataSourceNodeType - crawlResult: CrawlResult | undefined - setCrawlResult: (payload: CrawlResult) => void - step: CrawlStep - setStep: (step: CrawlStep) => void - checkedCrawlResult: CrawlResultItem[] - onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void - previewIndex?: number - onPreview?: (payload: CrawlResultItem, index: number) => void - isInPipeline?: boolean -} - -const Crawler = ({ - nodeId, - nodeData, - crawlResult, - setCrawlResult, - step, - setStep, - checkedCrawlResult, - onCheckedCrawlResultChange, - previewIndex, - onPreview, - isInPipeline = false, -}: CrawlerProps) => { - const { t } = useTranslation() - const [controlFoldOptions, setControlFoldOptions] = useState(0) - const [totalNum, setTotalNum] = useState(0) - const [crawledNum, setCrawledNum] = useState(0) - const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) - - const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams) - const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({ - pipeline_id: pipelineId!, - node_id: nodeId, - }, !!pipelineId && !!nodeId) - - useEffect(() => { - if (step !== CrawlStep.init) - setControlFoldOptions(Date.now()) - }, [step]) - - const isInit = step === CrawlStep.init - const isCrawlFinished = step === CrawlStep.finished - const isRunning = step === CrawlStep.running - const [crawlErrorMessage, setCrawlErrorMessage] = useState('') - const showError = isCrawlFinished && crawlErrorMessage - - const datasourceNodeRunURL = !isInPipeline - ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` - : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` - - const handleRun = useCallback(async (value: Record) => { - setStep(CrawlStep.running) - ssePost( - datasourceNodeRunURL, - { - body: { - inputs: value, - datasource_type: DatasourceType.websiteCrawl, - response_mode: 'streaming', - }, - }, - { - onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => { - setTotalNum(data.total ?? 0) - setCrawledNum(data.completed ?? 0) - }, - onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => { - const { data: crawlData, time_consuming } = data - const crawlResultData = { - data: crawlData.map((item: any) => { - const { content, ...rest } = item - return { - markdown: content || '', - ...rest, - } as CrawlResultItem - }), - time_consuming: time_consuming ?? 0, - } - setCrawlResult(crawlResultData) - onCheckedCrawlResultChange(crawlData || []) // default select the crawl result - setCrawlErrorMessage('') - setStep(CrawlStep.finished) - }, - onError: (message: string) => { - setCrawlErrorMessage(message || t(`${I18N_PREFIX}.unknownError`)) - setStep(CrawlStep.finished) - }, - }, - ) - }, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t]) - - const handleSubmit = useCallback((value: Record) => { - handleRun(value) - }, [handleRun]) - - const headerInfo = useMemo(() => { - return { - title: nodeData.title, - docTitle: 'How to use?', - docLink: 'https://docs.dify.ai', - } - }, [nodeData]) - - return ( -
-
-
- -
- {!isInit && ( -
- {isRunning && ( - - )} - {showError && ( - - )} - {isCrawlFinished && !showError && ( - - )} -
- )} -
- ) -} -export default React.memo(Crawler) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx index 67f953a8de..4c4582d8ee 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl/index.tsx @@ -1,9 +1,41 @@ 'use client' -import React from 'react' -import type { CrawlerProps } from './base/crawler' -import Crawler from './base/crawler' +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useTranslation } from 'react-i18next' +import type { CrawlResult, CrawlResultItem } from '@/models/datasets' +import { CrawlStep } from '@/models/datasets' +import Header from '@/app/components/datasets/create/website/base/header' +import Options from './base/options' +import Crawling from './base/crawling' +import ErrorMessage from './base/error-message' +import CrawledResult from './base/crawled-result' +import { + useDraftPipelinePreProcessingParams, + usePublishedPipelinePreProcessingParams, +} from '@/service/use-pipeline' +import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' +import { DatasourceType } from '@/models/pipeline' +import { ssePost } from '@/service/base' +import type { + DataSourceNodeCompletedResponse, + DataSourceNodeProcessingResponse, +} from '@/types/pipeline' +import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' -type WebsiteCrawlProps = CrawlerProps +const I18N_PREFIX = 'datasetCreation.stepOne.website' + +export type WebsiteCrawlProps = { + nodeId: string + nodeData: DataSourceNodeType + crawlResult: CrawlResult | undefined + setCrawlResult: (payload: CrawlResult) => void + step: CrawlStep + setStep: (step: CrawlStep) => void + checkedCrawlResult: CrawlResultItem[] + onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void + previewIndex?: number + onPreview?: (payload: CrawlResultItem, index: number) => void + isInPipeline?: boolean +} const WebsiteCrawl = ({ nodeId, @@ -16,22 +48,133 @@ const WebsiteCrawl = ({ onCheckedCrawlResultChange, previewIndex, onPreview, - isInPipeline, + isInPipeline = false, }: WebsiteCrawlProps) => { + const { t } = useTranslation() + const [controlFoldOptions, setControlFoldOptions] = useState(0) + const [totalNum, setTotalNum] = useState(0) + const [crawledNum, setCrawledNum] = useState(0) + const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) + + const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams) + const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({ + pipeline_id: pipelineId!, + node_id: nodeId, + }, !!pipelineId && !!nodeId) + + useEffect(() => { + if (step !== CrawlStep.init) + setControlFoldOptions(Date.now()) + }, [step]) + + const isInit = step === CrawlStep.init + const isCrawlFinished = step === CrawlStep.finished + const isRunning = step === CrawlStep.running + const [crawlErrorMessage, setCrawlErrorMessage] = useState('') + const showError = isCrawlFinished && crawlErrorMessage + + const datasourceNodeRunURL = !isInPipeline + ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` + : `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run` + + const handleRun = useCallback(async (value: Record) => { + setStep(CrawlStep.running) + ssePost( + datasourceNodeRunURL, + { + body: { + inputs: value, + datasource_type: DatasourceType.websiteCrawl, + response_mode: 'streaming', + }, + }, + { + onDataSourceNodeProcessing: (data: DataSourceNodeProcessingResponse) => { + setTotalNum(data.total ?? 0) + setCrawledNum(data.completed ?? 0) + }, + onDataSourceNodeCompleted: (data: DataSourceNodeCompletedResponse) => { + const { data: crawlData, time_consuming } = data + const crawlResultData = { + data: crawlData.map((item: any) => { + const { content, ...rest } = item + return { + markdown: content || '', + ...rest, + } as CrawlResultItem + }), + time_consuming: time_consuming ?? 0, + } + setCrawlResult(crawlResultData) + onCheckedCrawlResultChange(crawlData || []) // default select the crawl result + setCrawlErrorMessage('') + setStep(CrawlStep.finished) + }, + onError: (message: string) => { + setCrawlErrorMessage(message || t(`${I18N_PREFIX}.unknownError`)) + setStep(CrawlStep.finished) + }, + }, + ) + }, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t]) + + const handleSubmit = useCallback((value: Record) => { + handleRun(value) + }, [handleRun]) + + const headerInfo = useMemo(() => { + return { + title: nodeData.title, + docTitle: 'How to use?', + docLink: 'https://docs.dify.ai', + } + }, [nodeData]) + return ( - +
+
+
+ +
+ {!isInit && ( +
+ {isRunning && ( + + )} + {showError && ( + + )} + {isCrawlFinished && !showError && ( + + )} +
+ )} +
) } -export default WebsiteCrawl +export default React.memo(WebsiteCrawl) diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index e4976293ca..5eb4c14dee 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -122,10 +122,11 @@ const TestRunPanel = () => { /> {datasourceType === DatasourceType.localFile && ( )} @@ -133,8 +134,8 @@ const TestRunPanel = () => { doc.page_id)} + onSelect={updateOnlineDocuments} isInPipeline /> )}