mirror of
https://github.com/langgenius/dify.git
synced 2026-04-13 14:48:11 +08:00
refactor: Refactor data source components
This commit is contained in:
twwu
parent
816b49483a
commit
f2960989c1
@ -5,7 +5,7 @@ import { useContext } from 'use-context-selector'
|
||||
import { RiDeleteBinLine, RiErrorWarningFill, RiUploadCloud2Line } from '@remixicon/react'
|
||||
import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon'
|
||||
import cn from '@/utils/classnames'
|
||||
import type { CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
|
||||
import { ToastContext } from '@/app/components/base/toast'
|
||||
import SimplePieChart from '@/app/components/base/simple-pie-chart'
|
||||
import { upload } from '@/service/base'
|
||||
@ -15,35 +15,34 @@ import { IS_CE_EDITION } from '@/config'
|
||||
import { Theme } from '@/types/app'
|
||||
import useTheme from '@/hooks/use-theme'
|
||||
import { useFileUploadConfig } from '@/service/use-common'
|
||||
import { useDataSourceStore } from '../store'
|
||||
import produce from 'immer'
|
||||
|
||||
const FILES_NUMBER_LIMIT = 20
|
||||
|
||||
export type LocalFileProps = {
|
||||
fileList: FileItem[]
|
||||
allowedExtensions: string[]
|
||||
prepareFileList: (files: FileItem[]) => void
|
||||
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
|
||||
onFileListUpdate?: (files: FileItem[]) => void
|
||||
onPreview?: (file: File) => void
|
||||
notSupportBatchUpload?: boolean
|
||||
}
|
||||
|
||||
const LocalFile = ({
|
||||
fileList,
|
||||
allowedExtensions,
|
||||
prepareFileList,
|
||||
onFileUpdate,
|
||||
onFileListUpdate,
|
||||
onPreview,
|
||||
notSupportBatchUpload,
|
||||
}: LocalFileProps) => {
|
||||
const { t } = useTranslation()
|
||||
const { notify } = useContext(ToastContext)
|
||||
const { locale } = useContext(I18n)
|
||||
const fileList = useDataSourceStore(state => state.localFileList)
|
||||
const setFileList = useDataSourceStore(state => state.setLocalFileList)
|
||||
const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile)
|
||||
const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef)
|
||||
const [dragging, setDragging] = useState(false)
|
||||
|
||||
const dropRef = useRef<HTMLDivElement>(null)
|
||||
const dragRef = useRef<HTMLDivElement>(null)
|
||||
const fileUploader = useRef<HTMLInputElement>(null)
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
|
||||
const hideUpload = notSupportBatchUpload && fileList.length > 0
|
||||
|
||||
const { data: fileUploadConfigResponse } = useFileUploadConfig()
|
||||
@ -69,7 +68,26 @@ const LocalFile = ({
|
||||
batch_count_limit: 5,
|
||||
}, [fileUploadConfigResponse])
|
||||
|
||||
const fileListRef = useRef<FileItem[]>([])
|
||||
const updateFile = useCallback((fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
const newList = produce(list, (draft) => {
|
||||
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
|
||||
draft[targetIndex] = {
|
||||
...draft[targetIndex],
|
||||
progress,
|
||||
}
|
||||
})
|
||||
setFileList(newList)
|
||||
previewFileRef.current = newList[0].file as DocumentItem
|
||||
}, [previewFileRef, setFileList])
|
||||
|
||||
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
|
||||
setFileList(preparedFiles)
|
||||
}, [setFileList])
|
||||
|
||||
const handlePreview = useCallback((file: File) => {
|
||||
if (file.id)
|
||||
setCurrentFile(file)
|
||||
}, [setCurrentFile])
|
||||
|
||||
// utils
|
||||
const getFileType = (currentFile: File) => {
|
||||
@ -107,7 +125,7 @@ const LocalFile = ({
|
||||
const onProgress = (e: ProgressEvent) => {
|
||||
if (e.lengthComputable) {
|
||||
const percent = Math.floor(e.loaded / e.total * 100)
|
||||
onFileUpdate(fileItem, percent, fileListRef.current)
|
||||
updateFile(fileItem, percent, fileListRef.current)
|
||||
}
|
||||
}
|
||||
|
||||
@ -124,16 +142,16 @@ const LocalFile = ({
|
||||
}
|
||||
const index = fileListRef.current.findIndex(item => item.fileID === fileItem.fileID)
|
||||
fileListRef.current[index] = completeFile
|
||||
onFileUpdate(completeFile, 100, fileListRef.current)
|
||||
updateFile(completeFile, 100, fileListRef.current)
|
||||
return Promise.resolve({ ...completeFile })
|
||||
})
|
||||
.catch((e) => {
|
||||
notify({ type: 'error', message: e?.response?.code === 'forbidden' ? e?.response?.message : t('datasetCreation.stepOne.uploader.failed') })
|
||||
onFileUpdate(fileItem, -2, fileListRef.current)
|
||||
updateFile(fileItem, -2, fileListRef.current)
|
||||
return Promise.resolve({ ...fileItem })
|
||||
})
|
||||
.finally()
|
||||
}, [fileListRef, notify, onFileUpdate, t])
|
||||
}, [fileListRef, notify, updateFile, t])
|
||||
|
||||
const uploadBatchFiles = useCallback((bFiles: FileItem[]) => {
|
||||
bFiles.forEach(bf => (bf.progress = 0))
|
||||
@ -172,10 +190,10 @@ const LocalFile = ({
|
||||
progress: -1,
|
||||
}))
|
||||
const newFiles = [...fileListRef.current, ...preparedFiles]
|
||||
prepareFileList(newFiles)
|
||||
updateFileList(newFiles)
|
||||
fileListRef.current = newFiles
|
||||
uploadMultipleFiles(preparedFiles)
|
||||
}, [prepareFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
}, [updateFileList, uploadMultipleFiles, notify, t, fileList])
|
||||
|
||||
const handleDragEnter = (e: DragEvent) => {
|
||||
e.preventDefault()
|
||||
@ -207,17 +225,17 @@ const LocalFile = ({
|
||||
initialUpload(validFiles)
|
||||
}, [initialUpload, isValid, notSupportBatchUpload])
|
||||
|
||||
const selectHandle = () => {
|
||||
const selectHandle = useCallback(() => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.click()
|
||||
}
|
||||
}, [])
|
||||
|
||||
const removeFile = (fileID: string) => {
|
||||
if (fileUploader.current)
|
||||
fileUploader.current.value = ''
|
||||
|
||||
fileListRef.current = fileListRef.current.filter(item => item.fileID !== fileID)
|
||||
onFileListUpdate?.([...fileListRef.current])
|
||||
updateFileList([...fileListRef.current])
|
||||
}
|
||||
const fileChangeHandle = useCallback((e: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = [...(e.target.files ?? [])] as File[]
|
||||
@ -287,7 +305,7 @@ const LocalFile = ({
|
||||
return (
|
||||
<div
|
||||
key={`${fileItem.fileID}-${index}`}
|
||||
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
|
||||
onClick={handlePreview.bind(null, fileItem.file)}
|
||||
className={cn(
|
||||
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
|
||||
isError && 'border-state-destructive-border bg-state-destructive-hover',
|
||||
|
||||
@ -2,7 +2,7 @@ import { useCallback, useEffect, useMemo } from 'react'
|
||||
import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector'
|
||||
import SearchInput from '@/app/components/base/notion-page-selector/search-input'
|
||||
import PageSelector from './page-selector'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace } from '@/models/common'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
@ -10,43 +10,44 @@ import { ssePost } from '@/service/base'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import type { DataSourceNodeCompletedResponse } from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import { useDataSourceStore } from '../store'
|
||||
|
||||
type OnlineDocumentsProps = {
|
||||
onSelect: (selectedPages: NotionPage[]) => void
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPage: NotionPage) => void
|
||||
isInPipeline?: boolean
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
documentsData: DataSourceNotionWorkspace[]
|
||||
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void
|
||||
searchValue: string
|
||||
setSearchValue: (value: string) => void
|
||||
currentWorkspaceId: string
|
||||
setCurrentWorkspaceId: (workspaceId: string) => void
|
||||
PagesMapAndSelectedPagesId: DataSourceNotionPageMap
|
||||
selectedPagesId: Set<string>
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) => void
|
||||
}
|
||||
|
||||
const OnlineDocuments = ({
|
||||
onSelect,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
nodeId,
|
||||
nodeData,
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
}: OnlineDocumentsProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const documentsData = useDataSourceStore(state => state.documentsData)
|
||||
const setDocumentsData = useDataSourceStore(state => state.setDocumentsData)
|
||||
const searchValue = useDataSourceStore(state => state.searchValue)
|
||||
const setSearchValue = useDataSourceStore(state => state.setSearchValue)
|
||||
const currentWorkspaceId = useDataSourceStore(state => state.currentWorkspaceId)
|
||||
const setCurrentWorkspaceId = useDataSourceStore(state => state.setCurrentWorkspaceId)
|
||||
const setOnlineDocuments = useDataSourceStore(state => state.setOnlineDocuments)
|
||||
const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument)
|
||||
const selectedPagesId = useDataSourceStore(state => state.selectedPagesId)
|
||||
const setSelectedPagesId = useDataSourceStore(state => state.setSelectedPagesId)
|
||||
|
||||
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
|
||||
const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
}, {})
|
||||
return pagesMap
|
||||
}, [documentsData])
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
@ -96,13 +97,12 @@ const OnlineDocuments = ({
|
||||
const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[pageId])
|
||||
|
||||
setSelectedPagesId(new Set(Array.from(newSelectedPagesId)))
|
||||
onSelect(selectedPages)
|
||||
}, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId])
|
||||
setOnlineDocuments(selectedPages)
|
||||
}, [setSelectedPagesId, setOnlineDocuments, PagesMapAndSelectedPagesId])
|
||||
|
||||
const handlePreviewPage = useCallback((previewPageId: string) => {
|
||||
if (onPreview)
|
||||
onPreview(PagesMapAndSelectedPagesId[previewPageId])
|
||||
}, [PagesMapAndSelectedPagesId, onPreview])
|
||||
setCurrentDocument(PagesMapAndSelectedPagesId[previewPageId])
|
||||
}, [PagesMapAndSelectedPagesId, setCurrentDocument])
|
||||
|
||||
const headerInfo = useMemo(() => {
|
||||
return {
|
||||
@ -144,7 +144,6 @@ const OnlineDocuments = ({
|
||||
pagesMap={PagesMapAndSelectedPagesId}
|
||||
onSelect={handleSelectPages}
|
||||
canPreview={!isInPipeline}
|
||||
previewPageId={previewPageId}
|
||||
onPreview={handlePreviewPage}
|
||||
isMultipleChoice={!isInPipeline}
|
||||
currentWorkspaceId={currentWorkspaceId}
|
||||
|
||||
@ -13,7 +13,6 @@ type PageSelectorProps = {
|
||||
list: DataSourceNotionPage[]
|
||||
onSelect: (selectedPagesId: Set<string>) => void
|
||||
canPreview?: boolean
|
||||
previewPageId?: string
|
||||
onPreview?: (selectedPageId: string) => void
|
||||
isMultipleChoice?: boolean
|
||||
currentWorkspaceId: string
|
||||
@ -41,14 +40,13 @@ const PageSelector = ({
|
||||
list,
|
||||
onSelect,
|
||||
canPreview = true,
|
||||
previewPageId,
|
||||
onPreview,
|
||||
isMultipleChoice = true,
|
||||
currentWorkspaceId,
|
||||
}: PageSelectorProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [dataList, setDataList] = useState<NotionPageItem[]>([])
|
||||
const [localPreviewPageId, setLocalPreviewPageId] = useState('')
|
||||
const [currentPreviewPageId, setCurrentPreviewPageId] = useState('')
|
||||
|
||||
useEffect(() => {
|
||||
setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => {
|
||||
@ -71,7 +69,6 @@ const PageSelector = ({
|
||||
}
|
||||
})
|
||||
const currentDataList = searchValue ? searchDataList : dataList
|
||||
const currentPreviewPageId = previewPageId === undefined ? localPreviewPageId : previewPageId
|
||||
|
||||
const listMapWithChildrenAndDescendants = useMemo(() => {
|
||||
return list.reduce((prev: NotionPageTreeMap, next: DataSourceNotionPage) => {
|
||||
@ -149,7 +146,7 @@ const PageSelector = ({
|
||||
const current = currentDataList[index]
|
||||
const pageId = current.page_id
|
||||
|
||||
setLocalPreviewPageId(pageId)
|
||||
setCurrentPreviewPageId(pageId)
|
||||
|
||||
if (onPreview)
|
||||
onPreview(pageId)
|
||||
|
||||
@ -1,36 +1,77 @@
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import Header from './header'
|
||||
import { useCallback } from 'react'
|
||||
import { useCallback, useEffect } from 'react'
|
||||
import FileList from './file-list'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
|
||||
import { ssePost } from '@/service/base'
|
||||
import type { DataSourceNodeCompletedResponse, DataSourceNodeErrorResponse } from '@/types/pipeline'
|
||||
import Toast from '@/app/components/base/toast'
|
||||
import { useDataSourceStore } from '../store'
|
||||
|
||||
type OnlineDriveProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
prefix: string[]
|
||||
setPrefix: (prefix: string[]) => void
|
||||
keywords: string
|
||||
setKeywords: (keywords: string) => void
|
||||
startAfter: string
|
||||
setStartAfter: (startAfter: string) => void
|
||||
selectedFileList: string[]
|
||||
setSelectedFileList: (selectedFileList: string[]) => void
|
||||
fileList: OnlineDriveFile[]
|
||||
setFileList: (fileList: OnlineDriveFile[]) => void
|
||||
isInPipeline?: boolean
|
||||
}
|
||||
|
||||
const OnlineDrive = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
setSelectedFileList,
|
||||
fileList,
|
||||
setFileList,
|
||||
isInPipeline = false,
|
||||
}: OnlineDriveProps) => {
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const prefix = useDataSourceStore(state => state.prefix)
|
||||
const setPrefix = useDataSourceStore(state => state.setPrefix)
|
||||
const keywords = useDataSourceStore(state => state.keywords)
|
||||
const setKeywords = useDataSourceStore(state => state.setKeywords)
|
||||
const bucket = useDataSourceStore(state => state.bucket)
|
||||
const setBucket = useDataSourceStore(state => state.setBucket)
|
||||
const startAfter = useDataSourceStore(state => state.startAfter)
|
||||
const setStartAfter = useDataSourceStore(state => state.setStartAfter)
|
||||
const selectedFileList = useDataSourceStore(state => state.selectedFileList)
|
||||
const setSelectedFileList = useDataSourceStore(state => state.setSelectedFileList)
|
||||
const fileList = useDataSourceStore(state => state.fileList)
|
||||
const setFileList = useDataSourceStore(state => state.setFileList)
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const getOnlineDrive = useCallback(async () => {
|
||||
ssePost(
|
||||
datasourceNodeRunURL,
|
||||
{
|
||||
body: {
|
||||
inputs: {
|
||||
prefix: prefix.join('/'),
|
||||
bucket,
|
||||
start_after: startAfter,
|
||||
max_keys: 30, // Adjust as needed
|
||||
},
|
||||
datasource_type: DatasourceType.onlineDrive,
|
||||
},
|
||||
},
|
||||
{
|
||||
onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => {
|
||||
console.log('Online Drive documents data:', documentsData)
|
||||
},
|
||||
onDataSourceNodeError: (error: DataSourceNodeErrorResponse) => {
|
||||
Toast.notify({
|
||||
type: 'error',
|
||||
message: error.error,
|
||||
})
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [bucket, datasourceNodeRunURL, prefix, startAfter])
|
||||
|
||||
useEffect(() => {
|
||||
if (!fileList.length)
|
||||
getOnlineDrive()
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [])
|
||||
|
||||
const updateKeywords = useCallback((keywords: string) => {
|
||||
setKeywords(keywords)
|
||||
}, [setKeywords])
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import type { StateCreator } from 'zustand'
|
||||
import type { DocumentItem, FileItem } from '@/models/datasets'
|
||||
import type { DocumentItem, CustomFile as File, FileItem } from '@/models/datasets'
|
||||
|
||||
export type LocalFileSliceShape = {
|
||||
localFileList: FileItem[]
|
||||
|
||||
@ -2,8 +2,8 @@ import type { StateCreator } from 'zustand'
|
||||
import type { DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
|
||||
export type OnlineDocumentSliceShape = {
|
||||
documentData: DataSourceNotionWorkspace[]
|
||||
setDocumentData: (documentData: DataSourceNotionWorkspace[]) => void
|
||||
documentsData: DataSourceNotionWorkspace[]
|
||||
setDocumentsData: (documentData: DataSourceNotionWorkspace[]) => void
|
||||
searchValue: string
|
||||
setSearchValue: (searchValue: string) => void
|
||||
currentWorkspaceId: string
|
||||
@ -14,13 +14,14 @@ export type OnlineDocumentSliceShape = {
|
||||
setCurrentDocument: (document: NotionPage | undefined) => void
|
||||
selectedPagesId: Set<string>
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) => void
|
||||
previewOnlineDocumentRef: React.MutableRefObject<NotionPage | undefined>
|
||||
}
|
||||
|
||||
export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> = (set) => {
|
||||
return ({
|
||||
documentData: [],
|
||||
setDocumentData: (documentData: DataSourceNotionWorkspace[]) => set(() => ({
|
||||
documentData,
|
||||
documentsData: [],
|
||||
setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => set(() => ({
|
||||
documentsData,
|
||||
})),
|
||||
searchValue: '',
|
||||
setSearchValue: (searchValue: string) => set(() => ({
|
||||
@ -42,5 +43,6 @@ export const createOnlineDocumentSlice: StateCreator<OnlineDocumentSliceShape> =
|
||||
setSelectedPagesId: (selectedPagesId: Set<string>) => set(() => ({
|
||||
selectedPagesId,
|
||||
})),
|
||||
previewOnlineDocumentRef: { current: undefined },
|
||||
})
|
||||
}
|
||||
|
||||
@ -12,6 +12,8 @@ export type OnlineDriveSliceShape = {
|
||||
setSelectedFileList: (selectedFileList: string[]) => void
|
||||
fileList: OnlineDriveFile[]
|
||||
setFileList: (fileList: OnlineDriveFile[]) => void
|
||||
bucket: string
|
||||
setBucket: (bucket: string) => void
|
||||
}
|
||||
|
||||
export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set) => {
|
||||
@ -36,5 +38,9 @@ export const createOnlineDriveSlice: StateCreator<OnlineDriveSliceShape> = (set)
|
||||
setFileList: (fileList: OnlineDriveFile[]) => set(() => ({
|
||||
fileList,
|
||||
})),
|
||||
bucket: '',
|
||||
setBucket: (bucket: string) => set(() => ({
|
||||
bucket,
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
@ -13,6 +13,7 @@ export type WebsiteCrawlSliceShape = {
|
||||
setStep: (step: CrawlStep) => void
|
||||
previewIndex: number
|
||||
setPreviewIndex: (index: number) => void
|
||||
previewWebsitePageRef: React.MutableRefObject<CrawlResultItem | undefined>
|
||||
}
|
||||
|
||||
export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (set) => {
|
||||
@ -37,5 +38,6 @@ export const createWebsiteCrawlSlice: StateCreator<WebsiteCrawlSliceShape> = (se
|
||||
setPreviewIndex: (index: number) => set(() => ({
|
||||
previewIndex: index,
|
||||
})),
|
||||
previewWebsitePageRef: { current: undefined },
|
||||
})
|
||||
}
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
'use client'
|
||||
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import type { CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import Header from '@/app/components/datasets/create/website/base/header'
|
||||
import Options from './base/options'
|
||||
@ -20,41 +20,37 @@ import type {
|
||||
DataSourceNodeProcessingResponse,
|
||||
} from '@/types/pipeline'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import { useDataSourceStore } from '../store'
|
||||
|
||||
const I18N_PREFIX = 'datasetCreation.stepOne.website'
|
||||
|
||||
export type WebsiteCrawlProps = {
|
||||
nodeId: string
|
||||
nodeData: DataSourceNodeType
|
||||
crawlResult: CrawlResult | undefined
|
||||
setCrawlResult: (payload: CrawlResult) => void
|
||||
step: CrawlStep
|
||||
setStep: (step: CrawlStep) => void
|
||||
checkedCrawlResult: CrawlResultItem[]
|
||||
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
|
||||
previewIndex?: number
|
||||
onPreview?: (payload: CrawlResultItem, index: number) => void
|
||||
isInPipeline?: boolean
|
||||
}
|
||||
|
||||
const WebsiteCrawl = ({
|
||||
nodeId,
|
||||
nodeData,
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
step,
|
||||
setStep,
|
||||
checkedCrawlResult,
|
||||
onCheckedCrawlResultChange,
|
||||
previewIndex,
|
||||
onPreview,
|
||||
isInPipeline = false,
|
||||
}: WebsiteCrawlProps) => {
|
||||
const { t } = useTranslation()
|
||||
const [controlFoldOptions, setControlFoldOptions] = useState<number>(0)
|
||||
const [totalNum, setTotalNum] = useState(0)
|
||||
const [crawledNum, setCrawledNum] = useState(0)
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
|
||||
const crawlResult = useDataSourceStore(state => state.crawlResult)
|
||||
const setCrawlResult = useDataSourceStore(state => state.setCrawlResult)
|
||||
const step = useDataSourceStore(state => state.step)
|
||||
const setStep = useDataSourceStore(state => state.setStep)
|
||||
const checkedCrawlResult = useDataSourceStore(state => state.websitePages)
|
||||
const setWebsitePages = useDataSourceStore(state => state.setWebsitePages)
|
||||
const previewWebsitePageRef = useDataSourceStore(state => state.previewWebsitePageRef)
|
||||
const previewIndex = useDataSourceStore(state => state.previewIndex)
|
||||
const setCurrentWebsite = useDataSourceStore(state => state.setCurrentWebsite)
|
||||
const setPreviewIndex = useDataSourceStore(state => state.setPreviewIndex)
|
||||
|
||||
const usePreProcessingParams = useRef(!isInPipeline ? usePublishedPipelinePreProcessingParams : useDraftPipelinePreProcessingParams)
|
||||
const { data: paramsConfig, isFetching: isFetchingParams } = usePreProcessingParams.current({
|
||||
@ -70,13 +66,21 @@ const WebsiteCrawl = ({
|
||||
const isInit = step === CrawlStep.init
|
||||
const isCrawlFinished = step === CrawlStep.finished
|
||||
const isRunning = step === CrawlStep.running
|
||||
const [crawlErrorMessage, setCrawlErrorMessage] = useState('')
|
||||
const showError = isCrawlFinished && crawlErrorMessage
|
||||
|
||||
const datasourceNodeRunURL = !isInPipeline
|
||||
? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run`
|
||||
: `/rag/pipelines/${pipelineId}/workflows/draft/datasource/nodes/${nodeId}/run`
|
||||
|
||||
const handleCheckedCrawlResultChange = useCallback((checkedCrawlResult: CrawlResultItem[]) => {
|
||||
setWebsitePages(checkedCrawlResult)
|
||||
previewWebsitePageRef.current = checkedCrawlResult[0]
|
||||
}, [previewWebsitePageRef, setWebsitePages])
|
||||
|
||||
const handlePreview = useCallback((website: CrawlResultItem, index: number) => {
|
||||
setCurrentWebsite(website)
|
||||
setPreviewIndex(index)
|
||||
}, [setCurrentWebsite, setPreviewIndex])
|
||||
|
||||
const handleRun = useCallback(async (value: Record<string, any>) => {
|
||||
setStep(CrawlStep.running)
|
||||
ssePost(
|
||||
@ -106,7 +110,7 @@ const WebsiteCrawl = ({
|
||||
time_consuming: time_consuming ?? 0,
|
||||
}
|
||||
setCrawlResult(crawlResultData)
|
||||
onCheckedCrawlResultChange(crawlData || []) // default select the crawl result
|
||||
handleCheckedCrawlResultChange(crawlData || []) // default select the crawl result
|
||||
setCrawlErrorMessage('')
|
||||
setStep(CrawlStep.finished)
|
||||
},
|
||||
@ -116,7 +120,7 @@ const WebsiteCrawl = ({
|
||||
},
|
||||
},
|
||||
)
|
||||
}, [datasourceNodeRunURL, onCheckedCrawlResultChange, setCrawlResult, setStep, t])
|
||||
}, [datasourceNodeRunURL, handleCheckedCrawlResultChange, setCrawlResult, setStep, t])
|
||||
|
||||
const handleSubmit = useCallback((value: Record<string, any>) => {
|
||||
handleRun(value)
|
||||
@ -165,10 +169,10 @@ const WebsiteCrawl = ({
|
||||
className='mt-2'
|
||||
list={crawlResult?.data || []}
|
||||
checkedList={checkedCrawlResult}
|
||||
onSelectedChange={onCheckedCrawlResultChange}
|
||||
onSelectedChange={handleCheckedCrawlResultChange}
|
||||
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
|
||||
previewIndex={previewIndex}
|
||||
onPreview={onPreview}
|
||||
onPreview={handlePreview}
|
||||
isMultipleChoice={!isInPipeline} // only support single choice in test run
|
||||
/>
|
||||
)}
|
||||
|
||||
@ -4,11 +4,9 @@ import type { DataSourceOption } from '@/app/components/rag-pipeline/components/
|
||||
import { useCallback, useMemo, useRef, useState } from 'react'
|
||||
import { BlockEnum, type Node } from '@/app/components/workflow/types'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets'
|
||||
import type { CrawlResult, CrawlResultItem } from '@/models/datasets'
|
||||
import { CrawlStep } from '@/models/datasets'
|
||||
import produce from 'immer'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import { type OnlineDriveFile, OnlineDriveFileType } from '@/models/pipeline'
|
||||
import { useDataSourceStore } from './data-source/store'
|
||||
|
||||
export const useAddDocumentsSteps = () => {
|
||||
const { t } = useTranslation()
|
||||
@ -58,26 +56,6 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
|
||||
data: node.data,
|
||||
})
|
||||
})
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
// todo: delete mock data
|
||||
options.push({
|
||||
label: 'Google Drive',
|
||||
value: '123456',
|
||||
// @ts-expect-error mock data
|
||||
data: {
|
||||
datasource_parameters: {},
|
||||
datasource_configurations: {},
|
||||
type: BlockEnum.DataSource,
|
||||
title: 'Google Drive',
|
||||
plugin_id: 'langgenius/google-drive',
|
||||
provider_type: 'online_drive',
|
||||
provider_name: 'google_drive',
|
||||
datasource_name: 'google-drive',
|
||||
datasource_label: 'Google Drive',
|
||||
selected: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
return options
|
||||
}, [datasourceNodes])
|
||||
|
||||
@ -85,102 +63,41 @@ export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[])
|
||||
}
|
||||
|
||||
export const useLocalFile = () => {
|
||||
const [fileList, setFileList] = useState<FileItem[]>([])
|
||||
const [currentFile, setCurrentFile] = useState<File | undefined>()
|
||||
|
||||
const previewFile = useRef<DocumentItem>()
|
||||
const fileList = useDataSourceStore(state => state.localFileList)
|
||||
const previewFileRef = useDataSourceStore(state => state.previewLocalFileRef)
|
||||
const currentLocalFile = useDataSourceStore(state => state.currentLocalFile)
|
||||
const setCurrentFile = useDataSourceStore(state => state.setCurrentLocalFile)
|
||||
|
||||
const allFileLoaded = useMemo(() => (fileList.length > 0 && fileList.every(file => file.file.id)), [fileList])
|
||||
|
||||
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
const newList = produce(list, (draft) => {
|
||||
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
|
||||
draft[targetIndex] = {
|
||||
...draft[targetIndex],
|
||||
progress,
|
||||
}
|
||||
})
|
||||
setFileList(newList)
|
||||
previewFile.current = newList[0].file as DocumentItem
|
||||
}
|
||||
|
||||
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
|
||||
setFileList(preparedFiles)
|
||||
}, [])
|
||||
|
||||
const updateCurrentFile = useCallback((file: File) => {
|
||||
setCurrentFile(file)
|
||||
}, [])
|
||||
|
||||
const hideFilePreview = useCallback(() => {
|
||||
const hidePreviewLocalFile = useCallback(() => {
|
||||
setCurrentFile(undefined)
|
||||
}, [])
|
||||
}, [setCurrentFile])
|
||||
|
||||
return {
|
||||
fileList,
|
||||
previewFile,
|
||||
previewFileRef,
|
||||
allFileLoaded,
|
||||
updateFile,
|
||||
updateFileList,
|
||||
currentFile,
|
||||
updateCurrentFile,
|
||||
hideFilePreview,
|
||||
currentLocalFile,
|
||||
hidePreviewLocalFile,
|
||||
}
|
||||
}
|
||||
|
||||
export const useOnlineDocuments = () => {
|
||||
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
|
||||
const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
|
||||
const [currentDocument, setCurrentDocument] = useState<NotionPage | undefined>()
|
||||
const onlineDocuments = useDataSourceStore(state => state.onlineDocuments)
|
||||
const previewOnlineDocumentRef = useDataSourceStore(state => state.previewOnlineDocumentRef)
|
||||
const setCurrentDocument = useDataSourceStore(state => state.setCurrentDocument)
|
||||
const currentDocument = useDataSourceStore(state => state.currentDocument)
|
||||
|
||||
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
|
||||
const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
}, {})
|
||||
return pagesMap
|
||||
}, [documentsData])
|
||||
const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
|
||||
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
|
||||
|
||||
const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
|
||||
|
||||
const updateOnlineDocuments = (value: NotionPage[]) => {
|
||||
setOnlineDocuments(value)
|
||||
}
|
||||
|
||||
const updateCurrentPage = useCallback((page: NotionPage) => {
|
||||
setCurrentDocument(page)
|
||||
}, [])
|
||||
|
||||
const hideOnlineDocumentPreview = useCallback(() => {
|
||||
const hidePreviewOnlineDocument = useCallback(() => {
|
||||
setCurrentDocument(undefined)
|
||||
}, [])
|
||||
}, [setCurrentDocument])
|
||||
|
||||
return {
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
onlineDocuments,
|
||||
previewOnlineDocument,
|
||||
updateOnlineDocuments,
|
||||
currentDocument,
|
||||
updateCurrentPage,
|
||||
hideOnlineDocumentPreview,
|
||||
previewOnlineDocumentRef,
|
||||
hidePreviewOnlineDocument,
|
||||
}
|
||||
}
|
||||
|
||||
@ -224,28 +141,5 @@ export const useWebsiteCrawl = () => {
|
||||
}
|
||||
|
||||
export const useOnlineDrive = () => {
|
||||
const [prefix, setPrefix] = useState<string[]>([])
|
||||
const [keywords, setKeywords] = useState('')
|
||||
const [startAfter, setStartAfter] = useState('')
|
||||
const [selectedFileList, setSelectedFileList] = useState<string[]>([])
|
||||
const [fileList, setFileList] = useState<OnlineDriveFile[]>([
|
||||
{
|
||||
key: 'Bucket_1',
|
||||
size: 1024, // unit bytes
|
||||
type: OnlineDriveFileType.bucket,
|
||||
},
|
||||
])
|
||||
|
||||
return {
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
setSelectedFileList,
|
||||
fileList,
|
||||
setFileList,
|
||||
}
|
||||
return {}
|
||||
}
|
||||
|
||||
@ -27,7 +27,7 @@ import Processing from './processing'
|
||||
import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
|
||||
import { DatasourceType } from '@/models/pipeline'
|
||||
import { TransferMethod } from '@/types/app'
|
||||
import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useOnlineDrive, useWebsiteCrawl } from './hooks'
|
||||
import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
|
||||
import DataSourceProvider from './data-source/store/provider'
|
||||
|
||||
const CreateFormPipeline = () => {
|
||||
@ -53,56 +53,24 @@ const CreateFormPipeline = () => {
|
||||
} = useAddDocumentsSteps()
|
||||
const {
|
||||
fileList,
|
||||
previewFile,
|
||||
previewFileRef,
|
||||
allFileLoaded,
|
||||
updateFile,
|
||||
updateFileList,
|
||||
currentFile,
|
||||
updateCurrentFile,
|
||||
hideFilePreview,
|
||||
currentLocalFile,
|
||||
hidePreviewLocalFile,
|
||||
} = useLocalFile()
|
||||
const {
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
onlineDocuments,
|
||||
previewOnlineDocument,
|
||||
updateOnlineDocuments,
|
||||
currentDocument,
|
||||
updateCurrentPage,
|
||||
hideOnlineDocumentPreview,
|
||||
previewOnlineDocumentRef,
|
||||
hidePreviewOnlineDocument,
|
||||
} = useOnlineDocuments()
|
||||
const {
|
||||
websitePages,
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
step,
|
||||
setStep,
|
||||
previewWebsitePage,
|
||||
updataCheckedCrawlResultChange,
|
||||
currentWebsite,
|
||||
updateCurrentWebsite,
|
||||
previewIndex,
|
||||
hideWebsitePreview,
|
||||
} = useWebsiteCrawl()
|
||||
const {
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
setSelectedFileList,
|
||||
fileList: onlineDriveFileList,
|
||||
setFileList,
|
||||
} = useOnlineDrive()
|
||||
// const { } = useOnlineDrive()
|
||||
|
||||
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
|
||||
const isShowVectorSpaceFull = allFileLoaded && isVectorSpaceFull && enableBilling
|
||||
@ -127,7 +95,7 @@ const CreateFormPipeline = () => {
|
||||
return
|
||||
const datasourceInfoList: Record<string, any>[] = []
|
||||
if (datasourceType === DatasourceType.localFile) {
|
||||
const { id, name, type, size, extension, mime_type } = previewFile.current as File
|
||||
const { id, name, type, size, extension, mime_type } = previewFileRef.current as File
|
||||
const documentInfo = {
|
||||
related_id: id,
|
||||
name,
|
||||
@ -141,7 +109,7 @@ const CreateFormPipeline = () => {
|
||||
datasourceInfoList.push(documentInfo)
|
||||
}
|
||||
if (datasourceType === DatasourceType.onlineDocument) {
|
||||
const { workspace_id, ...rest } = previewOnlineDocument.current
|
||||
const { workspace_id, ...rest } = previewOnlineDocumentRef.current!
|
||||
const documentInfo = {
|
||||
workspace_id,
|
||||
page: rest,
|
||||
@ -162,7 +130,7 @@ const CreateFormPipeline = () => {
|
||||
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
|
||||
},
|
||||
})
|
||||
}, [datasource, datasourceType, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline])
|
||||
}, [datasource, datasourceType, pipelineId, previewFileRef, previewOnlineDocumentRef, previewWebsitePage, runPublishedPipeline])
|
||||
|
||||
const handleProcess = useCallback(async (data: Record<string, any>) => {
|
||||
if (!datasource)
|
||||
@ -230,14 +198,14 @@ const CreateFormPipeline = () => {
|
||||
}, [handlePreviewChunks, handleProcess])
|
||||
|
||||
const handlePreviewFileChange = useCallback((file: DocumentItem) => {
|
||||
previewFile.current = file
|
||||
previewFileRef.current = file
|
||||
onClickPreview()
|
||||
}, [onClickPreview, previewFile])
|
||||
}, [onClickPreview, previewFileRef])
|
||||
|
||||
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
|
||||
previewOnlineDocument.current = page
|
||||
previewOnlineDocumentRef.current = page
|
||||
onClickPreview()
|
||||
}, [onClickPreview, previewOnlineDocument])
|
||||
}, [onClickPreview, previewOnlineDocumentRef])
|
||||
|
||||
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
|
||||
previewWebsitePage.current = website
|
||||
@ -272,59 +240,26 @@ const CreateFormPipeline = () => {
|
||||
/>
|
||||
{datasourceType === DatasourceType.localFile && (
|
||||
<LocalFile
|
||||
fileList={fileList}
|
||||
allowedExtensions={datasource!.nodeData.fileExtensions || []}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
onPreview={updateCurrentFile}
|
||||
notSupportBatchUpload={notSupportBatchUpload}
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.onlineDocument && (
|
||||
<OnlineDocuments
|
||||
documentsData={documentsData}
|
||||
setDocumentsData={setDocumentsData}
|
||||
searchValue={searchValue}
|
||||
setSearchValue={setSearchValue}
|
||||
currentWorkspaceId={currentWorkspaceId}
|
||||
setCurrentWorkspaceId={setCurrentWorkspaceId}
|
||||
PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
|
||||
selectedPagesId={selectedPagesId}
|
||||
setSelectedPagesId={setSelectedPagesId}
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
onSelect={updateOnlineDocuments}
|
||||
onPreview={updateCurrentPage}
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.websiteCrawl && (
|
||||
<WebsiteCrawl
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
crawlResult={crawlResult}
|
||||
setCrawlResult={setCrawlResult}
|
||||
step={step}
|
||||
setStep={setStep}
|
||||
checkedCrawlResult={websitePages}
|
||||
onCheckedCrawlResultChange={updataCheckedCrawlResultChange}
|
||||
onPreview={updateCurrentWebsite}
|
||||
previewIndex={previewIndex}
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.onlineDrive && (
|
||||
<OnlineDrive
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
prefix={prefix}
|
||||
setPrefix={setPrefix}
|
||||
keywords={keywords}
|
||||
setKeywords={setKeywords}
|
||||
startAfter={startAfter}
|
||||
setStartAfter={setStartAfter}
|
||||
selectedFileList={selectedFileList}
|
||||
setSelectedFileList={setSelectedFileList}
|
||||
fileList={onlineDriveFileList}
|
||||
setFileList={setFileList}
|
||||
/>
|
||||
)}
|
||||
{isShowVectorSpaceFull && (
|
||||
@ -362,15 +297,25 @@ const CreateFormPipeline = () => {
|
||||
currentStep === 1 && (
|
||||
<div className='h-full min-w-0 flex-1'>
|
||||
<div className='flex h-full flex-col pl-2 pt-2'>
|
||||
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
|
||||
{currentLocalFile && (
|
||||
<FilePreview
|
||||
file={currentLocalFile}
|
||||
hidePreview={hidePreviewLocalFile}
|
||||
/>
|
||||
)}
|
||||
{currentDocument && (
|
||||
<OnlineDocumentPreview
|
||||
datasourceNodeId={datasource!.nodeId}
|
||||
currentPage={currentDocument}
|
||||
hidePreview={hideOnlineDocumentPreview}
|
||||
hidePreview={hidePreviewOnlineDocument}
|
||||
/>
|
||||
)}
|
||||
{currentWebsite && (
|
||||
<WebsitePreview
|
||||
currentWebsite={currentWebsite}
|
||||
hidePreview={hideWebsitePreview}
|
||||
/>
|
||||
)}
|
||||
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
'use client'
|
||||
import React from 'react'
|
||||
import React, { useMemo } from 'react'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import Loading from './loading'
|
||||
import type { CustomFile as File } from '@/models/datasets'
|
||||
import { RiCloseLine } from '@remixicon/react'
|
||||
import { useFilePreview } from '@/service/use-common'
|
||||
import DocumentFileIcon from '../../../common/document-file-icon'
|
||||
import { formatNumberAbbreviated } from '@/utils/format'
|
||||
import { formatFileSize, formatNumberAbbreviated } from '@/utils/format'
|
||||
|
||||
type FilePreviewProps = {
|
||||
file: File
|
||||
@ -20,31 +20,19 @@ const FilePreview = ({
|
||||
const { t } = useTranslation()
|
||||
const { data: fileData, isFetching } = useFilePreview(file.id || '')
|
||||
|
||||
const getFileName = (currentFile?: File) => {
|
||||
if (!currentFile)
|
||||
const fileName = useMemo(() => {
|
||||
if (!file)
|
||||
return ''
|
||||
const arr = currentFile.name.split('.')
|
||||
const arr = file.name.split('.')
|
||||
return arr.slice(0, -1).join()
|
||||
}
|
||||
|
||||
const getFileSize = (fileSize: number) => {
|
||||
if (!fileSize)
|
||||
return fileSize
|
||||
const units = ['', 'K', 'M', 'G', 'T', 'P']
|
||||
let index = 0
|
||||
while (fileSize >= 1024 && index < units.length) {
|
||||
fileSize = fileSize / 1024
|
||||
index++
|
||||
}
|
||||
return `${fileSize.toFixed(2)} ${units[index]}B`
|
||||
}
|
||||
}, [file])
|
||||
|
||||
return (
|
||||
<div className='flex h-full w-full flex-col rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
|
||||
<div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
|
||||
<div className='flex grow flex-col gap-y-1'>
|
||||
<div className='system-2xs-semibold-uppercase text-text-accent'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{`${fileName}.${file.extension || ''}`}</div>
|
||||
<div className='system-xs-medium flex items-center gap-x-1 text-text-tertiary'>
|
||||
<DocumentFileIcon
|
||||
className='size-3.5 shrink-0'
|
||||
@ -53,7 +41,7 @@ const FilePreview = ({
|
||||
/>
|
||||
<span className='uppercase'>{file.extension}</span>
|
||||
<span>·</span>
|
||||
<span>{getFileSize(file.size)}</span>
|
||||
<span>{formatFileSize(file.size)}</span>
|
||||
{fileData && (
|
||||
<>
|
||||
<span>·</span>
|
||||
|
||||
@ -6,12 +6,12 @@ import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
|
||||
import { formatNumberAbbreviated } from '@/utils/format'
|
||||
|
||||
type WebsitePreviewProps = {
|
||||
payload: CrawlResultItem
|
||||
currentWebsite: CrawlResultItem
|
||||
hidePreview: () => void
|
||||
}
|
||||
|
||||
const WebsitePreview = ({
|
||||
payload,
|
||||
currentWebsite,
|
||||
hidePreview,
|
||||
}: WebsitePreviewProps) => {
|
||||
const { t } = useTranslation()
|
||||
@ -21,13 +21,13 @@ const WebsitePreview = ({
|
||||
<div className='flex gap-x-2 border-b border-divider-subtle pb-3 pl-6 pr-4 pt-4'>
|
||||
<div className='flex grow flex-col gap-y-1'>
|
||||
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
|
||||
<div className='title-md-semi-bold text-tex-primary'>{currentWebsite.title}</div>
|
||||
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
|
||||
<RiGlobalLine className='size-3.5' />
|
||||
<span className='uppercase' title={payload.source_url}>{payload.source_url}</span>
|
||||
<span className='uppercase' title={currentWebsite.source_url}>{currentWebsite.source_url}</span>
|
||||
<span>·</span>
|
||||
<span>·</span>
|
||||
<span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
|
||||
<span>{`${formatNumberAbbreviated(currentWebsite.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
@ -39,7 +39,7 @@ const WebsitePreview = ({
|
||||
</button>
|
||||
</div>
|
||||
<div className='body-md-regular grow overflow-hidden px-6 py-5 text-text-secondary'>
|
||||
{payload.markdown}
|
||||
{currentWebsite.markdown}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
|
||||
@ -5,12 +5,7 @@ import { useNodes } from 'reactflow'
|
||||
import { BlockEnum } from '@/app/components/workflow/types'
|
||||
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
|
||||
import { useCallback, useMemo, useState } from 'react'
|
||||
import type { CrawlResult } from '@/models/datasets'
|
||||
import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets'
|
||||
import produce from 'immer'
|
||||
import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common'
|
||||
import type { OnlineDriveFile } from '@/models/pipeline'
|
||||
import { OnlineDriveFileType } from '@/models/pipeline'
|
||||
|
||||
export const useTestRunSteps = () => {
|
||||
const { t } = useTranslation()
|
||||
@ -57,131 +52,27 @@ export const useDatasourceOptions = () => {
|
||||
data: node.data,
|
||||
})
|
||||
})
|
||||
if (process.env.NODE_ENV === 'development') {
|
||||
// todo: delete mock data
|
||||
options.push({
|
||||
label: 'Google Drive',
|
||||
value: '123456',
|
||||
// @ts-expect-error mock data
|
||||
data: {
|
||||
datasource_parameters: {},
|
||||
datasource_configurations: {},
|
||||
type: BlockEnum.DataSource,
|
||||
title: 'Google Drive',
|
||||
plugin_id: 'langgenius/google-drive',
|
||||
provider_type: 'online_drive',
|
||||
provider_name: 'google_drive',
|
||||
datasource_name: 'google-drive',
|
||||
datasource_label: 'Google Drive',
|
||||
selected: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
return options
|
||||
}, [datasourceNodes])
|
||||
|
||||
return options
|
||||
}
|
||||
|
||||
export const useLocalFile = () => {
|
||||
const [fileList, setFileList] = useState<FileItem[]>([])
|
||||
|
||||
const updateFile = (fileItem: FileItem, progress: number, list: FileItem[]) => {
|
||||
const newList = produce(list, (draft) => {
|
||||
const targetIndex = draft.findIndex(file => file.fileID === fileItem.fileID)
|
||||
draft[targetIndex] = {
|
||||
...draft[targetIndex],
|
||||
progress,
|
||||
}
|
||||
})
|
||||
setFileList(newList)
|
||||
}
|
||||
|
||||
const updateFileList = (preparedFiles: FileItem[]) => {
|
||||
setFileList(preparedFiles)
|
||||
}
|
||||
|
||||
return {
|
||||
fileList,
|
||||
updateFile,
|
||||
updateFileList,
|
||||
}
|
||||
}
|
||||
|
||||
export const useOnlineDocuments = () => {
|
||||
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
|
||||
const [searchValue, setSearchValue] = useState('')
|
||||
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
|
||||
const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
|
||||
|
||||
const PagesMapAndSelectedPagesId: DataSourceNotionPageMap = useMemo(() => {
|
||||
const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
|
||||
next.pages.forEach((page) => {
|
||||
prev[page.page_id] = {
|
||||
...page,
|
||||
workspace_id: next.workspace_id,
|
||||
}
|
||||
})
|
||||
|
||||
return prev
|
||||
}, {})
|
||||
return pagesMap
|
||||
}, [documentsData])
|
||||
const defaultSelectedPagesId = [...(onlineDocuments.map(doc => doc.page_id) || [])]
|
||||
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
|
||||
|
||||
const updateOnlineDocuments = (value: NotionPage[]) => {
|
||||
setOnlineDocuments(value)
|
||||
}
|
||||
|
||||
return {
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
onlineDocuments,
|
||||
updateOnlineDocuments,
|
||||
}
|
||||
}
|
||||
|
||||
export const useWebsiteCrawl = () => {
|
||||
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
|
||||
const [crawlResult, setCrawlResult] = useState<CrawlResult | undefined>()
|
||||
const [step, setStep] = useState<CrawlStep>(CrawlStep.init)
|
||||
|
||||
return {
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
websitePages,
|
||||
setWebsitePages,
|
||||
step,
|
||||
setStep,
|
||||
}
|
||||
}
|
||||
|
||||
export const useOnlineDrive = () => {
|
||||
const [prefix, setPrefix] = useState<string[]>([])
|
||||
const [keywords, setKeywords] = useState('')
|
||||
const [bucket, setBucket] = useState('')
|
||||
const [startAfter, setStartAfter] = useState('')
|
||||
const [selectedFileList, setSelectedFileList] = useState<string[]>([])
|
||||
const [fileList, setFileList] = useState<OnlineDriveFile[]>([
|
||||
{
|
||||
key: 'Bucket_1',
|
||||
size: 1024, // unit bytes
|
||||
type: OnlineDriveFileType.bucket,
|
||||
},
|
||||
])
|
||||
const [fileList, setFileList] = useState<OnlineDriveFile[]>([])
|
||||
|
||||
return {
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
bucket,
|
||||
setBucket,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
|
||||
import { useCallback, useMemo, useState } from 'react'
|
||||
import { useLocalFile, useOnlineDocuments, useOnlineDrive, useTestRunSteps, useWebsiteCrawl } from './hooks'
|
||||
import { useTestRunSteps } from './hooks'
|
||||
import DataSourceOptions from './data-source-options'
|
||||
import LocalFile from '@/app/components/datasets/documents/create-from-pipeline/data-source/local-file'
|
||||
import OnlineDocuments from '@/app/components/datasets/documents/create-from-pipeline/data-source/online-documents'
|
||||
@ -16,9 +16,14 @@ import CloseButton from './close-button'
|
||||
import Header from './header'
|
||||
import FooterTips from './footer-tips'
|
||||
import DataSourceProvider from '@/app/components/datasets/documents/create-from-pipeline/data-source/store/provider'
|
||||
import { useDataSourceStore } from '@/app/components/datasets/documents/create-from-pipeline/data-source/store'
|
||||
|
||||
const TestRunPanel = () => {
|
||||
const setShowDebugAndPreviewPanel = useWorkflowStoreWithSelector(state => state.setShowDebugAndPreviewPanel)
|
||||
const fileList = useDataSourceStore(state => state.localFileList)
|
||||
const onlineDocuments = useDataSourceStore(state => state.onlineDocuments)
|
||||
const websitePages = useDataSourceStore(state => state.websitePages)
|
||||
const selectedFileList = useDataSourceStore(state => state.selectedFileList)
|
||||
const [datasource, setDatasource] = useState<Datasource>()
|
||||
|
||||
const {
|
||||
@ -27,45 +32,6 @@ const TestRunPanel = () => {
|
||||
handleNextStep,
|
||||
handleBackStep,
|
||||
} = useTestRunSteps()
|
||||
const {
|
||||
fileList,
|
||||
updateFile,
|
||||
updateFileList,
|
||||
} = useLocalFile()
|
||||
const {
|
||||
documentsData,
|
||||
setDocumentsData,
|
||||
searchValue,
|
||||
setSearchValue,
|
||||
currentWorkspaceId,
|
||||
setCurrentWorkspaceId,
|
||||
PagesMapAndSelectedPagesId,
|
||||
selectedPagesId,
|
||||
setSelectedPagesId,
|
||||
onlineDocuments,
|
||||
updateOnlineDocuments,
|
||||
} = useOnlineDocuments()
|
||||
const {
|
||||
crawlResult,
|
||||
setCrawlResult,
|
||||
websitePages,
|
||||
setWebsitePages,
|
||||
step,
|
||||
setStep,
|
||||
} = useWebsiteCrawl()
|
||||
const {
|
||||
prefix,
|
||||
setPrefix,
|
||||
keywords,
|
||||
setKeywords,
|
||||
startAfter,
|
||||
setStartAfter,
|
||||
selectedFileList,
|
||||
setSelectedFileList,
|
||||
fileList: onlineDriveFileList,
|
||||
setFileList,
|
||||
} = useOnlineDrive()
|
||||
const { handleRun } = useWorkflowRun()
|
||||
|
||||
const datasourceType = datasource?.nodeData.provider_type
|
||||
|
||||
@ -77,12 +43,16 @@ const TestRunPanel = () => {
|
||||
return !onlineDocuments.length
|
||||
if (datasourceType === DatasourceType.websiteCrawl)
|
||||
return !websitePages.length
|
||||
if (datasourceType === DatasourceType.onlineDrive)
|
||||
return !selectedFileList.length
|
||||
return false
|
||||
}, [datasource, datasourceType, fileList, onlineDocuments.length, websitePages.length])
|
||||
}, [datasource, datasourceType, fileList, onlineDocuments.length, selectedFileList.length, websitePages.length])
|
||||
|
||||
const handleClose = () => {
|
||||
const handleClose = useCallback(() => {
|
||||
setShowDebugAndPreviewPanel(false)
|
||||
}
|
||||
}, [setShowDebugAndPreviewPanel])
|
||||
|
||||
const { handleRun } = useWorkflowRun()
|
||||
|
||||
const handleProcess = useCallback((data: Record<string, any>) => {
|
||||
if (!datasource)
|
||||
@ -137,57 +107,29 @@ const TestRunPanel = () => {
|
||||
/>
|
||||
{datasourceType === DatasourceType.localFile && (
|
||||
<LocalFile
|
||||
fileList={fileList}
|
||||
allowedExtensions={datasource!.nodeData.fileExtensions || []}
|
||||
prepareFileList={updateFileList}
|
||||
onFileListUpdate={updateFileList}
|
||||
onFileUpdate={updateFile}
|
||||
notSupportBatchUpload={false} // only support single file upload in test run
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.onlineDocument && (
|
||||
<OnlineDocuments
|
||||
documentsData={documentsData}
|
||||
setDocumentsData={setDocumentsData}
|
||||
searchValue={searchValue}
|
||||
setSearchValue={setSearchValue}
|
||||
currentWorkspaceId={currentWorkspaceId}
|
||||
setCurrentWorkspaceId={setCurrentWorkspaceId}
|
||||
PagesMapAndSelectedPagesId={PagesMapAndSelectedPagesId}
|
||||
selectedPagesId={selectedPagesId}
|
||||
setSelectedPagesId={setSelectedPagesId}
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
onSelect={updateOnlineDocuments}
|
||||
isInPipeline
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.websiteCrawl && (
|
||||
<WebsiteCrawl
|
||||
nodeId={datasource!.nodeId}
|
||||
checkedCrawlResult={websitePages}
|
||||
nodeData={datasource!.nodeData}
|
||||
crawlResult={crawlResult}
|
||||
setCrawlResult={setCrawlResult}
|
||||
step={step}
|
||||
setStep={setStep}
|
||||
onCheckedCrawlResultChange={setWebsitePages}
|
||||
isInPipeline
|
||||
/>
|
||||
)}
|
||||
{datasourceType === DatasourceType.onlineDrive && (
|
||||
<OnlineDrive
|
||||
nodeId={datasource!.nodeId}
|
||||
nodeData={datasource!.nodeData}
|
||||
prefix={prefix}
|
||||
setPrefix={setPrefix}
|
||||
keywords={keywords}
|
||||
setKeywords={setKeywords}
|
||||
startAfter={startAfter}
|
||||
setStartAfter={setStartAfter}
|
||||
selectedFileList={selectedFileList}
|
||||
setSelectedFileList={setSelectedFileList}
|
||||
fileList={onlineDriveFileList}
|
||||
setFileList={setFileList}
|
||||
isInPipeline
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
Loading…
Reference in New Issue
Block a user