opensource
/
dify
mirror of https://github.com/langgenius/dify.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: refactor online documents handling and update related components

This commit is contained in:
twwu 2025-06-06 10:08:19 +08:00
parent bce2bdd0de
commit cbf0864edc
7 changed files with 96 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
web/app/components/datasets/documents/create-from-pipeline/hooks.ts
View File

@ -119,31 +119,31 @@ export const useLocalFile = () => {
}
}
export const useNotionsPages = () => {
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
export const useOnlineDocuments = () => {
const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
const [currentDocuments, setCurrentDocuments] = useState<NotionPage | undefined>()
const previewNotionPage = useRef<NotionPage>(notionPages[0])
const previewOnlineDocument = useRef<NotionPage>(onlineDocuments[0])
const updateNotionPages = (value: NotionPage[]) => {
setNotionPages(value)
const updateOnlineDocuments = (value: NotionPage[]) => {
setOnlineDocuments(value)
}
const updateCurrentPage = useCallback((page: NotionPage) => {
setCurrentNotionPage(page)
setCurrentDocuments(page)
}, [])
const hideNotionPagePreview = useCallback(() => {
setCurrentNotionPage(undefined)
const hideOnlineDocumentPreview = useCallback(() => {
setCurrentDocuments(undefined)
}, [])
return {
notionPages,
previewNotionPage,
updateNotionPages,
currentNotionPage,
onlineDocuments,
previewOnlineDocument,
updateOnlineDocuments,
currentDocuments,
updateCurrentPage,
hideNotionPagePreview,
hideOnlineDocumentPreview,
}
}

46
web/app/components/datasets/documents/create-from-pipeline/index.tsx
View File

@ -5,7 +5,7 @@ import type { CrawlResultItem, DocumentItem, CustomFile as File, FileIndexingEst
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
import { useProviderContextSelector } from '@/context/provider-context'
import type { NotionPage } from '@/models/common'
import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl'
import Actions from './data-source/actions'
@ -26,7 +26,7 @@ import Processing from './processing'
import type { InitialDocumentDetail, PublishedPipelineRunPreviewResponse, PublishedPipelineRunResponse } from '@/models/pipeline'
import { DatasourceType } from '@/models/pipeline'
import { TransferMethod } from '@/types/app'
import { useAddDocumentsSteps, useLocalFile, useNotionsPages, useWebsiteCrawl } from './hooks'
import { useAddDocumentsSteps, useLocalFile, useOnlineDocuments, useWebsiteCrawl } from './hooks'
const CreateFormPipeline = () => {
const { t } = useTranslation()
@ -63,13 +63,13 @@ const CreateFormPipeline = () => {
hideFilePreview,
} = useLocalFile()
const {
notionPages,
previewNotionPage,
updateNotionPages,
currentNotionPage,
onlineDocuments,
previewOnlineDocument,
updateOnlineDocuments,
currentDocuments,
updateCurrentPage,
hideNotionPagePreview,
} = useNotionsPages()
hideOnlineDocumentPreview,
} = useOnlineDocuments()
const {
websitePages,
websiteCrawlJobId,
@ -90,11 +90,11 @@ const CreateFormPipeline = () => {
if (datasource.type === DatasourceType.localFile)
return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
if (datasource.type === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !notionPages.length
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasource.type === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
return false
}, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
}, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
const { mutateAsync: runPublishedPipeline, isIdle, isPending } = useRunPublishedPipeline()
@ -117,7 +117,7 @@ const CreateFormPipeline = () => {
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.onlineDocument) {
const { workspace_id, ...rest } = previewNotionPage.current
const { workspace_id, ...rest } = previewOnlineDocument.current
const documentInfo = {
workspace_id,
page: rest,
@ -143,7 +143,7 @@ const CreateFormPipeline = () => {
setEstimateData((res as PublishedPipelineRunPreviewResponse).data.outputs)
},
})
}, [datasource, pipelineId, previewFile, previewNotionPage, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
}, [datasource, pipelineId, previewFile, previewOnlineDocument, previewWebsitePage, runPublishedPipeline, websiteCrawlJobId])
const handleProcess = useCallback(async (data: Record<string, any>) => {
if (!datasource)
@ -166,7 +166,7 @@ const CreateFormPipeline = () => {
})
}
if (datasource.type === DatasourceType.onlineDocument) {
notionPages.forEach((page) => {
onlineDocuments.forEach((page) => {
const { workspace_id, ...rest } = page
const documentInfo = {
workspace_id,
@ -196,7 +196,7 @@ const CreateFormPipeline = () => {
handleNextStep()
},
})
}, [datasource, fileList, handleNextStep, notionPages, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
}, [datasource, fileList, handleNextStep, onlineDocuments, pipelineId, runPublishedPipeline, websiteCrawlJobId, websitePages])
const onClickProcess = useCallback(() => {
isPreview.current = false
@ -217,10 +217,10 @@ const CreateFormPipeline = () => {
onClickPreview()
}, [onClickPreview, previewFile])
const handlePreviewNotionPageChange = useCallback((page: NotionPage) => {
previewNotionPage.current = page
const handlePreviewOnlineDocumentChange = useCallback((page: NotionPage) => {
previewOnlineDocument.current = page
onClickPreview()
}, [onClickPreview, previewNotionPage])
}, [onClickPreview, previewOnlineDocument])
const handlePreviewWebsiteChange = useCallback((website: CrawlResultItem) => {
previewWebsitePage.current = website
@ -263,15 +263,15 @@ const CreateFormPipeline = () => {
/>
)}
{datasource?.type === DatasourceType.onlineDocument && (
<Notion
<OnlineDocuments
nodeId={datasource?.nodeId || ''}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onlineDocuments={onlineDocuments}
updateOnlineDocuments={updateOnlineDocuments}
canPreview
onPreview={updateCurrentPage}
/>
@ -327,7 +327,7 @@ const CreateFormPipeline = () => {
currentStep === 1 && (
<div className='flex h-full w-[752px] shrink-0 pl-2 pt-2'>
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
{currentDocuments && <NotionPagePreview currentPage={currentDocuments} hidePreview={hideOnlineDocumentPreview} />}
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
</div>
)
@ -339,14 +339,14 @@ const CreateFormPipeline = () => {
<ChunkPreview
datasource={datasource!}
files={fileList.map(file => file.file)}
notionPages={notionPages}
onlineDocuments={onlineDocuments}
websitePages={websitePages}
isIdle={isIdle && isPreview.current}
isPending={isPending && isPreview.current}
estimateData={estimateData}
onPreview={onClickPreview}
handlePreviewFileChange={handlePreviewFileChange}
handlePreviewNotionPageChange={handlePreviewNotionPageChange}
handlePreviewOnlineDocumentChange={handlePreviewOnlineDocumentChange}
handlePreviewWebsitePageChange={handlePreviewWebsiteChange}
/>
)}

22
web/app/components/datasets/documents/create-from-pipeline/preview/chunk-preview.tsx
View File

@ -20,35 +20,35 @@ import { DatasourceType } from '@/models/pipeline'
type ChunkPreviewProps = {
datasource: Datasource
files: CustomFile[]
notionPages: NotionPage[]
onlineDocuments: NotionPage[]
websitePages: CrawlResultItem[]
isIdle: boolean
isPending: boolean
estimateData: FileIndexingEstimateResponse | undefined
onPreview: () => void
handlePreviewFileChange: (file: DocumentItem) => void
handlePreviewNotionPageChange: (page: NotionPage) => void
handlePreviewOnlineDocumentChange: (page: NotionPage) => void
handlePreviewWebsitePageChange: (page: CrawlResultItem) => void
}
const ChunkPreview = ({
datasource,
files,
notionPages,
onlineDocuments,
websitePages,
isIdle,
isPending,
estimateData,
onPreview,
handlePreviewFileChange,
handlePreviewNotionPageChange,
handlePreviewOnlineDocumentChange,
handlePreviewWebsitePageChange,
}: ChunkPreviewProps) => {
const { t } = useTranslation()
const currentDocForm = useDatasetDetailContextWithSelector(s => s.dataset?.doc_form)
const [previewFile, setPreviewFile] = useState<DocumentItem>(files[0] as DocumentItem)
const [previewNotionPage, setPreviewNotionPage] = useState<NotionPage>(notionPages[0])
const [previewOnlineDocument, setPreviewOnlineDocument] = useState<NotionPage>(onlineDocuments[0])
const [previewWebsitePage, setPreviewWebsitePage] = useState<CrawlResultItem>(websitePages[0])
const dataSourceType = datasource?.type
@ -72,20 +72,20 @@ const ChunkPreview = ({
{dataSourceType === DatasourceType.onlineDocument
&& <PreviewDocumentPicker
files={
notionPages.map(page => ({
onlineDocuments.map(page => ({
id: page.page_id,
name: page.page_name,
extension: 'md',
}))
}
onChange={(selected) => {
const selectedPage = notionPages.find(page => page.page_id === selected.id)
setPreviewNotionPage(selectedPage!)
handlePreviewNotionPageChange(selectedPage!)
const selectedPage = onlineDocuments.find(page => page.page_id === selected.id)
setPreviewOnlineDocument(selectedPage!)
handlePreviewOnlineDocumentChange(selectedPage!)
}}
value={{
id: previewNotionPage?.page_id || '',
name: previewNotionPage?.page_name || '',
id: previewOnlineDocument?.page_id || '',
name: previewOnlineDocument?.page_name || '',
extension: 'md',
}}
/>

24
web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx → web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx
View File

@ -1,35 +1,35 @@
import type { NotionPage } from '@/models/common'
import NotionPageSelector from './notion-page-selector'
import OnlineDocumentSelector from './online-document-selector'
type NotionProps = {
type OnlineDocumentsProps = {
nodeId: string
headerInfo: {
title: string
docTitle: string
docLink: string
}
notionPages: NotionPage[]
updateNotionPages: (value: NotionPage[]) => void
onlineDocuments: NotionPage[]
updateOnlineDocuments: (value: NotionPage[]) => void
canPreview?: boolean
onPreview?: (selectedPage: NotionPage) => void
isInPipeline?: boolean
}
const Notion = ({
const OnlineDocuments = ({
nodeId,
headerInfo,
notionPages,
updateNotionPages,
onlineDocuments,
updateOnlineDocuments,
canPreview = false,
onPreview,
isInPipeline = false,
}: NotionProps) => {
}: OnlineDocumentsProps) => {
return (
<NotionPageSelector
<OnlineDocumentSelector
nodeId={nodeId}
headerInfo={headerInfo}
value={notionPages.map(page => page.page_id)}
onSelect={updateNotionPages}
value={onlineDocuments.map(page => page.page_id)}
onSelect={updateOnlineDocuments}
canPreview={canPreview}
onPreview={onPreview}
isInPipeline={isInPipeline}
@ -37,4 +37,4 @@ const Notion = ({
)
}
export default Notion
export default OnlineDocuments

36
web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/notion-page-selector.tsx → web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/online-document-selector.tsx
View File

@ -8,7 +8,7 @@ import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { useDraftDatasourceNodeRun, usePublishedDatasourceNodeRun } from '@/service/use-pipeline'
import { DatasourceType } from '@/models/pipeline'
type NotionPageSelectorProps = {
type OnlineDocumentSelectorProps = {
value?: string[]
onSelect: (selectedPages: NotionPage[]) => void
canPreview?: boolean
@ -23,7 +23,7 @@ type NotionPageSelectorProps = {
}
}
const NotionPageSelector = ({
const OnlineDocumentSelector = ({
value,
onSelect,
canPreview,
@ -32,42 +32,42 @@ const NotionPageSelector = ({
isInPipeline = false,
nodeId,
headerInfo,
}: NotionPageSelectorProps) => {
}: OnlineDocumentSelectorProps) => {
const pipeline_id = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)
const [notionData, setNotionData] = useState<DataSourceNotionWorkspace[]>([])
const [documentsData, setDocumentsData] = useState<DataSourceNotionWorkspace[]>([])
const [searchValue, setSearchValue] = useState('')
const [currentWorkspaceId, setCurrentWorkspaceId] = useState('')
const useDatasourceNodeRun = useRef(!isInPipeline ? usePublishedDatasourceNodeRun : useDraftDatasourceNodeRun)
const { mutateAsync: getNotionPages } = useDatasourceNodeRun.current()
const { mutateAsync: crawlOnlineDocuments } = useDatasourceNodeRun.current()
const getNotionData = useCallback(async () => {
const getOnlineDocuments = useCallback(async () => {
if (pipeline_id) {
await getNotionPages({
await crawlOnlineDocuments({
pipeline_id,
node_id: nodeId,
inputs: {},
datasource_type: DatasourceType.onlineDocument,
}, {
onSuccess(notionData) {
setNotionData(notionData as DataSourceNotionWorkspace[])
onSuccess(documentsData) {
setDocumentsData(documentsData as DataSourceNotionWorkspace[])
},
})
}
}, [getNotionPages, nodeId, pipeline_id])
}, [crawlOnlineDocuments, nodeId, pipeline_id])
useEffect(() => {
getNotionData()
getOnlineDocuments()
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])
const firstWorkspaceId = notionData[0]?.workspace_id
const currentWorkspace = notionData.find(workspace => workspace.workspace_id === currentWorkspaceId)
const firstWorkspaceId = documentsData[0]?.workspace_id
const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId)
const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set<string>, Set<string>] = useMemo(() => {
const selectedPagesId = new Set<string>()
const boundPagesId = new Set<string>()
const pagesMap = notionData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => {
next.pages.forEach((page) => {
if (page.is_bound) {
selectedPagesId.add(page.page_id)
@ -82,7 +82,7 @@ const NotionPageSelector = ({
return prev
}, {})
return [pagesMap, selectedPagesId, boundPagesId]
}, [notionData])
}, [documentsData])
const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(value || [])]
const [selectedPagesId, setSelectedPagesId] = useState<Set<string>>(new Set(defaultSelectedPagesId))
@ -107,7 +107,7 @@ const NotionPageSelector = ({
setCurrentWorkspaceId(firstWorkspaceId)
}, [firstWorkspaceId])
if (!notionData?.length)
if (!documentsData?.length)
return null
return (
@ -121,7 +121,7 @@ const NotionPageSelector = ({
<div className='flex grow items-center gap-x-1'>
<WorkspaceSelector
value={currentWorkspaceId || firstWorkspaceId}
items={notionData}
items={documentsData}
onSelect={handleSelectWorkspace}
/>
</div>
@ -148,4 +148,4 @@ const NotionPageSelector = ({
)
}
export default NotionPageSelector
export default OnlineDocumentSelector

12
web/app/components/rag-pipeline/components/panel/test-run/hooks.ts
View File

@ -101,16 +101,16 @@ export const useLocalFile = () => {
}
}
export const useNotionPages = () => {
const [notionPages, setNotionPages] = useState<NotionPage[]>([])
export const useOnlineDocuments = () => {
const [onlineDocuments, setOnlineDocuments] = useState<NotionPage[]>([])
const updateNotionPages = (value: NotionPage[]) => {
setNotionPages(value)
const updateOnlineDocuments = (value: NotionPage[]) => {
setOnlineDocuments(value)
}
return {
notionPages,
updateNotionPages,
onlineDocuments,
updateOnlineDocuments,
}
}

24
web/app/components/rag-pipeline/components/panel/test-run/index.tsx
View File

@ -1,10 +1,10 @@
import { useStore as useWorkflowStoreWithSelector } from '@/app/components/workflow/store'
import { useCallback, useMemo, useState } from 'react'
import { useLocalFile, useNotionPages, useTestRunSteps, useWebsiteCrawl } from './hooks'
import { useLocalFile, useOnlineDocuments, useTestRunSteps, useWebsiteCrawl } from './hooks'
import DataSourceOptions from './data-source-options'
import LocalFile from './data-source/local-file'
import { useProviderContextSelector } from '@/context/provider-context'
import Notion from './data-source/notion'
import OnlineDocuments from './data-source/online-documents'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import WebsiteCrawl from './data-source/website-crawl'
import Actions from './data-source/actions'
@ -35,9 +35,9 @@ const TestRunPanel = () => {
updateFileList,
} = useLocalFile()
const {
notionPages,
updateNotionPages,
} = useNotionPages()
onlineDocuments,
updateOnlineDocuments,
} = useOnlineDocuments()
const {
websitePages,
websiteCrawlJobId,
@ -54,11 +54,11 @@ const TestRunPanel = () => {
if (datasource.type === DatasourceType.localFile)
return isShowVectorSpaceFull || !fileList.length || fileList.some(file => !file.file.id)
if (datasource.type === DatasourceType.onlineDocument)
return isShowVectorSpaceFull || !notionPages.length
return isShowVectorSpaceFull || !onlineDocuments.length
if (datasource.type === DatasourceType.websiteCrawl)
return isShowVectorSpaceFull || !websitePages.length
return false
}, [datasource, isShowVectorSpaceFull, fileList, notionPages.length, websitePages.length])
}, [datasource, isShowVectorSpaceFull, fileList, onlineDocuments.length, websitePages.length])
const handleClose = () => {
setShowDebugAndPreviewPanel(false)
@ -83,7 +83,7 @@ const TestRunPanel = () => {
datasourceInfoList.push(documentInfo)
}
if (datasource.type === DatasourceType.onlineDocument) {
const { workspace_id, ...rest } = notionPages[0]
const { workspace_id, ...rest } = onlineDocuments[0]
const documentInfo = {
workspace_id,
page: rest,
@ -103,7 +103,7 @@ const TestRunPanel = () => {
datasource_type: datasource.type,
datasource_info_list: datasourceInfoList,
})
}, [datasource, fileList, handleRun, notionPages, websiteCrawlJobId, websitePages])
}, [datasource, fileList, handleRun, onlineDocuments, websiteCrawlJobId, websitePages])
return (
<div
@ -130,15 +130,15 @@ const TestRunPanel = () => {
/>
)}
{datasource?.type === DatasourceType.onlineDocument && (
<Notion
<OnlineDocuments
nodeId={datasource?.nodeId || ''}
headerInfo={{
title: datasource.description,
docTitle: datasource.docTitle || '',
docLink: datasource.docLink || '',
}}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
onlineDocuments={onlineDocuments}
updateOnlineDocuments={updateOnlineDocuments}
isInPipeline
/>
)}