diff --git a/web/app/components/base/notion-page-selector/base.tsx b/web/app/components/base/notion-page-selector/base.tsx index 203eee77e6..a36dee0e10 100644 --- a/web/app/components/base/notion-page-selector/base.tsx +++ b/web/app/components/base/notion-page-selector/base.tsx @@ -15,7 +15,6 @@ type NotionPageSelectorProps = { previewPageId?: string onPreview?: (selectedPage: NotionPage) => void datasetId?: string - isInPipeline?: boolean } const NotionPageSelector = ({ @@ -25,7 +24,6 @@ const NotionPageSelector = ({ previewPageId, onPreview, datasetId = '', - isInPipeline = false, }: NotionPageSelectorProps) => { const { data, refetch } = usePreImportNotionPages({ url: '/notion/pre-import/pages', datasetId }) const [prevData, setPrevData] = useState(data) @@ -98,7 +96,6 @@ const NotionPageSelector = ({ ? (
void + showSelect?: boolean } const Actions = ({ diff --git a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts index 5139605ee6..82203abd3f 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/hooks.ts +++ b/web/app/components/datasets/documents/create-from-pipeline/hooks.ts @@ -7,7 +7,7 @@ import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-so import type { CrawlResult, CrawlResultItem, DocumentItem, FileItem } from '@/models/datasets' import { CrawlStep } from '@/models/datasets' import produce from 'immer' -import type { NotionPage } from '@/models/common' +import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' export const useAddDocumentsSteps = () => { const { t } = useTranslation() @@ -128,9 +128,34 @@ export const useLocalFile = () => { } export const useOnlineDocuments = () => { + const [documentsData, setDocumentsData] = useState([]) + const [searchValue, setSearchValue] = useState('') + const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') const [onlineDocuments, setOnlineDocuments] = useState([]) const [currentDocument, setCurrentDocument] = useState() + const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { + const selectedPagesId = new Set() + const boundPagesId = new Set() + const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + next.pages.forEach((page) => { + if (page.is_bound) { + selectedPagesId.add(page.page_id) + boundPagesId.add(page.page_id) + } + prev[page.page_id] = { + ...page, + workspace_id: next.workspace_id, + } + }) + + return prev + }, {}) + return [pagesMap, selectedPagesId, boundPagesId] + }, [documentsData]) + const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])] + const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) + const previewOnlineDocument = useRef(onlineDocuments[0]) const updateOnlineDocuments = (value: NotionPage[]) => { @@ -146,6 +171,15 @@ export const useOnlineDocuments = () => { }, []) return { + documentsData, + setDocumentsData, + searchValue, + setSearchValue, + currentWorkspaceId, + setCurrentWorkspaceId, + PagesMapAndSelectedPagesId, + selectedPagesId, + setSelectedPagesId, onlineDocuments, previewOnlineDocument, updateOnlineDocuments, diff --git a/web/app/components/datasets/documents/create-from-pipeline/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/index.tsx index 7c60363bf2..bef2cd7964 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/index.tsx @@ -8,7 +8,7 @@ import type { NotionPage } from '@/models/common' import OnlineDocuments from '@/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents' import VectorSpaceFull from '@/app/components/billing/vector-space-full' import WebsiteCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website-crawl' -import Actions from './data-source/actions' +import Actions from './actions' import { useTranslation } from 'react-i18next' import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' import LeftHeader from './left-header' @@ -61,6 +61,15 @@ const CreateFormPipeline = () => { hideFilePreview, } = useLocalFile() const { + documentsData, + setDocumentsData, + searchValue, + setSearchValue, + currentWorkspaceId, + setCurrentWorkspaceId, + PagesMapAndSelectedPagesId, + selectedPagesId, + setSelectedPagesId, onlineDocuments, previewOnlineDocument, updateOnlineDocuments, @@ -261,9 +270,17 @@ const CreateFormPipeline = () => { )} {datasourceType === DatasourceType.onlineDocument && ( doc.page_id)} onSelect={updateOnlineDocuments} onPreview={updateCurrentPage} /> diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/actions.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/actions/index.tsx similarity index 100% rename from web/app/components/rag-pipeline/components/panel/test-run/data-source/actions.tsx rename to web/app/components/rag-pipeline/components/panel/test-run/data-source/actions/index.tsx diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx index 07024b4e33..c505be6995 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/index.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo } from 'react' import WorkspaceSelector from '@/app/components/base/notion-page-selector/workspace-selector' import SearchInput from '@/app/components/base/notion-page-selector/search-input' import PageSelector from './page-selector' @@ -12,28 +12,41 @@ import type { DataSourceNodeCompletedResponse } from '@/types/pipeline' import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' type OnlineDocumentsProps = { - pageIdList?: string[] onSelect: (selectedPages: NotionPage[]) => void previewPageId?: string onPreview?: (selectedPage: NotionPage) => void isInPipeline?: boolean nodeId: string nodeData: DataSourceNodeType + documentsData: DataSourceNotionWorkspace[] + setDocumentsData: (documentsData: DataSourceNotionWorkspace[]) => void + searchValue: string + setSearchValue: (value: string) => void + currentWorkspaceId: string + setCurrentWorkspaceId: (workspaceId: string) => void + PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] + selectedPagesId: Set + setSelectedPagesId: (selectedPagesId: Set) => void } const OnlineDocuments = ({ - pageIdList, onSelect, previewPageId, onPreview, isInPipeline = false, nodeId, nodeData, + documentsData, + setDocumentsData, + searchValue, + setSearchValue, + currentWorkspaceId, + setCurrentWorkspaceId, + PagesMapAndSelectedPagesId, + selectedPagesId, + setSelectedPagesId, }: OnlineDocumentsProps) => { const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) - const [documentsData, setDocumentsData] = useState([]) - const [searchValue, setSearchValue] = useState('') - const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') const datasourceNodeRunURL = !isInPipeline ? `/rag/pipelines/${pipelineId}/workflows/published/datasource/nodes/${nodeId}/run` @@ -51,6 +64,7 @@ const OnlineDocuments = ({ { onDataSourceNodeCompleted: (documentsData: DataSourceNodeCompletedResponse) => { setDocumentsData(documentsData.data as DataSourceNotionWorkspace[]) + setCurrentWorkspaceId(documentsData.data[0].workspace_id) }, onError: (message: string) => { Toast.notify({ @@ -60,62 +74,36 @@ const OnlineDocuments = ({ }, }, ) - }, [datasourceNodeRunURL]) + }, [datasourceNodeRunURL, setCurrentWorkspaceId, setDocumentsData]) useEffect(() => { - getOnlineDocuments() + if (!documentsData.length) + getOnlineDocuments() // eslint-disable-next-line react-hooks/exhaustive-deps }, []) - const firstWorkspaceId = documentsData[0]?.workspace_id const currentWorkspace = documentsData.find(workspace => workspace.workspace_id === currentWorkspaceId) - const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { - const selectedPagesId = new Set() - const boundPagesId = new Set() - const pagesMap = documentsData.reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { - next.pages.forEach((page) => { - if (page.is_bound) { - selectedPagesId.add(page.page_id) - boundPagesId.add(page.page_id) - } - prev[page.page_id] = { - ...page, - workspace_id: next.workspace_id, - } - }) - - return prev - }, {}) - return [pagesMap, selectedPagesId, boundPagesId] - }, [documentsData]) - const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(pageIdList || [])] - const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) - const handleSearchValueChange = useCallback((value: string) => { setSearchValue(value) - }, []) + }, [setSearchValue]) const handleSelectWorkspace = useCallback((workspaceId: string) => { setCurrentWorkspaceId(workspaceId) - }, []) + }, [setCurrentWorkspaceId]) const handleSelectPages = useCallback((newSelectedPagesId: Set) => { const selectedPages = Array.from(newSelectedPagesId).map(pageId => PagesMapAndSelectedPagesId[0][pageId]) setSelectedPagesId(new Set(Array.from(newSelectedPagesId))) onSelect(selectedPages) - }, [onSelect, PagesMapAndSelectedPagesId]) + }, [setSelectedPagesId, onSelect, PagesMapAndSelectedPagesId]) const handlePreviewPage = useCallback((previewPageId: string) => { if (onPreview) onPreview(PagesMapAndSelectedPagesId[0][previewPageId]) }, [PagesMapAndSelectedPagesId, onPreview]) - useEffect(() => { - setCurrentWorkspaceId(firstWorkspaceId) - }, [firstWorkspaceId]) - const headerInfo = useMemo(() => { return { title: nodeData.title, @@ -137,7 +125,7 @@ const OnlineDocuments = ({
@@ -149,7 +137,7 @@ const OnlineDocuments = ({
diff --git a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx index 36e4bfeb82..8c12c15c1d 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/data-source/online-documents/page-selector/index.tsx @@ -6,7 +6,7 @@ import Item from './item' import { recursivePushInParentDescendants } from './utils' type PageSelectorProps = { - value: Set + checkedIds: Set disabledValue: Set searchValue: string pagesMap: DataSourceNotionPageMap @@ -16,6 +16,7 @@ type PageSelectorProps = { previewPageId?: string onPreview?: (selectedPageId: string) => void isMultipleChoice?: boolean + currentWorkspaceId: string } export type NotionPageTreeItem = { @@ -33,7 +34,7 @@ type NotionPageItem = { } & DataSourceNotionPage const PageSelector = ({ - value, + checkedIds, disabledValue, searchValue, pagesMap, @@ -43,24 +44,22 @@ const PageSelector = ({ previewPageId, onPreview, isMultipleChoice = true, + currentWorkspaceId, }: PageSelectorProps) => { const { t } = useTranslation() - const [prevDataList, setPrevDataList] = useState(list) const [dataList, setDataList] = useState([]) const [localPreviewPageId, setLocalPreviewPageId] = useState('') useEffect(() => { - if (prevDataList !== list) { - setPrevDataList(list) - setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => { - return { - ...item, - expand: false, - depth: 0, - } - })) - } - }, [prevDataList, list, pagesMap]) + setDataList(list.filter(item => item.parent_id === 'root' || !pagesMap[item.parent_id]).map((item) => { + return { + ...item, + expand: false, + depth: 0, + } + })) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [currentWorkspaceId]) const searchDataList = list.filter((item) => { return item.page_name.includes(searchValue) @@ -108,13 +107,14 @@ const PageSelector = ({ expand: false, depth: listMapWithChildrenAndDescendants[item].depth, })), - ...dataList.slice(index + 1)] + ...dataList.slice(index + 1), + ] } setDataList(newDataList) }, [dataList, listMapWithChildrenAndDescendants, pagesMap]) const handleCheck = useCallback((index: number) => { - const copyValue = new Set([...value]) + const copyValue = new Set([...checkedIds]) const current = currentDataList[index] const pageId = current.page_id const currentWithChildrenAndDescendants = listMapWithChildrenAndDescendants[pageId] @@ -143,7 +143,7 @@ const PageSelector = ({ } onSelect(new Set([...copyValue])) - }, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, value]) + }, [currentDataList, isMultipleChoice, listMapWithChildrenAndDescendants, onSelect, searchValue, checkedIds]) const handlePreview = useCallback((index: number) => { const current = currentDataList[index] @@ -174,7 +174,7 @@ const PageSelector = ({ itemData={{ dataList: currentDataList, handleToggle, - checkedIds: value, + checkedIds, disabledCheckedIds: disabledValue, handleCheck, canPreview, diff --git a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts index 262e4216a3..c777cab963 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts +++ b/web/app/components/rag-pipeline/components/panel/test-run/hooks.ts @@ -8,7 +8,7 @@ import { useCallback, useMemo, useState } from 'react' import type { CrawlResult } from '@/models/datasets' import { type CrawlResultItem, CrawlStep, type FileItem } from '@/models/datasets' import produce from 'immer' -import type { NotionPage } from '@/models/common' +import type { DataSourceNotionPageMap, DataSourceNotionWorkspace, NotionPage } from '@/models/common' export const useTestRunSteps = () => { const { t } = useTranslation() @@ -107,13 +107,47 @@ export const useLocalFile = () => { } export const useOnlineDocuments = () => { + const [documentsData, setDocumentsData] = useState([]) + const [searchValue, setSearchValue] = useState('') + const [currentWorkspaceId, setCurrentWorkspaceId] = useState('') const [onlineDocuments, setOnlineDocuments] = useState([]) + const PagesMapAndSelectedPagesId: [DataSourceNotionPageMap, Set, Set] = useMemo(() => { + const selectedPagesId = new Set() + const boundPagesId = new Set() + const pagesMap = (documentsData || []).reduce((prev: DataSourceNotionPageMap, next: DataSourceNotionWorkspace) => { + next.pages.forEach((page) => { + if (page.is_bound) { + selectedPagesId.add(page.page_id) + boundPagesId.add(page.page_id) + } + prev[page.page_id] = { + ...page, + workspace_id: next.workspace_id, + } + }) + + return prev + }, {}) + return [pagesMap, selectedPagesId, boundPagesId] + }, [documentsData]) + const defaultSelectedPagesId = [...Array.from(PagesMapAndSelectedPagesId[1]), ...(onlineDocuments.map(doc => doc.page_id) || [])] + const [selectedPagesId, setSelectedPagesId] = useState>(new Set(defaultSelectedPagesId)) + const updateOnlineDocuments = (value: NotionPage[]) => { setOnlineDocuments(value) } return { + documentsData, + setDocumentsData, + searchValue, + setSearchValue, + currentWorkspaceId, + setCurrentWorkspaceId, + PagesMapAndSelectedPagesId, + selectedPagesId, + setSelectedPagesId, onlineDocuments, updateOnlineDocuments, } diff --git a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx index f5eb375e1f..75b6b854d3 100644 --- a/web/app/components/rag-pipeline/components/panel/test-run/index.tsx +++ b/web/app/components/rag-pipeline/components/panel/test-run/index.tsx @@ -32,6 +32,15 @@ const TestRunPanel = () => { updateFileList, } = useLocalFile() const { + documentsData, + setDocumentsData, + searchValue, + setSearchValue, + currentWorkspaceId, + setCurrentWorkspaceId, + PagesMapAndSelectedPagesId, + selectedPagesId, + setSelectedPagesId, onlineDocuments, updateOnlineDocuments, } = useOnlineDocuments() @@ -125,9 +134,17 @@ const TestRunPanel = () => { )} {datasourceType === DatasourceType.onlineDocument && ( doc.page_id)} onSelect={updateOnlineDocuments} isInPipeline />