From 447b016e9ea635ccea5af24680416f6338eefd15 Mon Sep 17 00:00:00 2001 From: twwu Date: Fri, 5 Sep 2025 14:21:41 +0800 Subject: [PATCH 1/3] refactor(NotionPageSelector): Remove NotionPageSelectorModal component and associated styles --- .../base/notion-page-selector/index.tsx | 1 - .../index.module.css | 28 ----- .../notion-page-selector-modal/index.tsx | 92 -------------- .../components/datasets/documents/index.tsx | 117 +++++------------- .../components/datasets/documents/list.tsx | 35 ++---- 5 files changed, 41 insertions(+), 232 deletions(-) delete mode 100644 web/app/components/base/notion-page-selector/notion-page-selector-modal/index.module.css delete mode 100644 web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx diff --git a/web/app/components/base/notion-page-selector/index.tsx b/web/app/components/base/notion-page-selector/index.tsx index 0a44c70fe8..0eee506153 100644 --- a/web/app/components/base/notion-page-selector/index.tsx +++ b/web/app/components/base/notion-page-selector/index.tsx @@ -1,2 +1 @@ -export { default as NotionPageSelectorModal } from './notion-page-selector-modal' export { default as NotionPageSelector } from './base' diff --git a/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.module.css b/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.module.css deleted file mode 100644 index cd1f9c76ab..0000000000 --- a/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.module.css +++ /dev/null @@ -1,28 +0,0 @@ -.modal { - width: 600px !important; - max-width: 600px !important; - padding: 24px 32px !important; -} - -.operate { - padding: 0 8px; - min-width: 96px; - height: 36px; - line-height: 36px; - text-align: center; - background-color: #ffffff; - box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05); - border-radius: 8px; - border: 0.5px solid #eaecf0; - font-size: 14px; - font-weight: 500; - color: #667085; - cursor: pointer; -} - -.operate-save { - margin-left: 8px; - border-color: #155eef; - background-color: #155eef; - color: #ffffff; -} diff --git a/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx b/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx deleted file mode 100644 index 5d045f1b37..0000000000 --- a/web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx +++ /dev/null @@ -1,92 +0,0 @@ -import { useCallback, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { XMarkIcon } from '@heroicons/react/24/outline' -import NotionPageSelector from '../base' -import s from './index.module.css' -import type { NotionPage } from '@/models/common' -import cn from '@/utils/classnames' -import Modal from '@/app/components/base/modal' -import { noop } from 'lodash-es' -import { useGetDefaultDataSourceListAuth } from '@/service/use-datasource' -import NotionConnector from '../../notion-connector' -import { useModalContextSelector } from '@/context/modal-context' - -type NotionPageSelectorModalProps = { - isShow: boolean - onClose: () => void - onSave: (selectedPages: NotionPage[]) => void - datasetId: string -} -const NotionPageSelectorModal = ({ - isShow, - onClose, - onSave, - datasetId, -}: NotionPageSelectorModalProps) => { - const { t } = useTranslation() - const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal) - const [selectedPages, setSelectedPages] = useState([]) - - const { data: dataSourceList } = useGetDefaultDataSourceListAuth() - - const handleClose = useCallback(() => { - onClose() - }, [onClose]) - - const handleSelectPage = useCallback((newSelectedPages: NotionPage[]) => { - setSelectedPages(newSelectedPages) - }, []) - - const handleSave = useCallback(() => { - onSave(selectedPages) - }, [onSave]) - - const handleOpenSetting = useCallback(() => { - setShowAccountSettingModal({ payload: 'data-source' }) - }, [setShowAccountSettingModal]) - - const authedDataSourceList = dataSourceList?.result || [] - - const isNotionAuthed = useMemo(() => { - if (!authedDataSourceList) return false - const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource') - if (!notionSource) return false - return notionSource.credentials_list.length > 0 - }, [authedDataSourceList]) - - const notionCredentialList = useMemo(() => { - return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || [] - }, [authedDataSourceList]) - - return ( - -
-
{t('common.dataSource.notion.selector.addPages')}
-
- -
-
- {!isNotionAuthed && } - {isNotionAuthed && ( - - )} -
-
{t('common.operation.cancel')}
-
{t('common.operation.save')}
-
- - ) -} - -export default NotionPageSelectorModal diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index 39e708a4ef..6da5ba2e0e 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -4,7 +4,6 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' import { useRouter } from 'next/navigation' import { useDebounce, useDebounceFn } from 'ahooks' -import { groupBy } from 'lodash-es' import { PlusIcon } from '@heroicons/react/24/solid' import { RiDraftLine, RiExternalLinkLine } from '@remixicon/react' import AutoDisabledDocument from '../common/document-status-with-action/auto-disabled-document' @@ -13,13 +12,8 @@ import s from './style.module.css' import Loading from '@/app/components/base/loading' import Button from '@/app/components/base/button' import Input from '@/app/components/base/input' -import { get } from '@/service/base' -import { createDocument } from '@/service/datasets' import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' -import { NotionPageSelectorModal } from '@/app/components/base/notion-page-selector' -import type { NotionPage } from '@/models/common' -import type { CreateDocumentReq } from '@/models/datasets' -import { DataSourceType, ProcessMode } from '@/models/datasets' +import { DataSourceType } from '@/models/datasets' import IndexFailed from '@/app/components/datasets/common/document-status-with-action/index-failed' import { useProviderContext } from '@/context/provider-context' import cn from '@/utils/classnames' @@ -31,7 +25,6 @@ import useEditDocumentMetadata from '../metadata/hooks/use-edit-dataset-metadata import DatasetMetadataDrawer from '../metadata/metadata-dataset/dataset-metadata-drawer' import StatusWithAction from '../common/document-status-with-action/status-with-action' import { useDocLink } from '@/context/i18n' -import { useFetchDefaultProcessRule } from '@/service/knowledge/use-create-dataset' import { SimpleSelect } from '../../base/select' import StatusItem from './detail/completed/status-item' import type { Item } from '@/app/components/base/select' @@ -86,8 +79,6 @@ type IDocumentsProps = { datasetId: string } -export const fetcher = (url: string) => get(url, {}, {}) - const Documents: FC = ({ datasetId }) => { const { t } = useTranslation() const docLink = useDocLink() @@ -105,7 +96,6 @@ const Documents: FC = ({ datasetId }) => { const router = useRouter() const dataset = useDatasetDetailContextWithSelector(s => s.dataset) - const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false) const [timerCanRun, setTimerCanRun] = useState(true) const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB @@ -231,66 +221,14 @@ const Documents: FC = ({ datasetId }) => { const total = documentsRes?.total || 0 const routeToDocCreate = () => { - // if dataset is create from pipeline, redirect to create from pipeline page + // if dataset is create from pipeline, go to create from pipeline page if (dataset?.runtime_mode === 'rag_pipeline') { router.push(`/datasets/${datasetId}/documents/create-from-pipeline`) return } - if (isDataSourceNotion) { - setNotionPageSelectorModalVisible(true) - return - } router.push(`/datasets/${datasetId}/documents/create`) } - const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule() - - const handleSaveNotionPageSelected = async (selectedPages: NotionPage[]) => { - const workspacesMap = groupBy(selectedPages, 'workspace_id') - const workspaces = Object.keys(workspacesMap).map((workspaceId) => { - return { - workspaceId, - pages: workspacesMap[workspaceId], - } - }) - const { rules } = await fetchDefaultProcessRuleMutation.mutateAsync('/datasets/process-rule') - const params = { - data_source: { - type: dataset?.data_source_type, - info_list: { - data_source_type: dataset?.data_source_type, - notion_info_list: workspaces.map((workspace) => { - return { - workspace_id: workspace.workspaceId, - pages: workspace.pages.map((page) => { - const { page_id, page_name, page_icon, type } = page - return { - page_id, - page_name, - page_icon, - type, - } - }), - } - }), - }, - }, - indexing_technique: dataset?.indexing_technique, - process_rule: { - rules, - mode: ProcessMode.general, - }, - } as CreateDocumentReq - - await createDocument({ - datasetId, - body: params, - }) - invalidDocumentList() - setTimerCanRun(true) - setNotionPageSelectorModalVisible(false) - } - const documentsList = isDataSourceNotion ? documentsWithProgress?.data : documentsRes?.data const [selectedIds, setSelectedIds] = useState([]) @@ -402,32 +340,33 @@ const Documents: FC = ({ datasetId }) => { ? // eslint-disable-next-line sonarjs/no-nested-conditional : total > 0 - ? - : + ? ( + + ) + : ( + + ) } - setNotionPageSelectorModalVisible(false)} - onSave={handleSaveNotionPageSelected} - datasetId={dataset?.id || ''} - /> ) diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index 7e11b7567e..cc3842ed94 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -68,7 +68,6 @@ type IDocumentListProps = { onUpdate: () => void onManageMetadata: () => void statusFilter: Item - onStatusFilterChange: (filter: string) => void } /** @@ -261,9 +260,9 @@ const DocumentList: FC = ({ return parts[parts.length - 1].toLowerCase() }, []) - const isCreateFromRAGPipeline = useMemo(() => { - return datasetConfig?.runtime_mode === 'rag_pipeline' - }, [datasetConfig?.runtime_mode]) + const isCreateFromRAGPipeline = useCallback((createdFrom: string) => { + return createdFrom === 'rag_pipeline' + }, []) /** * Calculate the data source type @@ -271,25 +270,17 @@ const DocumentList: FC = ({ * DatasourceType: localFile, onlineDocument, websiteCrawl, onlineDrive (new) */ const isLocalFile = useCallback((dataSourceType: DataSourceType | DatasourceType) => { - if (isCreateFromRAGPipeline) - return dataSourceType === DatasourceType.localFile - return dataSourceType === DataSourceType.FILE - }, [isCreateFromRAGPipeline]) + return dataSourceType === DatasourceType.localFile || dataSourceType === DataSourceType.FILE + }, []) const isOnlineDocument = useCallback((dataSourceType: DataSourceType | DatasourceType) => { - if (isCreateFromRAGPipeline) - return dataSourceType === DatasourceType.onlineDocument - return dataSourceType === DataSourceType.NOTION - }, [isCreateFromRAGPipeline]) + return dataSourceType === DatasourceType.onlineDocument || dataSourceType === DataSourceType.NOTION + }, []) const isWebsiteCrawl = useCallback((dataSourceType: DataSourceType | DatasourceType) => { - if (isCreateFromRAGPipeline) - return dataSourceType === DatasourceType.websiteCrawl - return dataSourceType === DataSourceType.WEB - }, [isCreateFromRAGPipeline]) + return dataSourceType === DatasourceType.websiteCrawl || dataSourceType === DataSourceType.WEB + }, []) const isOnlineDrive = useCallback((dataSourceType: DataSourceType | DatasourceType) => { - if (isCreateFromRAGPipeline) - return dataSourceType === DatasourceType.onlineDrive - return false - }, [isCreateFromRAGPipeline]) + return dataSourceType === DatasourceType.onlineDrive + }, []) return (
@@ -361,7 +352,7 @@ const DocumentList: FC = ({ className='mr-1.5' type='page' src={ - isCreateFromRAGPipeline + isCreateFromRAGPipeline(doc.created_from) ? (doc.data_source_info as OnlineDocumentInfo).page.page_icon : (doc.data_source_info as LegacyDataSourceInfo).notion_page_icon } @@ -371,7 +362,7 @@ const DocumentList: FC = ({ Date: Fri, 5 Sep 2025 16:13:15 +0800 Subject: [PATCH 2/3] refactor(Documents): Remove ProgressBar component and simplify document loading logic --- .../components/base/progress-bar/index.tsx | 20 ------------------ .../components/datasets/documents/index.tsx | 17 ++++++--------- .../components/datasets/documents/list.tsx | 21 ++++++------------- 3 files changed, 12 insertions(+), 46 deletions(-) delete mode 100644 web/app/components/base/progress-bar/index.tsx diff --git a/web/app/components/base/progress-bar/index.tsx b/web/app/components/base/progress-bar/index.tsx deleted file mode 100644 index 759c9ea846..0000000000 --- a/web/app/components/base/progress-bar/index.tsx +++ /dev/null @@ -1,20 +0,0 @@ -type ProgressBarProps = { - percent: number -} -const ProgressBar = ({ - percent = 0, -}: ProgressBarProps) => { - return ( -
-
-
-
-
{percent}%
-
- ) -} - -export default ProgressBar diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index 6da5ba2e0e..7f690aae25 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -154,14 +154,14 @@ const Documents: FC = ({ datasetId }) => { } }, [debouncedSearchValue, query.keyword, updateQuery]) - const { data: documentsRes, isFetching: isListLoading } = useDocumentList({ + const { data: documentsRes, isLoading: isListLoading } = useDocumentList({ datasetId, query: { page: currPage + 1, limit, keyword: debouncedSearchValue, }, - refetchInterval: (isDataSourceNotion && timerCanRun) ? 2500 : 0, + refetchInterval: timerCanRun ? 2500 : 0, }) const invalidDocumentList = useInvalidDocumentList(datasetId) @@ -187,10 +187,10 @@ const Documents: FC = ({ datasetId }) => { }, 5000) }, []) - const documentsWithProgress = useMemo(() => { + useEffect(() => { let completedNum = 0 let percent = 0 - const documentsData = documentsRes?.data?.map((documentItem) => { + documentsRes?.data?.forEach((documentItem) => { const { indexing_status, completed_segments, total_segments } = documentItem const isEmbedded = indexing_status === 'completed' || indexing_status === 'paused' || indexing_status === 'error' @@ -211,12 +211,7 @@ const Documents: FC = ({ datasetId }) => { percent, } }) - if (completedNum === documentsRes?.data?.length) - setTimerCanRun(false) - return { - ...documentsRes, - data: documentsData, - } + setTimerCanRun(completedNum !== documentsRes?.data?.length) }, [documentsRes]) const total = documentsRes?.total || 0 @@ -229,7 +224,7 @@ const Documents: FC = ({ datasetId }) => { router.push(`/datasets/${datasetId}/documents/create`) } - const documentsList = isDataSourceNotion ? documentsWithProgress?.data : documentsRes?.data + const documentsList = documentsRes?.data const [selectedIds, setSelectedIds] = useState([]) // Clear selection when search changes to avoid confusion diff --git a/web/app/components/datasets/documents/list.tsx b/web/app/components/datasets/documents/list.tsx index cc3842ed94..60e4a02a75 100644 --- a/web/app/components/datasets/documents/list.tsx +++ b/web/app/components/datasets/documents/list.tsx @@ -1,6 +1,6 @@ 'use client' import type { FC } from 'react' -import React, { useCallback, useEffect, useMemo, useState } from 'react' +import React, { useCallback, useMemo, useState } from 'react' import { useBoolean } from 'ahooks' import { ArrowDownIcon } from '@heroicons/react/24/outline' import { pick, uniq } from 'lodash-es' @@ -22,7 +22,6 @@ import type { Item } from '@/app/components/base/select' import { asyncRunSafe } from '@/utils' import { formatNumber } from '@/utils/format' import NotionIcon from '@/app/components/base/notion-icon' -import ProgressBar from '@/app/components/base/progress-bar' import type { LegacyDataSourceInfo, LocalFileInfo, OnlineDocumentInfo, OnlineDriveInfo } from '@/models/datasets' import { ChunkingMode, DataSourceType, DocumentActionType, type SimpleDocumentDetail } from '@/models/datasets' import type { CommonResponse } from '@/models/common' @@ -91,7 +90,6 @@ const DocumentList: FC = ({ const chunkingMode = datasetConfig?.doc_form const isGeneralMode = chunkingMode !== ChunkingMode.parentChild const isQAMode = chunkingMode === ChunkingMode.qa - const [localDocs, setLocalDocs] = useState(documents) const [sortField, setSortField] = useState<'name' | 'word_count' | 'hit_count' | 'created_at' | null>('created_at') const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc') @@ -108,7 +106,7 @@ const DocumentList: FC = ({ onUpdate, }) - useEffect(() => { + const localDocs = useMemo(() => { let filteredDocs = documents if (statusFilter.value !== 'all') { @@ -119,10 +117,8 @@ const DocumentList: FC = ({ ) } - if (!sortField) { - setLocalDocs(filteredDocs) - return - } + if (!sortField) + return filteredDocs const sortedDocs = [...filteredDocs].sort((a, b) => { let aValue: any @@ -159,7 +155,7 @@ const DocumentList: FC = ({ } }) - setLocalDocs(sortedDocs) + return sortedDocs }, [documents, sortField, sortOrder, statusFilter]) const handleSort = (field: 'name' | 'word_count' | 'hit_count' | 'created_at') => { @@ -418,12 +414,7 @@ const DocumentList: FC = ({ {formatTime(doc.created_at, t('datasetHitTesting.dateTimeFormat') as string)} - { - (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) - && isOnlineDocument(doc.data_source_type)) - ? - : - } + Date: Fri, 5 Sep 2025 16:14:38 +0800 Subject: [PATCH 3/3] Update web/app/components/datasets/documents/index.tsx Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- web/app/components/datasets/documents/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/documents/index.tsx b/web/app/components/datasets/documents/index.tsx index 7f690aae25..613257efee 100644 --- a/web/app/components/datasets/documents/index.tsx +++ b/web/app/components/datasets/documents/index.tsx @@ -216,7 +216,7 @@ const Documents: FC = ({ datasetId }) => { const total = documentsRes?.total || 0 const routeToDocCreate = () => { - // if dataset is create from pipeline, go to create from pipeline page + // if dataset is created from pipeline, go to create from pipeline page if (dataset?.runtime_mode === 'rag_pipeline') { router.push(`/datasets/${datasetId}/documents/create-from-pipeline`) return