From 33cd32382fb94430b5174ed846524a2440716412 Mon Sep 17 00:00:00 2001 From: twwu Date: Tue, 17 Jun 2025 11:29:56 +0800 Subject: [PATCH] feat: add indexing status batch and process rule hooks; refactor Notion page preview types --- .../components/billing/upgrade-btn/index.tsx | 4 +- .../processing/embedding-process/index.tsx | 88 +++++++------------ web/models/datasets.ts | 15 ++++ web/service/knowledge/use-dataset.ts | 44 +++++++--- 4 files changed, 82 insertions(+), 69 deletions(-) diff --git a/web/app/components/billing/upgrade-btn/index.tsx b/web/app/components/billing/upgrade-btn/index.tsx index 45f4d44136..f3ae95a10b 100644 --- a/web/app/components/billing/upgrade-btn/index.tsx +++ b/web/app/components/billing/upgrade-btn/index.tsx @@ -50,8 +50,8 @@ const UpgradeBtn: FC = ({ return ( diff --git a/web/app/components/datasets/documents/create-from-pipeline/processing/embedding-process/index.tsx b/web/app/components/datasets/documents/create-from-pipeline/processing/embedding-process/index.tsx index 47572515aa..a01c208bb0 100644 --- a/web/app/components/datasets/documents/create-from-pipeline/processing/embedding-process/index.tsx +++ b/web/app/components/datasets/documents/create-from-pipeline/processing/embedding-process/index.tsx @@ -1,8 +1,6 @@ -import React, { useEffect, useMemo, useRef, useState } from 'react' -import useSWR from 'swr' +import React, { useEffect, useMemo, useState } from 'react' import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' -import { omit } from 'lodash-es' import { ArrowRightIcon } from '@heroicons/react/24/solid' import { RiCheckboxCircleFill, @@ -13,14 +11,12 @@ import { import cn from '@/utils/classnames' import Button from '@/app/components/base/button' import type { IndexingStatusResponse } from '@/models/datasets' -import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets' import NotionIcon from '@/app/components/base/notion-icon' import PriorityLabel from '@/app/components/billing/priority-label' import { Plan } from '@/app/components/billing/type' import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general' import UpgradeBtn from '@/app/components/billing/upgrade-btn' import { useProviderContext } from '@/context/provider-context' -import { sleep } from '@/utils' import Tooltip from '@/app/components/base/tooltip' import { useInvalidDocumentList } from '@/service/knowledge/use-document' import DocumentFileIcon from '@/app/components/datasets/common/document-file-icon' @@ -28,6 +24,7 @@ import RuleDetail from './rule-detail' import type { IndexingType } from '@/app/components/datasets/create/step-two' import type { RETRIEVE_METHOD } from '@/types/app' import { DatasourceType, type InitialDocumentDetail } from '@/models/pipeline' +import { useIndexingStatusBatch, useProcessRule } from '@/service/knowledge/use-dataset' type EmbeddingProcessProps = { datasetId: string @@ -45,64 +42,43 @@ const EmbeddingProcess = ({ retrievalMethod, }: EmbeddingProcessProps) => { const { t } = useTranslation() + const router = useRouter() const { enableBilling, plan } = useProviderContext() - - const firstDocument = documents[0] - const [indexingStatusBatchDetail, setIndexingStatusDetail] = useState([]) - const fetchIndexingStatus = async () => { - const status = await doFetchIndexingStatus({ datasetId, batchId }) - setIndexingStatusDetail(status.data) - return status.data - } - - const [isStopQuery, setIsStopQuery] = useState(false) - const isStopQueryRef = useRef(isStopQuery) - useEffect(() => { - isStopQueryRef.current = isStopQuery - }, [isStopQuery]) - const stopQueryStatus = () => { - setIsStopQuery(true) - } - - const startQueryStatus = async () => { - if (isStopQueryRef.current) - return - - try { - const indexingStatusBatchDetail = await fetchIndexingStatus() - const isCompleted = indexingStatusBatchDetail.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status)) - if (isCompleted) { - stopQueryStatus() - return - } - await sleep(2500) - await startQueryStatus() - } - catch { - await sleep(2500) - await startQueryStatus() - } - } + const [shouldPoll, setShouldPoll] = useState(true) + const { mutateAsync: fetchIndexingStatus } = useIndexingStatusBatch({ datasetId, batchId }) useEffect(() => { - setIsStopQuery(false) - startQueryStatus() + let timeoutId: ReturnType + + const fetchData = async () => { + await fetchIndexingStatus(undefined, { + onSuccess: (res) => { + const indexingStatusDetailList = res.data + setIndexingStatusDetail(indexingStatusDetailList) + const isCompleted = indexingStatusDetailList.every(indexingStatusDetail => ['completed', 'error', 'paused'].includes(indexingStatusDetail.indexing_status)) + if (isCompleted) + setShouldPoll(false) + }, + onSettled: () => { + if (shouldPoll) + timeoutId = setTimeout(fetchData, 2500) + }, + }) + } + + fetchData() + return () => { - stopQueryStatus() + clearTimeout(timeoutId) } // eslint-disable-next-line react-hooks/exhaustive-deps - }, []) + }, [shouldPoll]) // get rule - const { data: ruleDetail } = useSWR({ - action: 'fetchProcessRule', - params: { documentId: firstDocument.id }, - }, apiParams => fetchProcessRule(omit(apiParams, 'action')), { - revalidateOnFocus: false, - }) + const firstDocument = documents[0] + const { data: ruleDetail } = useProcessRule(firstDocument.id) - const router = useRouter() const invalidDocumentList = useInvalidDocumentList() const navToDocumentList = () => { invalidDocumentList() @@ -136,7 +112,6 @@ const EmbeddingProcess = ({ const doc = documents.find(document => document.id === id) return doc?.data_source_type } - const getIcon = (id: string) => { const doc = documents.find(document => document.id === id) @@ -176,7 +151,10 @@ const EmbeddingProcess = ({ indexingStatusDetail.indexing_status === 'error' && 'bg-state-destructive-hover-alt', )}> {isSourceEmbedding(indexingStatusDetail) && ( -
+
)}
{getSourceType(indexingStatusDetail.id) === DatasourceType.localFile && ( diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 37e8682eb8..b118a71816 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -764,3 +764,18 @@ export type CreateDatasetResponse = { updated_at: number pipeline_id: string } + +export type NotionPagePreviewRequest = { + workspaceID: string + pageID: string + pageType: string +} + +export type NotionPagePreviewResponse = { + content: string +} + +export type IndexingStatusBatchRequest = { + datasetId: string + batchId: string +} diff --git a/web/service/knowledge/use-dataset.ts b/web/service/knowledge/use-dataset.ts index ff88a748ed..adf9b05654 100644 --- a/web/service/knowledge/use-dataset.ts +++ b/web/service/knowledge/use-dataset.ts @@ -1,5 +1,16 @@ -import { useInfiniteQuery, useQuery } from '@tanstack/react-query' -import type { DataSet, DataSetListResponse, DatasetListRequest, RelatedAppResponse } from '@/models/datasets' +import type { MutationOptions } from '@tanstack/react-query' +import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query' +import type { + DataSet, + DataSetListResponse, + DatasetListRequest, + IndexingStatusBatchRequest, + IndexingStatusBatchResponse, + NotionPagePreviewRequest, + NotionPagePreviewResponse, + ProcessRuleResponse, + RelatedAppResponse, +} from '@/models/datasets' import { get } from '../base' import { useReset } from '../use-base' import qs from 'qs' @@ -45,16 +56,6 @@ export const useDatasetRelatedApps = (datasetId: string) => { }) } -type NotionPagePreviewRequest = { - workspaceID: string - pageID: string - pageType: string -} - -type NotionPagePreviewResponse = { - content: string -} - export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => { const { workspaceID, pageID, pageType } = params return useQuery({ @@ -63,3 +64,22 @@ export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => { enabled: !!workspaceID && !!pageID && !!pageType, }) } + +export const useIndexingStatusBatch = ( + params: IndexingStatusBatchRequest, + mutationOptions: MutationOptions = {}, +) => { + const { datasetId, batchId } = params + return useMutation({ + mutationKey: [NAME_SPACE, 'indexing-status-batch', datasetId, batchId], + mutationFn: () => get(`/datasets/${datasetId}/batch/${batchId}/indexing-status`), + ...mutationOptions, + }) +} + +export const useProcessRule = (documentId: string) => { + return useQuery({ + queryKey: [NAME_SPACE, 'process-rule', documentId], + queryFn: () => get('/datasets/process-rule', { params: { document_id: documentId } }), + }) +}