diff --git a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx index ca2e119941..6857c38e1e 100644 --- a/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx +++ b/web/app/components/app/configuration/dataset-config/select-dataset/index.tsx @@ -1,18 +1,18 @@ 'use client' import type { FC } from 'react' -import React, { useRef, useState } from 'react' -import { useGetState, useInfiniteScroll } from 'ahooks' +import React, { useEffect, useMemo, useRef, useState } from 'react' +import { useInfiniteScroll } from 'ahooks' import { useTranslation } from 'react-i18next' import Link from 'next/link' import Modal from '@/app/components/base/modal' import type { DataSet } from '@/models/datasets' import Button from '@/app/components/base/button' -import { fetchDatasets } from '@/service/datasets' import Loading from '@/app/components/base/loading' import Badge from '@/app/components/base/badge' import { useKnowledge } from '@/hooks/use-knowledge' import cn from '@/utils/classnames' import AppIcon from '@/app/components/base/app-icon' +import { useInfiniteDatasets } from '@/service/knowledge/use-dataset' import { ModelFeatureEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' import FeatureIcon from '@/app/components/header/account-setting/model-provider-page/model-selector/feature-icon' @@ -30,51 +30,70 @@ const SelectDataSet: FC = ({ onSelect, }) => { const { t } = useTranslation() - const [selected, setSelected] = React.useState([]) - const [loaded, setLoaded] = React.useState(false) - const [datasets, setDataSets] = React.useState(null) - const [hasInitialized, setHasInitialized] = React.useState(false) - const hasNoData = !datasets || datasets?.length === 0 + const [selected, setSelected] = useState([]) const canSelectMulti = true + const { formatIndexingTechniqueAndMethod } = useKnowledge() + const { data, isLoading, isFetchingNextPage, fetchNextPage, hasNextPage } = useInfiniteDatasets( + { page: 1 }, + { enabled: isShow, staleTime: 0, refetchOnMount: 'always' }, + ) + const pages = data?.pages || [] + const datasets = useMemo(() => { + return pages.flatMap(page => page.data.filter(item => item.indexing_technique || item.provider === 'external')) + }, [pages]) + const hasNoData = !isLoading && datasets.length === 0 const listRef = useRef(null) - const [page, setPage, getPage] = useGetState(1) - const [isNoMore, setIsNoMore] = useState(false) - const { formatIndexingTechniqueAndMethod } = useKnowledge() + const isNoMore = hasNextPage === false useInfiniteScroll( async () => { - if (!isNoMore) { - const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } }) - setPage(getPage() + 1) - setIsNoMore(!has_more) - const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique || item.provider === 'external')] - setDataSets(newList) - setLoaded(true) - - // Initialize selected datasets based on selectedIds and available datasets - if (!hasInitialized) { - if (selectedIds.length > 0) { - const validSelectedDatasets = selectedIds - .map(id => newList.find(item => item.id === id)) - .filter(Boolean) as DataSet[] - setSelected(validSelectedDatasets) - } - setHasInitialized(true) - } - } + if (!hasNextPage || isFetchingNextPage) + return { list: [] } + await fetchNextPage() return { list: [] } }, { target: listRef, - isNoMore: () => { - return isNoMore - }, - reloadDeps: [isNoMore], + isNoMore: () => isNoMore, + reloadDeps: [isNoMore, isFetchingNextPage], }, ) + const prevSelectedIdsRef = useRef([]) + const hasUserModifiedSelectionRef = useRef(false) + useEffect(() => { + if (isShow) + hasUserModifiedSelectionRef.current = false + }, [isShow]) + useEffect(() => { + const prevSelectedIds = prevSelectedIdsRef.current + const idsChanged = selectedIds.length !== prevSelectedIds.length + || selectedIds.some((id, idx) => id !== prevSelectedIds[idx]) + + if (!selectedIds.length && (!hasUserModifiedSelectionRef.current || idsChanged)) { + setSelected([]) + prevSelectedIdsRef.current = selectedIds + hasUserModifiedSelectionRef.current = false + return + } + + if (!idsChanged && hasUserModifiedSelectionRef.current) + return + + setSelected((prev) => { + const prevMap = new Map(prev.map(item => [item.id, item])) + const nextSelected = selectedIds + .map(id => datasets.find(item => item.id === id) || prevMap.get(id)) + .filter(Boolean) as DataSet[] + return nextSelected + }) + prevSelectedIdsRef.current = selectedIds + hasUserModifiedSelectionRef.current = false + }, [datasets, selectedIds]) + const toggleSelect = (dataSet: DataSet) => { + hasUserModifiedSelectionRef.current = true const isSelected = selected.some(item => item.id === dataSet.id) if (isSelected) { setSelected(selected.filter(item => item.id !== dataSet.id)) @@ -98,13 +117,13 @@ const SelectDataSet: FC = ({ className='w-[400px]' title={t('appDebug.feature.dataSet.selectTitle')} > - {!loaded && ( + {(isLoading && datasets.length === 0) && (
)} - {(loaded && hasNoData) && ( + {hasNoData && (
= ({
)} - {datasets && datasets?.length > 0 && ( + {datasets.length > 0 && ( <>
{datasets.map(item => ( @@ -171,7 +190,7 @@ const SelectDataSet: FC = ({
)} - {loaded && ( + {!isLoading && (
{selected.length > 0 && `${selected.length} ${t('appDebug.feature.dataSet.selected')}`} diff --git a/web/app/components/datasets/common/document-status-with-action/index-failed.tsx b/web/app/components/datasets/common/document-status-with-action/index-failed.tsx index 802e3d872f..4713d944e0 100644 --- a/web/app/components/datasets/common/document-status-with-action/index-failed.tsx +++ b/web/app/components/datasets/common/document-status-with-action/index-failed.tsx @@ -2,11 +2,11 @@ import type { FC } from 'react' import React, { useEffect, useReducer } from 'react' import { useTranslation } from 'react-i18next' -import useSWR from 'swr' import StatusWithAction from './status-with-action' -import { getErrorDocs, retryErrorDocs } from '@/service/datasets' +import { retryErrorDocs } from '@/service/datasets' import type { IndexingStatusResponse } from '@/models/datasets' import { noop } from 'lodash-es' +import { useDatasetErrorDocs } from '@/service/knowledge/use-dataset' type Props = { datasetId: string @@ -35,16 +35,19 @@ const indexStateReducer = (state: IIndexState, action: IAction) => { const RetryButton: FC = ({ datasetId }) => { const { t } = useTranslation() const [indexState, dispatch] = useReducer(indexStateReducer, { value: 'success' }) - const { data: errorDocs, isLoading } = useSWR({ datasetId }, getErrorDocs) + const { data: errorDocs, isLoading, refetch: refetchErrorDocs } = useDatasetErrorDocs(datasetId) const onRetryErrorDocs = async () => { dispatch({ type: 'retry' }) const document_ids = errorDocs?.data.map((doc: IndexingStatusResponse) => doc.id) || [] const res = await retryErrorDocs({ datasetId, document_ids }) - if (res.result === 'success') + if (res.result === 'success') { + refetchErrorDocs() dispatch({ type: 'success' }) - else + } + else { dispatch({ type: 'error' }) + } } useEffect(() => { diff --git a/web/app/components/datasets/create/embedding-process/index.tsx b/web/app/components/datasets/create/embedding-process/index.tsx index 7b2eda1dcd..4e78eb2034 100644 --- a/web/app/components/datasets/create/embedding-process/index.tsx +++ b/web/app/components/datasets/create/embedding-process/index.tsx @@ -1,9 +1,7 @@ import type { FC } from 'react' import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import useSWR from 'swr' import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' -import { omit } from 'lodash-es' import { RiArrowRightLine, RiCheckboxCircleFill, @@ -25,7 +23,7 @@ import type { LegacyDataSourceInfo, ProcessRuleResponse, } from '@/models/datasets' -import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets' +import { fetchIndexingStatusBatch as doFetchIndexingStatus } from '@/service/datasets' import { DataSourceType, ProcessMode } from '@/models/datasets' import NotionIcon from '@/app/components/base/notion-icon' import PriorityLabel from '@/app/components/billing/priority-label' @@ -40,6 +38,7 @@ import { useInvalidDocumentList } from '@/service/knowledge/use-document' import Divider from '@/app/components/base/divider' import { useDatasetApiAccessUrl } from '@/hooks/use-api-access-url' import Link from 'next/link' +import { useProcessRule } from '@/service/knowledge/use-dataset' type Props = { datasetId: string @@ -207,12 +206,7 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index }, []) // get rule - const { data: ruleDetail } = useSWR({ - action: 'fetchProcessRule', - params: { documentId: getFirstDocument.id }, - }, apiParams => fetchProcessRule(omit(apiParams, 'action')), { - revalidateOnFocus: false, - }) + const { data: ruleDetail } = useProcessRule(getFirstDocument?.id) const router = useRouter() const invalidDocumentList = useInvalidDocumentList() diff --git a/web/app/components/datasets/create/file-uploader/index.tsx b/web/app/components/datasets/create/file-uploader/index.tsx index d258ed694e..abe2564ad2 100644 --- a/web/app/components/datasets/create/file-uploader/index.tsx +++ b/web/app/components/datasets/create/file-uploader/index.tsx @@ -2,7 +2,6 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' -import useSWR from 'swr' import { RiDeleteBinLine, RiUploadCloud2Line } from '@remixicon/react' import DocumentFileIcon from '../../common/document-file-icon' import cn from '@/utils/classnames' @@ -11,8 +10,7 @@ import { ToastContext } from '@/app/components/base/toast' import SimplePieChart from '@/app/components/base/simple-pie-chart' import { upload } from '@/service/base' -import { fetchFileUploadConfig } from '@/service/common' -import { fetchSupportFileTypes } from '@/service/datasets' +import { useFileSupportTypes, useFileUploadConfig } from '@/service/use-common' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n-config/language' import { IS_CE_EDITION } from '@/config' @@ -48,8 +46,8 @@ const FileUploader = ({ const fileUploader = useRef(null) const hideUpload = notSupportBatchUpload && fileList.length > 0 - const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig) - const { data: supportFileTypesResponse } = useSWR({ url: '/files/support-type' }, fetchSupportFileTypes) + const { data: fileUploadConfigResponse } = useFileUploadConfig() + const { data: supportFileTypesResponse } = useFileSupportTypes() const supportTypes = supportFileTypesResponse?.allowed_extensions || [] const supportTypesShowNames = (() => { const extensionMap: { [key: string]: string } = { diff --git a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx index 317db84c43..2049ae0d03 100644 --- a/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx +++ b/web/app/components/datasets/documents/detail/batch-modal/csv-uploader.tsx @@ -13,11 +13,10 @@ import Button from '@/app/components/base/button' import type { FileItem } from '@/models/datasets' import { upload } from '@/service/base' import { getFileUploadErrorMessage } from '@/app/components/base/file-uploader/utils' -import useSWR from 'swr' -import { fetchFileUploadConfig } from '@/service/common' import SimplePieChart from '@/app/components/base/simple-pie-chart' import { Theme } from '@/types/app' import useTheme from '@/hooks/use-theme' +import { useFileUploadConfig } from '@/service/use-common' export type Props = { file: FileItem | undefined @@ -34,7 +33,7 @@ const CSVUploader: FC = ({ const dropRef = useRef(null) const dragRef = useRef(null) const fileUploader = useRef(null) - const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig) + const { data: fileUploadConfigResponse } = useFileUploadConfig() const fileUploadConfig = useMemo(() => fileUploadConfigResponse ?? { file_size_limit: 15, }, [fileUploadConfigResponse]) diff --git a/web/app/components/datasets/documents/detail/embedding/index.tsx b/web/app/components/datasets/documents/detail/embedding/index.tsx index 57c9b77960..ff5b7ec4b7 100644 --- a/web/app/components/datasets/documents/detail/embedding/index.tsx +++ b/web/app/components/datasets/documents/detail/embedding/index.tsx @@ -1,9 +1,7 @@ import type { FC } from 'react' import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' -import useSWR from 'swr' import { useContext } from 'use-context-selector' import { useTranslation } from 'react-i18next' -import { omit } from 'lodash-es' import { RiLoader2Line, RiPauseCircleLine, RiPlayCircleLine } from '@remixicon/react' import Image from 'next/image' import { FieldInfo } from '../metadata' @@ -21,10 +19,10 @@ import type { CommonResponse } from '@/models/common' import { asyncRunSafe, sleep } from '@/utils' import { fetchIndexingStatus as doFetchIndexingStatus, - fetchProcessRule, pauseDocIndexing, resumeDocIndexing, } from '@/service/datasets' +import { useProcessRule } from '@/service/knowledge/use-dataset' type IEmbeddingDetailProps = { datasetId?: string @@ -207,12 +205,7 @@ const EmbeddingDetail: FC = ({ } }, [startQueryStatus, stopQueryStatus]) - const { data: ruleDetail } = useSWR({ - action: 'fetchProcessRule', - params: { documentId: localDocumentId }, - }, apiParams => fetchProcessRule(omit(apiParams, 'action')), { - revalidateOnFocus: false, - }) + const { data: ruleDetail } = useProcessRule(localDocumentId) const isEmbedding = useMemo(() => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail]) const isEmbeddingCompleted = useMemo(() => ['completed'].includes(indexingStatusDetail?.indexing_status || ''), [indexingStatusDetail]) diff --git a/web/app/components/datasets/hit-testing/index.tsx b/web/app/components/datasets/hit-testing/index.tsx index 2917b8511a..e9e3b0014a 100644 --- a/web/app/components/datasets/hit-testing/index.tsx +++ b/web/app/components/datasets/hit-testing/index.tsx @@ -32,9 +32,8 @@ import Records from './components/records' import { useExternalKnowledgeBaseHitTesting, useHitTesting, - useHitTestingRecords, - useInvalidateHitTestingRecords, } from '@/service/knowledge/use-hit-testing' +import { useDatasetTestingRecords } from '@/service/knowledge/use-dataset' const limit = 10 @@ -48,14 +47,13 @@ const HitTestingPage: FC = ({ datasetId }: Props) => { const media = useBreakpoints() const isMobile = media === MediaType.mobile - const [hitResult, setHitResult] = useState() // 初始化记录为空数组 + const [hitResult, setHitResult] = useState() const [externalHitResult, setExternalHitResult] = useState() const [queries, setQueries] = useState([]) const [queryInputKey, setQueryInputKey] = useState(Date.now()) const [currPage, setCurrPage] = useState(0) - const { data: recordsRes, isLoading: isRecordsLoading } = useHitTestingRecords({ datasetId, page: currPage + 1, limit }) - const invalidateHitTestingRecords = useInvalidateHitTestingRecords(datasetId) + const { data: recordsRes, refetch: recordsRefetch, isLoading: isRecordsLoading } = useDatasetTestingRecords(datasetId, { limit, page: currPage + 1 }) const total = recordsRes?.total || 0 @@ -107,8 +105,7 @@ const HitTestingPage: FC = ({ datasetId }: Props) => { ) const handleClickRecord = useCallback((record: HitTestingRecord) => { - const { queries } = record - setQueries(queries) + setQueries(record.queries) setQueryInputKey(Date.now()) }, []) @@ -128,7 +125,7 @@ const HitTestingPage: FC = ({ datasetId }: Props) => { setHitResult={setHitResult} setExternalHitResult={setExternalHitResult} onSubmit={showRightPanel} - onUpdateList={invalidateHitTestingRecords} + onUpdateList={recordsRefetch} loading={isRetrievalLoading} queries={queries} setQueries={setQueries} @@ -140,11 +137,9 @@ const HitTestingPage: FC = ({ datasetId }: Props) => { externalKnowledgeBaseHitTestingMutation={externalKnowledgeBaseHitTestingMutation} />
{t('datasetHitTesting.records')}
- {isRecordsLoading - && ( -
- ) - } + {isRecordsLoading && ( +
+ )} {!isRecordsLoading && recordsRes?.data && recordsRes.data.length > 0 && ( <> diff --git a/web/app/components/develop/secret-key/secret-key-modal.tsx b/web/app/components/develop/secret-key/secret-key-modal.tsx index 0c0a5091b7..24935b5b98 100644 --- a/web/app/components/develop/secret-key/secret-key-modal.tsx +++ b/web/app/components/develop/secret-key/secret-key-modal.tsx @@ -5,7 +5,6 @@ import { import { useTranslation } from 'react-i18next' import { RiDeleteBinLine } from '@remixicon/react' import { PlusIcon, XMarkIcon } from '@heroicons/react/20/solid' -import useSWR from 'swr' import SecretKeyGenerateModal from './secret-key-generate' import s from './style.module.css' import ActionButton from '@/app/components/base/action-button' @@ -19,7 +18,6 @@ import { import { createApikey as createDatasetApikey, delApikey as delDatasetApikey, - fetchApiKeysList as fetchDatasetApiKeysList, } from '@/service/datasets' import type { CreateApiKeyResponse } from '@/models/app' import Loading from '@/app/components/base/loading' @@ -27,6 +25,7 @@ import Confirm from '@/app/components/base/confirm' import useTimestamp from '@/hooks/use-timestamp' import { useAppContext } from '@/context/app-context' import { useAppApiKeys, useInvalidateAppApiKeys } from '@/service/use-apps' +import { useDatasetApiKeys, useInvalidateDatasetApiKeys } from '@/service/knowledge/use-dataset' type ISecretKeyModalProps = { isShow: boolean @@ -46,11 +45,9 @@ const SecretKeyModal = ({ const [isVisible, setVisible] = useState(false) const [newKey, setNewKey] = useState(undefined) const invalidateAppApiKeys = useInvalidateAppApiKeys() + const invalidateDatasetApiKeys = useInvalidateDatasetApiKeys() const { data: appApiKeys, isLoading: isAppApiKeysLoading } = useAppApiKeys(appId, { enabled: !!appId && isShow }) - const { data: datasetApiKeys, isLoading: isDatasetApiKeysLoading, mutate: mutateDatasetApiKeys } = useSWR( - !appId && isShow ? { url: '/datasets/api-keys', params: {} } : null, - fetchDatasetApiKeysList, - ) + const { data: datasetApiKeys, isLoading: isDatasetApiKeysLoading } = useDatasetApiKeys({ enabled: !appId && isShow }) const apiKeysList = appId ? appApiKeys : datasetApiKeys const isApiKeysLoading = appId ? isAppApiKeysLoading : isDatasetApiKeysLoading @@ -69,7 +66,7 @@ const SecretKeyModal = ({ if (appId) invalidateAppApiKeys(appId) else - mutateDatasetApiKeys() + invalidateDatasetApiKeys() } const onCreate = async () => { @@ -83,7 +80,7 @@ const SecretKeyModal = ({ if (appId) invalidateAppApiKeys(appId) else - mutateDatasetApiKeys() + invalidateDatasetApiKeys() } const generateToken = (token: string) => { diff --git a/web/app/components/workflow/nodes/document-extractor/panel.tsx b/web/app/components/workflow/nodes/document-extractor/panel.tsx index 572ca366ca..7165dc06df 100644 --- a/web/app/components/workflow/nodes/document-extractor/panel.tsx +++ b/web/app/components/workflow/nodes/document-extractor/panel.tsx @@ -1,6 +1,5 @@ import type { FC } from 'react' import React from 'react' -import useSWR from 'swr' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import VarReferencePicker from '../_base/components/variable/var-reference-picker' @@ -9,11 +8,11 @@ import Split from '../_base/components/split' import { useNodeHelpLink } from '../_base/hooks/use-node-help-link' import useConfig from './use-config' import type { DocExtractorNodeType } from './types' -import { fetchSupportFileTypes } from '@/service/datasets' import Field from '@/app/components/workflow/nodes/_base/components/field' import { BlockEnum, type NodePanelProps } from '@/app/components/workflow/types' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n-config/language' +import { useFileSupportTypes } from '@/service/use-common' const i18nPrefix = 'workflow.nodes.docExtractor' @@ -24,7 +23,7 @@ const Panel: FC> = ({ const { t } = useTranslation() const { locale } = useContext(I18n) const link = useNodeHelpLink(BlockEnum.DocExtractor) - const { data: supportFileTypesResponse } = useSWR({ url: '/files/support-type' }, fetchSupportFileTypes) + const { data: supportFileTypesResponse } = useFileSupportTypes() const supportTypes = supportFileTypesResponse?.allowed_extensions || [] const supportTypesShowNames = (() => { const extensionMap: { [key: string]: string } = { diff --git a/web/context/external-knowledge-api-context.tsx b/web/context/external-knowledge-api-context.tsx index 5f2d2ff393..9bf6ece70b 100644 --- a/web/context/external-knowledge-api-context.tsx +++ b/web/context/external-knowledge-api-context.tsx @@ -1,10 +1,9 @@ 'use client' -import { createContext, useContext, useMemo } from 'react' +import { createContext, useCallback, useContext, useMemo } from 'react' import type { FC, ReactNode } from 'react' -import useSWR from 'swr' import type { ExternalAPIItem, ExternalAPIListResponse } from '@/models/datasets' -import { fetchExternalAPIList } from '@/service/datasets' +import { useExternalKnowledgeApiList } from '@/service/knowledge/use-dataset' type ExternalKnowledgeApiContextType = { externalKnowledgeApiList: ExternalAPIItem[] @@ -19,10 +18,11 @@ export type ExternalKnowledgeApiProviderProps = { } export const ExternalKnowledgeApiProvider: FC = ({ children }) => { - const { data, mutate: mutateExternalKnowledgeApis, isLoading } = useSWR( - { url: '/datasets/external-knowledge-api' }, - fetchExternalAPIList, - ) + const { data, refetch, isLoading } = useExternalKnowledgeApiList() + + const mutateExternalKnowledgeApis = useCallback(() => { + return refetch().then(res => res.data) + }, [refetch]) const contextValue = useMemo(() => ({ externalKnowledgeApiList: data?.data || [], diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 0160a8a940..624da433f8 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -1,4 +1,3 @@ -import type { Fetcher } from 'swr' import qs from 'qs' import { del, get, patch, post, put } from './base' import type { @@ -50,140 +49,143 @@ export type SortType = 'created_at' | 'hit_count' | '-created_at' | '-hit_count' export type MetadataType = 'all' | 'only' | 'without' -export const fetchDatasetDetail: Fetcher = (datasetId: string) => { +export const fetchDatasetDetail = (datasetId: string): Promise => { return get(`/datasets/${datasetId}`) } -export const updateDatasetSetting: Fetcher> -}> = ({ datasetId, body }) => { +}): Promise => { return patch(`/datasets/${datasetId}`, { body }) } -export const fetchDatasetRelatedApps: Fetcher = (datasetId: string) => { +export const fetchDatasetRelatedApps = (datasetId: string): Promise => { return get(`/datasets/${datasetId}/related-apps`) } -export const fetchDatasets: Fetcher = ({ url, params }) => { +export const fetchDatasets = ({ url, params }: FetchDatasetsParams): Promise => { const urlParams = qs.stringify(params, { indices: false }) return get(`${url}?${urlParams}`) } -export const createEmptyDataset: Fetcher = ({ name }) => { +export const createEmptyDataset = ({ name }: { name: string }): Promise => { return post('/datasets', { body: { name } }) } -export const checkIsUsedInApp: Fetcher<{ is_using: boolean }, string> = (id) => { +export const checkIsUsedInApp = (id: string): Promise<{ is_using: boolean }> => { return get<{ is_using: boolean }>(`/datasets/${id}/use-check`, {}, { silent: true, }) } -export const deleteDataset: Fetcher = (datasetID) => { +export const deleteDataset = (datasetID: string): Promise => { return del(`/datasets/${datasetID}`) } -export const fetchExternalAPIList: Fetcher = ({ url }) => { +export const fetchExternalAPIList = ({ url }: { url: string }): Promise => { return get(url) } -export const fetchExternalAPI: Fetcher = ({ apiTemplateId }) => { +export const fetchExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return get(`/datasets/external-knowledge-api/${apiTemplateId}`) } -export const updateExternalAPI: Fetcher = ({ apiTemplateId, body }) => { +export const updateExternalAPI = ({ apiTemplateId, body }: { apiTemplateId: string; body: ExternalAPIItem }): Promise => { return patch(`/datasets/external-knowledge-api/${apiTemplateId}`, { body }) } -export const deleteExternalAPI: Fetcher = ({ apiTemplateId }) => { +export const deleteExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return del(`/datasets/external-knowledge-api/${apiTemplateId}`) } -export const checkUsageExternalAPI: Fetcher = ({ apiTemplateId }) => { +export const checkUsageExternalAPI = ({ apiTemplateId }: { apiTemplateId: string }): Promise => { return get(`/datasets/external-knowledge-api/${apiTemplateId}/use-check`) } -export const createExternalAPI: Fetcher = ({ body }) => { +export const createExternalAPI = ({ body }: { body: CreateExternalAPIReq }): Promise => { return post('/datasets/external-knowledge-api', { body }) } -export const createExternalKnowledgeBase: Fetcher = ({ body }) => { +export const createExternalKnowledgeBase = ({ body }: { body: CreateKnowledgeBaseReq }): Promise => { return post('/datasets/external', { body }) } -export const fetchDefaultProcessRule: Fetcher = ({ url }) => { +export const fetchDefaultProcessRule = ({ url }: { url: string }): Promise => { return get(url) } -export const fetchProcessRule: Fetcher = ({ params: { documentId } }) => { +export const fetchProcessRule = ({ params: { documentId } }: { params: { documentId: string } }): Promise => { return get('/datasets/process-rule', { params: { document_id: documentId } }) } -export const createFirstDocument: Fetcher = ({ body }) => { +export const createFirstDocument = ({ body }: { body: CreateDocumentReq }): Promise => { return post('/datasets/init', { body }) } -export const createDocument: Fetcher = ({ datasetId, body }) => { +export const createDocument = ({ datasetId, body }: { datasetId: string; body: CreateDocumentReq }): Promise => { return post(`/datasets/${datasetId}/documents`, { body }) } -export const fetchIndexingEstimate: Fetcher = ({ datasetId, documentId }) => { +export const fetchIndexingEstimate = ({ datasetId, documentId }: CommonDocReq): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/indexing-estimate`, {}) } -export const fetchIndexingEstimateBatch: Fetcher = ({ datasetId, batchId }) => { +export const fetchIndexingEstimateBatch = ({ datasetId, batchId }: BatchReq): Promise => { return get(`/datasets/${datasetId}/batch/${batchId}/indexing-estimate`, {}) } -export const fetchIndexingStatus: Fetcher = ({ datasetId, documentId }) => { +export const fetchIndexingStatus = ({ datasetId, documentId }: CommonDocReq): Promise => { return get(`/datasets/${datasetId}/documents/${documentId}/indexing-status`, {}) } -export const fetchIndexingStatusBatch: Fetcher = ({ datasetId, batchId }) => { +export const fetchIndexingStatusBatch = ({ datasetId, batchId }: BatchReq): Promise => { return get(`/datasets/${datasetId}/batch/${batchId}/indexing-status`, {}) } -export const renameDocumentName: Fetcher = ({ datasetId, documentId, name }) => { +export const renameDocumentName = ({ datasetId, documentId, name }: CommonDocReq & { name: string }): Promise => { return post(`/datasets/${datasetId}/documents/${documentId}/rename`, { body: { name }, }) } -export const pauseDocIndexing: Fetcher = ({ datasetId, documentId }) => { +export const pauseDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise => { return patch(`/datasets/${datasetId}/documents/${documentId}/processing/pause`) } -export const resumeDocIndexing: Fetcher = ({ datasetId, documentId }) => { +export const resumeDocIndexing = ({ datasetId, documentId }: CommonDocReq): Promise => { return patch(`/datasets/${datasetId}/documents/${documentId}/processing/resume`) } -export const preImportNotionPages: Fetcher<{ notion_info: DataSourceNotionWorkspace[] }, { url: string; datasetId?: string }> = ({ url, datasetId }) => { +export const preImportNotionPages = ({ url, datasetId }: { url: string; datasetId?: string }): Promise<{ notion_info: DataSourceNotionWorkspace[] }> => { return get<{ notion_info: DataSourceNotionWorkspace[] }>(url, { params: { dataset_id: datasetId } }) } -export const modifyDocMetadata: Fetcher } }> = ({ datasetId, documentId, body }) => { +export const modifyDocMetadata = ({ datasetId, documentId, body }: CommonDocReq & { body: { doc_type: string; doc_metadata: Record } }): Promise => { return put(`/datasets/${datasetId}/documents/${documentId}/metadata`, { body }) } // hit testing -export const hitTesting: Fetcher = ({ datasetId, queryText, retrieval_model }) => { +export const hitTesting = ({ datasetId, queryText, retrieval_model }: { datasetId: string; queryText: string; retrieval_model: RetrievalConfig }): Promise => { return post(`/datasets/${datasetId}/hit-testing`, { body: { query: queryText, retrieval_model } }) } -export const externalKnowledgeBaseHitTesting: Fetcher = ({ datasetId, query, external_retrieval_model }) => { +export const externalKnowledgeBaseHitTesting = ({ datasetId, query, external_retrieval_model }: { datasetId: string; query: string; external_retrieval_model: { top_k: number; score_threshold: number; score_threshold_enabled: boolean } }): Promise => { return post(`/datasets/${datasetId}/external-hit-testing`, { body: { query, external_retrieval_model } }) } -export const fetchTestingRecords: Fetcher = ({ datasetId, params }) => { +export const fetchTestingRecords = ({ datasetId, params }: { datasetId: string; params: { page: number; limit: number } }): Promise => { return get(`/datasets/${datasetId}/queries`, { params }) } -export const fetchFileIndexingEstimate: Fetcher = (body: IndexingEstimateParams) => { +export const fetchFileIndexingEstimate = (body: IndexingEstimateParams): Promise => { return post('/datasets/indexing-estimate', { body }) } -export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: string; pageID: string; pageType: string; credentialID: string; }> = ({ workspaceID, pageID, pageType, credentialID }) => { +export const fetchNotionPagePreview = ({ workspaceID, pageID, pageType, credentialID }: { workspaceID: string; pageID: string; pageType: string; credentialID: string }): Promise<{ content: string }> => { return get<{ content: string }>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`, { params: { credential_id: credentialID, @@ -191,31 +193,31 @@ export const fetchNotionPagePreview: Fetcher<{ content: string }, { workspaceID: }) } -export const fetchApiKeysList: Fetcher }> = ({ url, params }) => { +export const fetchApiKeysList = ({ url, params }: { url: string; params: Record }): Promise => { return get(url, params) } -export const delApikey: Fetcher }> = ({ url, params }) => { +export const delApikey = ({ url, params }: { url: string; params: Record }): Promise => { return del(url, params) } -export const createApikey: Fetcher }> = ({ url, body }) => { +export const createApikey = ({ url, body }: { url: string; body: Record }): Promise => { return post(url, body) } -export const fetchDataSources = () => { +export const fetchDataSources = (): Promise => { return get('api-key-auth/data-source') } -export const createDataSourceApiKeyBinding: Fetcher> = (body) => { +export const createDataSourceApiKeyBinding = (body: Record): Promise => { return post('api-key-auth/data-source/binding', { body }) } -export const removeDataSourceApiKeyBinding: Fetcher = (id: string) => { +export const removeDataSourceApiKeyBinding = (id: string): Promise => { return del(`api-key-auth/data-source/${id}`) } -export const createFirecrawlTask: Fetcher> = (body) => { +export const createFirecrawlTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, @@ -224,7 +226,7 @@ export const createFirecrawlTask: Fetcher> = }) } -export const checkFirecrawlTaskStatus: Fetcher = (jobId: string) => { +export const checkFirecrawlTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: DataSourceProvider.fireCrawl, @@ -234,7 +236,7 @@ export const checkFirecrawlTaskStatus: Fetcher = (jobId: }) } -export const createJinaReaderTask: Fetcher> = (body) => { +export const createJinaReaderTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, @@ -243,7 +245,7 @@ export const createJinaReaderTask: Fetcher> }) } -export const checkJinaReaderTaskStatus: Fetcher = (jobId: string) => { +export const checkJinaReaderTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: 'jinareader', @@ -253,7 +255,7 @@ export const checkJinaReaderTaskStatus: Fetcher = (jobId }) } -export const createWatercrawlTask: Fetcher> = (body) => { +export const createWatercrawlTask = (body: Record): Promise => { return post('website/crawl', { body: { ...body, @@ -262,7 +264,7 @@ export const createWatercrawlTask: Fetcher> }) } -export const checkWatercrawlTaskStatus: Fetcher = (jobId: string) => { +export const checkWatercrawlTaskStatus = (jobId: string): Promise => { return get(`website/crawl/status/${jobId}`, { params: { provider: DataSourceProvider.waterCrawl, @@ -276,14 +278,14 @@ export type FileTypesRes = { allowed_extensions: string[] } -export const fetchSupportFileTypes: Fetcher = ({ url }) => { +export const fetchSupportFileTypes = ({ url }: { url: string }): Promise => { return get(url) } -export const getErrorDocs: Fetcher = ({ datasetId }) => { +export const getErrorDocs = ({ datasetId }: { datasetId: string }): Promise => { return get(`/datasets/${datasetId}/error-docs`) } -export const retryErrorDocs: Fetcher = ({ datasetId, document_ids }) => { +export const retryErrorDocs = ({ datasetId, document_ids }: { datasetId: string; document_ids: string[] }): Promise => { return post(`/datasets/${datasetId}/retry`, { body: { document_ids } }) } diff --git a/web/service/knowledge/use-dataset.ts b/web/service/knowledge/use-dataset.ts index 0caea05f6b..2b0c78b249 100644 --- a/web/service/knowledge/use-dataset.ts +++ b/web/service/knowledge/use-dataset.ts @@ -1,23 +1,86 @@ import type { MutationOptions } from '@tanstack/react-query' -import { useInfiniteQuery, useMutation, useQuery } from '@tanstack/react-query' +import { + keepPreviousData, + useInfiniteQuery, + useMutation, + useQuery, + useQueryClient, +} from '@tanstack/react-query' +import qs from 'qs' import type { DataSet, DataSetListResponse, DatasetListRequest, + ErrorDocsResponse, + ExternalAPIListResponse, + FetchDatasetsParams, + HitTestingRecordsResponse, IndexingStatusBatchRequest, IndexingStatusBatchResponse, ProcessRuleResponse, RelatedAppResponse, } from '@/models/datasets' +import type { ApiKeysListResponse } from '@/models/app' import { get, post } from '../base' import { useInvalid } from '../use-base' -import qs from 'qs' import type { CommonResponse } from '@/models/common' const NAME_SPACE = 'dataset' const DatasetListKey = [NAME_SPACE, 'list'] +const normalizeDatasetsParams = (params: Partial = {}) => { + const { + page = 1, + limit, + ids, + tag_ids, + include_all, + keyword, + } = params + + return { + page, + ...(limit ? { limit } : {}), + ...(ids?.length ? { ids } : {}), + ...(tag_ids?.length ? { tag_ids } : {}), + ...(include_all !== undefined ? { include_all } : {}), + ...(keyword ? { keyword } : {}), + } +} + +type UseInfiniteDatasetsOptions = { + enabled?: boolean + refetchOnMount?: boolean | 'always' + staleTime?: number + refetchOnReconnect?: boolean + refetchOnWindowFocus?: boolean +} + +export const useInfiniteDatasets = ( + params: Partial, + options?: UseInfiniteDatasetsOptions, +) => { + const normalizedParams = normalizeDatasetsParams(params) + const buildUrl = (pageParam: number | undefined) => { + const queryString = qs.stringify({ + ...normalizedParams, + page: pageParam ?? normalizedParams.page, + }, { indices: false }) + return `/datasets?${queryString}` + } + + return useInfiniteQuery({ + queryKey: [...DatasetListKey, 'infinite', normalizedParams], + queryFn: ({ pageParam = normalizedParams.page }) => get(buildUrl(pageParam as number | undefined)), + getNextPageParam: lastPage => lastPage.has_more ? lastPage.page + 1 : undefined, + initialPageParam: normalizedParams.page, + staleTime: 0, + refetchOnMount: 'always', + ...options, + }) +} + export const useDatasetList = (params: DatasetListRequest) => { const { initialPage, tag_ids, limit, include_all, keyword } = params return useInfiniteQuery({ @@ -70,10 +133,12 @@ export const useIndexingStatusBatch = ( }) } -export const useProcessRule = (documentId: string) => { +export const useProcessRule = (documentId?: string) => { return useQuery({ queryKey: [NAME_SPACE, 'process-rule', documentId], queryFn: () => get('/datasets/process-rule', { params: { document_id: documentId } }), + enabled: !!documentId, + refetchOnWindowFocus: false, }) } @@ -97,3 +162,57 @@ export const useDisableDatasetServiceApi = () => { mutationFn: (datasetId: string) => post(`/datasets/${datasetId}/api-keys/disable`), }) } + +export const useDatasetApiKeys = (options?: { enabled?: boolean }) => { + return useQuery({ + queryKey: [NAME_SPACE, 'api-keys'], + queryFn: () => get('/datasets/api-keys'), + enabled: options?.enabled ?? true, + }) +} + +export const useInvalidateDatasetApiKeys = () => { + const queryClient = useQueryClient() + return () => { + queryClient.invalidateQueries({ + queryKey: [NAME_SPACE, 'api-keys'], + }) + } +} + +export const useExternalKnowledgeApiList = (options?: { enabled?: boolean }) => { + return useQuery({ + queryKey: [NAME_SPACE, 'external-knowledge-api'], + queryFn: () => get('/datasets/external-knowledge-api'), + enabled: options?.enabled ?? true, + }) +} + +export const useInvalidateExternalKnowledgeApiList = () => { + const queryClient = useQueryClient() + return () => { + queryClient.invalidateQueries({ + queryKey: [NAME_SPACE, 'external-knowledge-api'], + }) + } +} + +export const useDatasetTestingRecords = ( + datasetId?: string, + params?: { page: number; limit: number }, +) => { + return useQuery({ + queryKey: [NAME_SPACE, 'testing-records', datasetId, params], + queryFn: () => get(`/datasets/${datasetId}/queries`, { params }), + enabled: !!datasetId && !!params, + placeholderData: keepPreviousData, + }) +} + +export const useDatasetErrorDocs = (datasetId?: string) => { + return useQuery({ + queryKey: [NAME_SPACE, 'error-docs', datasetId], + queryFn: () => get(`/datasets/${datasetId}/error-docs`), + enabled: !!datasetId, + }) +}