diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index c6e5e82194..30c2db1276 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -25,11 +25,7 @@ import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs' import PreviewItem, { PreviewType } from './preview-item' import cn from '@/utils/classnames' import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FullDocumentDetail, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' -import { - createDocument, - createFirstDocument, - fetchDefaultProcessRule, -} from '@/service/datasets' + import Button from '@/app/components/base/button' import FloatRightContainer from '@/app/components/base/float-right-container' import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' @@ -55,7 +51,7 @@ import { MessageChatSquare } from '@/app/components/base/icons/src/public/common import { IS_CE_EDITION } from '@/config' import Switch from '@/app/components/base/switch' import Divider from '@/app/components/base/divider' -import { getNotionInfo, getWebsiteInfo, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets' +import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/use-datasets' import Loading from '@/app/components/base/loading' const TextLabel: FC = (props) => { @@ -174,8 +170,7 @@ const StepTwo = ({ (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'), ) const [QATipHide, setQATipHide] = useState(false) - const [previewSwitched, setPreviewSwitched] = useState(false) - const [isCreating, setIsCreating] = useState(false) + const [qaPreviewSwitched, setQAPreviewSwitched] = useState(false) const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig) @@ -250,21 +245,21 @@ const StepTwo = ({ ? notionIndexingEstimateQuery.data : websiteIndexingEstimateQuery.data - const getIsEstimateReady = useCallback(() => { - if (dataSourceType === DataSourceType.FILE) - return fileIndexingEstimateQuery.isSuccess + // const getIsEstimateReady = useCallback(() => { + // if (dataSourceType === DataSourceType.FILE) + // return fileIndexingEstimateQuery.isSuccess - if (dataSourceType === DataSourceType.NOTION) - return notionIndexingEstimateQuery.isSuccess + // if (dataSourceType === DataSourceType.NOTION) + // return notionIndexingEstimateQuery.isSuccess - if (dataSourceType === DataSourceType.WEB) - return websiteIndexingEstimateQuery.isSuccess - }, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess]) + // if (dataSourceType === DataSourceType.WEB) + // return websiteIndexingEstimateQuery.isSuccess + // }, [dataSourceType, fileIndexingEstimateQuery.isSuccess, notionIndexingEstimateQuery.isSuccess, websiteIndexingEstimateQuery.isSuccess]) - const getFileName = (name: string) => { - const arr = name.split('.') - return arr.slice(0, -1).join('.') - } + // const getFileName = (name: string) => { + // const arr = name.split('.') + // return arr.slice(0, -1).join('.') + // } const getRuleName = (key: string) => { if (key === 'remove_extra_spaces') @@ -304,7 +299,7 @@ const StepTwo = ({ return } fetchEstimate() - setPreviewSwitched(false) + setQAPreviewSwitched(false) } const { @@ -403,20 +398,22 @@ const StepTwo = ({ return params } - const getRules = async () => { - try { - const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' }) - const separator = res.rules.segmentation.separator + const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({ + onSuccess(data) { + const separator = data.rules.segmentation.separator setSegmentIdentifier(separator) - setMax(res.rules.segmentation.max_tokens) - setOverlap(res.rules.segmentation.chunk_overlap!) - setRules(res.rules.pre_processing_rules) - setDefaultConfig(res.rules) - } - catch (err) { - console.log(err) - } - } + setMax(data.rules.segmentation.max_tokens) + setOverlap(data.rules.segmentation.chunk_overlap!) + setRules(data.rules.pre_processing_rules) + setDefaultConfig(data.rules) + }, + onError(error) { + Toast.notify({ + type: 'error', + message: `${error}`, + }) + }, + }) const getRulesFromDetail = () => { if (documentDetail) { @@ -426,7 +423,7 @@ const StepTwo = ({ const overlap = rules.segmentation.chunk_overlap setSegmentIdentifier(separator) setMax(max) - setOverlap(overlap) + setOverlap(overlap as number) setRules(rules.pre_processing_rules) setDefaultConfig(rules) } @@ -437,48 +434,55 @@ const StepTwo = ({ setSegmentationType(documentDetail.dataset_process_rule.mode) } - const createHandle = async () => { - if (isCreating) - return - setIsCreating(true) - try { - let res - const params = getCreationParams() - if (!params) - return false - - setIsCreating(true) - if (!datasetId) { - res = await createFirstDocument({ - body: params as CreateDocumentReq, - }) - updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) - updateResultCache && updateResultCache(res) - // eslint-disable-next-line @typescript-eslint/no-use-before-define - updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string) - } - else { - res = await createDocument({ - datasetId, - body: params as CreateDocumentReq, - }) - updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) - updateResultCache && updateResultCache(res) - } - if (mutateDatasetRes) - mutateDatasetRes() - onStepChange && onStepChange(+1) - isSetting && onSave && onSave() - } - catch (err) { + const createFirstDocumentMutation = useCreateFirstDocument({ + onError(error) { Toast.notify({ type: 'error', - message: `${err}`, + message: `${error}`, + }) + }, + }) + const createDocumentMutation = useCreateDocument(datasetId!, { + onError(error) { + Toast.notify({ + type: 'error', + message: `${error}`, + }) + }, + }) + + const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending + + const createHandle = async () => { + const params = getCreationParams() + if (!params) + return false + + if (!datasetId) { + await createFirstDocumentMutation.mutateAsync( + params, + { + onSuccess(data) { + updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) + updateResultCache && updateResultCache(data) + // eslint-disable-next-line @typescript-eslint/no-use-before-define + updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string) + }, + }, + ) + } + else { + await createDocumentMutation.mutateAsync(params, { + onSuccess(data) { + updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) + updateResultCache && updateResultCache(data) + }, }) } - finally { - setIsCreating(false) - } + if (mutateDatasetRes) + mutateDatasetRes() + onStepChange && onStepChange(+1) + isSetting && onSave && onSave() } const handleDocformSwitch = (isQAMode: boolean) => { @@ -488,8 +492,8 @@ const StepTwo = ({ setDocForm(DocForm.TEXT) } - const previewSwitch = async (language?: string) => { - setPreviewSwitched(true) + const previewSwitch = () => { + setQAPreviewSwitched(true) setIsLanguageSelectDisabled(true) fetchEstimate() } @@ -497,8 +501,8 @@ const StepTwo = ({ const handleSelect = (language: string) => { setDocLanguage(language) // Switch language, re-cutter - if (docForm === DocForm.QA && previewSwitched) - previewSwitch(language) + if (docForm === DocForm.QA && qaPreviewSwitched) + previewSwitch() } const changeToEconomicalType = () => { @@ -511,7 +515,7 @@ const StepTwo = ({ useEffect(() => { // fetch rules if (!isSetting) { - getRules() + fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule') } else { getRulesFromDetail() @@ -909,12 +913,12 @@ const StepTwo = ({
{t('datasetCreation.stepTwo.previewTitle')}
- {docForm === DocForm.QA && !previewSwitched && ( + {docForm === DocForm.QA && !qaPreviewSwitched && ( )}
- {docForm === DocForm.QA && !previewSwitched && ( + {docForm === DocForm.QA && !qaPreviewSwitched && (
{t('datasetCreation.stepTwo.previewSwitchTipStart')} {t('datasetCreation.stepTwo.previewSwitchTipEnd')} @@ -922,26 +926,26 @@ const StepTwo = ({ )}
- {previewSwitched && docForm === DocForm.QA && estimate?.qa_preview && ( + {qaPreviewSwitched && docForm === DocForm.QA && estimate?.qa_preview && ( <> {estimate?.qa_preview.map((item, index) => ( ))} )} - {(docForm === DocForm.TEXT || !previewSwitched) && estimate?.preview && ( + {(docForm === DocForm.TEXT || !qaPreviewSwitched) && estimate?.preview && ( <> {estimate?.preview.map((item, index) => ( ))} )} - {previewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && ( + {qaPreviewSwitched && docForm === DocForm.QA && !estimate?.qa_preview && (
)} - {!previewSwitched && !estimate?.preview && ( + {!qaPreviewSwitched && !estimate?.preview && (
diff --git a/web/service/use-datasets.ts b/web/service/use-datasets.ts index 221e258100..a00c34ec12 100644 --- a/web/service/use-datasets.ts +++ b/web/service/use-datasets.ts @@ -1,9 +1,9 @@ import groupBy from 'lodash-es/groupBy' import type { MutationOptions } from '@tanstack/react-query' import { useMutation } from '@tanstack/react-query' -import { fetchFileIndexingEstimate } from './datasets' +import { createDocument, createFirstDocument, fetchDefaultProcessRule, fetchFileIndexingEstimate } from './datasets' import { type IndexingType } from '@/app/components/datasets/create/step-two' -import type { CrawlOptions, CrawlResultItem, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule } from '@/models/datasets' +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DataSourceType, DocForm, FileIndexingEstimateResponse, IndexingEstimateParams, NotionInfo, ProcessRule, ProcessRuleResponse, createDocumentResponse } from '@/models/datasets' import type { DataSourceProvider, NotionPage } from '@/models/common' export const getNotionInfo = ( @@ -62,19 +62,6 @@ type GetFileIndexingEstimateParamsOptionFile = GetFileIndexingEstimateParamsOpti files: CustomFile[] } -type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & { - dataSourceType: DataSourceType.NOTION - notionPages: NotionPage[] -} - -type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & { - dataSourceType: DataSourceType.WEB - websitePages: CrawlResultItem[] - crawlOptions?: CrawlOptions - websiteCrawlProvider: DataSourceProvider - websiteCrawlJobId: string -} - const getFileIndexingEstimateParamsForFile = ({ docForm, docLanguage, @@ -99,6 +86,23 @@ const getFileIndexingEstimateParamsForFile = ({ } } +export const useFetchFileIndexingEstimateForFile = ( + options: GetFileIndexingEstimateParamsOptionFile, + mutationOptions: MutationOptions = {}, +) => { + return useMutation({ + mutationFn: async () => { + return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options)) + }, + ...mutationOptions, + }) +} + +type GetFileIndexingEstimateParamsOptionNotion = GetFileIndexingEstimateParamsOptionBase & { + dataSourceType: DataSourceType.NOTION + notionPages: NotionPage[] +} + const getFileIndexingEstimateParamsForNotion = ({ docForm, docLanguage, @@ -121,6 +125,26 @@ const getFileIndexingEstimateParamsForNotion = ({ } } +export const useFetchFileIndexingEstimateForNotion = ( + options: GetFileIndexingEstimateParamsOptionNotion, + mutationOptions: MutationOptions = {}, +) => { + return useMutation({ + mutationFn: async () => { + return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options)) + }, + ...mutationOptions, + }) +} + +type GetFileIndexingEstimateParamsOptionWeb = GetFileIndexingEstimateParamsOptionBase & { + dataSourceType: DataSourceType.WEB + websitePages: CrawlResultItem[] + crawlOptions?: CrawlOptions + websiteCrawlProvider: DataSourceProvider + websiteCrawlJobId: string +} + const getFileIndexingEstimateParamsForWeb = ({ docForm, docLanguage, @@ -151,30 +175,6 @@ const getFileIndexingEstimateParamsForWeb = ({ } } -export const useFetchFileIndexingEstimateForFile = ( - options: GetFileIndexingEstimateParamsOptionFile, - mutationOptions: MutationOptions = {}, -) => { - return useMutation({ - mutationFn: async () => { - return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForFile(options)) - }, - ...mutationOptions, - }) -} - -export const useFetchFileIndexingEstimateForNotion = ( - options: GetFileIndexingEstimateParamsOptionNotion, - mutationOptions: MutationOptions = {}, -) => { - return useMutation({ - mutationFn: async () => { - return fetchFileIndexingEstimate(getFileIndexingEstimateParamsForNotion(options)) - }, - ...mutationOptions, - }) -} - export const useFetchFileIndexingEstimateForWeb = ( options: GetFileIndexingEstimateParamsOptionWeb, mutationOptions: MutationOptions = {}, @@ -186,3 +186,38 @@ export const useFetchFileIndexingEstimateForWeb = ( ...mutationOptions, }) } + +export const useCreateFirstDocument = ( + mutationOptions: MutationOptions = {}, +) => { + return useMutation({ + mutationFn: async (createDocumentReq: CreateDocumentReq, + ) => { + return createFirstDocument({ body: createDocumentReq }) + }, + ...mutationOptions, + }) +} + +export const useCreateDocument = ( + datasetId: string, + mutationOptions: MutationOptions = {}, +) => { + return useMutation({ + mutationFn: async (req: CreateDocumentReq) => { + return createDocument({ datasetId, body: req }) + }, + ...mutationOptions, + }) +} + +export const useFetchDefaultProcessRule = ( + mutationOptions: MutationOptions = {}, +) => { + return useMutation({ + mutationFn: async (url: string) => { + return fetchDefaultProcessRule({ url }) + }, + ...mutationOptions, + }) +}