diff --git a/packages/contracts/generated/api/console/datasets/orpc.gen.ts b/packages/contracts/generated/api/console/datasets/orpc.gen.ts index 326bf34b30d..8a39e218bc8 100644 --- a/packages/contracts/generated/api/console/datasets/orpc.gen.ts +++ b/packages/contracts/generated/api/console/datasets/orpc.gen.ts @@ -66,6 +66,8 @@ import { zGetDatasetsByDatasetIdErrorDocsResponse, zGetDatasetsByDatasetIdIndexingStatusPath, zGetDatasetsByDatasetIdIndexingStatusResponse, + zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath, + zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse, zGetDatasetsByDatasetIdMetadataPath, zGetDatasetsByDatasetIdMetadataResponse, zGetDatasetsByDatasetIdNotionSyncPath, @@ -162,6 +164,10 @@ import { zPostDatasetsByDatasetIdHitTestingBody, zPostDatasetsByDatasetIdHitTestingPath, zPostDatasetsByDatasetIdHitTestingResponse, + zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath, + zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse, + zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath, + zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse, zPostDatasetsByDatasetIdMetadataBody, zPostDatasetsByDatasetIdMetadataBuiltInByActionPath, zPostDatasetsByDatasetIdMetadataBuiltInByActionResponse, @@ -1424,7 +1430,52 @@ export const indexingStatus3 = { get: get27, } +export const get28 = oc + .route({ + inputStructure: 'detailed', + method: 'GET', + operationId: 'getDatasetsByDatasetIdKnowledgeFsUpgradesByJobId', + path: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}', + tags: ['console'], + }) + .input(z.object({ params: zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath })) + .output(zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse) + export const post19 = oc + .route({ + inputStructure: 'detailed', + method: 'POST', + operationId: 'postDatasetsByDatasetIdKnowledgeFsUpgradesByJobId', + path: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}', + successStatus: 202, + tags: ['console'], + }) + .input(z.object({ params: zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath })) + .output(zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse) + +export const byJobId2 = { + get: get28, + post: post19, +} + +export const post20 = oc + .route({ + inputStructure: 'detailed', + method: 'POST', + operationId: 'postDatasetsByDatasetIdKnowledgeFsUpgrades', + path: '/datasets/{dataset_id}/knowledge-fs-upgrades', + successStatus: 202, + tags: ['console'], + }) + .input(z.object({ params: zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath })) + .output(zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse) + +export const knowledgeFsUpgrades = { + post: post20, + byJobId: byJobId2, +} + +export const post21 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1437,7 +1488,7 @@ export const post19 = oc .output(zPostDatasetsByDatasetIdMetadataBuiltInByActionResponse) export const byAction4 = { - post: post19, + post: post21, } export const builtIn2 = { @@ -1477,7 +1528,7 @@ export const byMetadataId = { patch: patch10, } -export const get28 = oc +export const get29 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1488,7 +1539,7 @@ export const get28 = oc .input(z.object({ params: zGetDatasetsByDatasetIdMetadataPath })) .output(zGetDatasetsByDatasetIdMetadataResponse) -export const post20 = oc +export const post22 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1506,13 +1557,13 @@ export const post20 = oc .output(zPostDatasetsByDatasetIdMetadataResponse) export const metadata4 = { - get: get28, - post: post20, + get: get29, + post: post22, builtIn: builtIn2, byMetadataId, } -export const get29 = oc +export const get30 = oc .route({ inputStructure: 'detailed', method: 'GET', @@ -1524,7 +1575,7 @@ export const get29 = oc .output(zGetDatasetsByDatasetIdNotionSyncResponse) export const sync2 = { - get: get29, + get: get30, } export const notion2 = { @@ -1534,7 +1585,7 @@ export const notion2 = { /** * Get dataset permission user list */ -export const get30 = oc +export const get31 = oc .route({ description: 'Get dataset permission user list', inputStructure: 'detailed', @@ -1547,13 +1598,13 @@ export const get30 = oc .output(zGetDatasetsByDatasetIdPermissionPartUsersResponse) export const permissionPartUsers = { - get: get30, + get: get31, } /** * Get dataset query history */ -export const get31 = oc +export const get32 = oc .route({ description: 'Get dataset query history', inputStructure: 'detailed', @@ -1566,13 +1617,13 @@ export const get31 = oc .output(zGetDatasetsByDatasetIdQueriesResponse) export const queries = { - get: get31, + get: get32, } /** * Get applications related to dataset */ -export const get32 = oc +export const get33 = oc .route({ description: 'Get applications related to dataset', inputStructure: 'detailed', @@ -1585,13 +1636,13 @@ export const get32 = oc .output(zGetDatasetsByDatasetIdRelatedAppsResponse) export const relatedApps = { - get: get32, + get: get33, } /** * retry document */ -export const post21 = oc +export const post23 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1610,13 +1661,13 @@ export const post21 = oc .output(zPostDatasetsByDatasetIdRetryResponse) export const retry = { - post: post21, + post: post23, } /** * Check if dataset is in use */ -export const get33 = oc +export const get34 = oc .route({ description: 'Check if dataset is in use', inputStructure: 'detailed', @@ -1629,7 +1680,7 @@ export const get33 = oc .output(zGetDatasetsByDatasetIdUseCheckResponse) export const useCheck2 = { - get: get33, + get: get34, } export const delete9 = oc @@ -1647,7 +1698,7 @@ export const delete9 = oc /** * Get dataset details */ -export const get34 = oc +export const get35 = oc .route({ description: 'Get dataset details', inputStructure: 'detailed', @@ -1676,7 +1727,7 @@ export const patch11 = oc export const byDatasetId = { delete: delete9, - get: get34, + get: get35, patch: patch11, apiKeys: apiKeys2, autoDisableLogs, @@ -1686,6 +1737,7 @@ export const byDatasetId = { externalHitTesting, hitTesting, indexingStatus: indexingStatus3, + knowledgeFsUpgrades, metadata: metadata4, notion: notion2, permissionPartUsers, @@ -1723,7 +1775,7 @@ export const byApiKeyId2 = { * * Get all API keys for a dataset */ -export const get35 = oc +export const get36 = oc .route({ description: 'Get all API keys for a dataset', inputStructure: 'detailed', @@ -1741,7 +1793,7 @@ export const get35 = oc * * Create a new API key for a dataset */ -export const post22 = oc +export const post24 = oc .route({ description: 'Create a new API key for a dataset', inputStructure: 'detailed', @@ -1756,8 +1808,8 @@ export const post22 = oc .output(zPostDatasetsByResourceIdApiKeysResponse) export const apiKeys3 = { - get: get35, - post: post22, + get: get36, + post: post24, byApiKeyId: byApiKeyId2, } @@ -1768,7 +1820,7 @@ export const byResourceId = { /** * Get list of datasets */ -export const get36 = oc +export const get37 = oc .route({ description: 'Get list of datasets', inputStructure: 'detailed', @@ -1783,7 +1835,7 @@ export const get36 = oc /** * Create a new dataset */ -export const post23 = oc +export const post25 = oc .route({ description: 'Create a new dataset', inputStructure: 'detailed', @@ -1797,8 +1849,8 @@ export const post23 = oc .output(zPostDatasetsResponse) export const datasets = { - get: get36, - post: post23, + get: get37, + post: post25, apiBaseInfo, apiKeys, batchImportStatus, diff --git a/packages/contracts/generated/api/console/datasets/types.gen.ts b/packages/contracts/generated/api/console/datasets/types.gen.ts index 74a1425d39a..164fef65d1d 100644 --- a/packages/contracts/generated/api/console/datasets/types.gen.ts +++ b/packages/contracts/generated/api/console/datasets/types.gen.ts @@ -513,6 +513,27 @@ export type HitTestingResponse = { records: Array } +export type KnowledgeFsUpgradeJobResponse = { + completed_at?: string | null + completed_documents: number + completed_sources: number + id: string + last_error_code?: string | null + last_error_message?: string | null + new_control_space_id?: string | null + old_dataset_id: string + snapshot_at: string + stage: KnowledgeFsUpgradeStage + status: KnowledgeFsUpgradeJobStatus + total_documents: number + total_sources: number +} + +export type KnowledgeFsUpgradeRetryResponse = { + id: string + status?: 'queued' +} + export type DatasetMetadataListResponse = { built_in_field_enabled: boolean doc_metadata: Array @@ -848,6 +869,17 @@ export type HitTestingRecord = { tsne_position: unknown | null } +export type KnowledgeFsUpgradeStage = + | 'completed' + | 'creating_sources' + | 'finalizing' + | 'migrating_access' + | 'submitting_documents' + | 'validating' + | 'waiting_for_space' + +export type KnowledgeFsUpgradeJobStatus = 'failed' | 'queued' | 'running' | 'succeeded' + export type DatasetMetadataListItemResponse = { count?: number id: string @@ -2302,6 +2334,56 @@ export type GetDatasetsByDatasetIdIndexingStatusResponses = { export type GetDatasetsByDatasetIdIndexingStatusResponse = GetDatasetsByDatasetIdIndexingStatusResponses[keyof GetDatasetsByDatasetIdIndexingStatusResponses] +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesData = { + body?: never + path: { + dataset_id: string + } + query?: never + url: '/datasets/{dataset_id}/knowledge-fs-upgrades' +} + +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses = { + 202: KnowledgeFsUpgradeJobResponse +} + +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesResponse = + PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses[keyof PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses] + +export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdData = { + body?: never + path: { + dataset_id: string + job_id: string + } + query?: never + url: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}' +} + +export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses = { + 200: KnowledgeFsUpgradeJobResponse +} + +export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse = + GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses[keyof GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses] + +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdData = { + body?: never + path: { + dataset_id: string + job_id: string + } + query?: never + url: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}' +} + +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses = { + 202: KnowledgeFsUpgradeRetryResponse +} + +export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse = + PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses[keyof PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses] + export type GetDatasetsByDatasetIdMetadataData = { body?: never path: { diff --git a/packages/contracts/generated/api/console/datasets/zod.gen.ts b/packages/contracts/generated/api/console/datasets/zod.gen.ts index 183a20a8480..78d31fcd888 100644 --- a/packages/contracts/generated/api/console/datasets/zod.gen.ts +++ b/packages/contracts/generated/api/console/datasets/zod.gen.ts @@ -244,6 +244,14 @@ export const zExternalHitTestingPayload = z.object({ query: z.string(), }) +/** + * KnowledgeFSUpgradeRetryResponse + */ +export const zKnowledgeFsUpgradeRetryResponse = z.object({ + id: z.string(), + status: z.literal('queued').optional().default('queued'), +}) + /** * MetadataArgs */ @@ -754,6 +762,43 @@ export const zHitTestingQuery = z.object({ content: z.string(), }) +/** + * KnowledgeFSUpgradeStage + */ +export const zKnowledgeFsUpgradeStage = z.enum([ + 'completed', + 'creating_sources', + 'finalizing', + 'migrating_access', + 'submitting_documents', + 'validating', + 'waiting_for_space', +]) + +/** + * KnowledgeFSUpgradeJobStatus + */ +export const zKnowledgeFsUpgradeJobStatus = z.enum(['failed', 'queued', 'running', 'succeeded']) + +/** + * KnowledgeFSUpgradeJobResponse + */ +export const zKnowledgeFsUpgradeJobResponse = z.object({ + completed_at: z.iso.datetime().nullish(), + completed_documents: z.int().gte(0), + completed_sources: z.int().gte(0), + id: z.string(), + last_error_code: z.string().nullish(), + last_error_message: z.string().nullish(), + new_control_space_id: z.string().nullish(), + old_dataset_id: z.string(), + snapshot_at: z.iso.datetime(), + stage: zKnowledgeFsUpgradeStage, + status: zKnowledgeFsUpgradeJobStatus, + total_documents: z.int().gte(0), + total_sources: z.int().gte(0), +}) + /** * DatasetMetadataListItemResponse */ @@ -2176,6 +2221,37 @@ export const zGetDatasetsByDatasetIdIndexingStatusPath = z.object({ */ export const zGetDatasetsByDatasetIdIndexingStatusResponse = zDocumentStatusListResponse +export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath = z.object({ + dataset_id: z.uuid(), +}) + +/** + * KnowledgeFS Dataset upgrade accepted + */ +export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse = zKnowledgeFsUpgradeJobResponse + +export const zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath = z.object({ + dataset_id: z.uuid(), + job_id: z.string(), +}) + +/** + * KnowledgeFS Dataset upgrade status + */ +export const zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse = + zKnowledgeFsUpgradeJobResponse + +export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath = z.object({ + dataset_id: z.uuid(), + job_id: z.string(), +}) + +/** + * KnowledgeFS Dataset upgrade retry accepted + */ +export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse = + zKnowledgeFsUpgradeRetryResponse + export const zGetDatasetsByDatasetIdMetadataPath = z.object({ dataset_id: z.uuid(), }) diff --git a/web/app/components/datasets/settings/form/components/knowledge-fs-upgrade.tsx b/web/app/components/datasets/settings/form/components/knowledge-fs-upgrade.tsx new file mode 100644 index 00000000000..7e82ad685ef --- /dev/null +++ b/web/app/components/datasets/settings/form/components/knowledge-fs-upgrade.tsx @@ -0,0 +1,155 @@ +'use client' + +import type { KnowledgeFsUpgradeJobResponse } from '@dify/contracts/api/console/datasets/types.gen' +import { Button } from '@langgenius/dify-ui/button' +import { useMutation, useQuery } from '@tanstack/react-query' +import { useState } from 'react' +import { useTranslation } from 'react-i18next' +import { consoleQuery } from '@/service/client' + +type Props = { + datasetId: string + disabled: boolean +} + +const POLL_INTERVAL = 2_000 + +const getErrorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error)) + +const UpgradeProgress = ({ + datasetId, + initialJob, +}: { + datasetId: string + initialJob: KnowledgeFsUpgradeJobResponse +}) => { + const { t } = useTranslation() + const [retryError, setRetryError] = useState() + const jobContract = consoleQuery.datasets.byDatasetId.knowledgeFsUpgrades.byJobId + const jobInput = { + params: { + dataset_id: datasetId, + job_id: initialJob.id, + }, + } + const { + data: job, + error: statusError, + refetch, + } = useQuery({ + ...jobContract.get.queryOptions({ input: jobInput }), + initialData: initialJob, + refetchInterval: (query) => { + const status = query.state.data?.status + return status === 'queued' || status === 'running' ? POLL_INTERVAL : false + }, + }) + const retryMutation = useMutation({ + ...jobContract.post.mutationOptions(), + onMutate: () => setRetryError(undefined), + onSuccess: () => void refetch(), + onError: (error) => setRetryError(getErrorMessage(error)), + }) + + const completed = job.completed_documents + job.completed_sources + const total = job.total_documents + job.total_sources + const progress = total > 0 ? Math.round((completed / total) * 100) : 0 + const statusLabel = { + queued: t(($) => $['newKnowledge.documentStatus.queued'], { ns: 'dataset' }), + running: t(($) => $['newKnowledge.documentStatus.processing'], { ns: 'dataset' }), + succeeded: t(($) => $['newKnowledge.processingTaskState.succeeded'], { ns: 'dataset' }), + failed: t(($) => $['newKnowledge.documentStatus.failed'], { ns: 'dataset' }), + }[job.status] + const errorMessage = + retryError || + job.last_error_message || + job.last_error_code || + (statusError && getErrorMessage(statusError)) + + return ( +
+
+
+ {statusLabel} + {job.stage} +
+
{progress}%
+
+
+
+
+
+ + {t(($) => $['newKnowledge.documents'], { ns: 'dataset' })}: {job.completed_documents}/ + {job.total_documents} + + + {t(($) => $['newKnowledge.sources'], { ns: 'dataset' })}: {job.completed_sources}/ + {job.total_sources} + +
+ {errorMessage && ( +
+ {t(($) => $.error, { ns: 'common' })}: {errorMessage} +
+ )} + {job.status === 'failed' && ( +
+ +
+ )} +
+ ) +} + +const KnowledgeFSUpgrade = ({ datasetId, disabled }: Props) => { + const { t } = useTranslation() + const [job, setJob] = useState() + const [startError, setStartError] = useState() + const upgradeContract = consoleQuery.datasets.byDatasetId.knowledgeFsUpgrades + const upgradeInput = { params: { dataset_id: datasetId } } + const startMutation = useMutation({ + ...upgradeContract.post.mutationOptions(), + onMutate: () => setStartError(undefined), + onSuccess: setJob, + onError: (error) => setStartError(getErrorMessage(error)), + }) + + if (job) return + + return ( +
+ + {startError && ( +
+ {t(($) => $.error, { ns: 'common' })}: {startError} +
+ )} +
+ ) +} + +export default KnowledgeFSUpgrade diff --git a/web/app/components/datasets/settings/form/index.tsx b/web/app/components/datasets/settings/form/index.tsx index 82410d540d2..09a3c949466 100644 --- a/web/app/components/datasets/settings/form/index.tsx +++ b/web/app/components/datasets/settings/form/index.tsx @@ -5,6 +5,7 @@ import Divider from '@/app/components/base/divider' import BasicInfoSection from './components/basic-info-section' import ExternalKnowledgeSection from './components/external-knowledge-section' import IndexingSection from './components/indexing-section' +import KnowledgeFSUpgrade from './components/knowledge-fs-upgrade' import { useFormState } from './hooks/use-form-state' const Form = () => { @@ -119,6 +120,12 @@ const Form = () => { + {!isExternalProvider && currentDataset && ( + + )} + + + {/* Save Button */}