feat(knowledge-fs): add KnowledgeFS upgrade functionality

packages/contracts/generated/api/console/datasets/orpc.gen.ts

@@ -66,6 +66,8 @@ import {
   zGetDatasetsByDatasetIdErrorDocsResponse,
   zGetDatasetsByDatasetIdIndexingStatusPath,
   zGetDatasetsByDatasetIdIndexingStatusResponse,
+  zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath,
+  zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse,
   zGetDatasetsByDatasetIdMetadataPath,
   zGetDatasetsByDatasetIdMetadataResponse,
   zGetDatasetsByDatasetIdNotionSyncPath,
@@ -162,6 +164,10 @@ import {
   zPostDatasetsByDatasetIdHitTestingBody,
   zPostDatasetsByDatasetIdHitTestingPath,
   zPostDatasetsByDatasetIdHitTestingResponse,
+  zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath,
+  zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse,
+  zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath,
+  zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse,
   zPostDatasetsByDatasetIdMetadataBody,
   zPostDatasetsByDatasetIdMetadataBuiltInByActionPath,
   zPostDatasetsByDatasetIdMetadataBuiltInByActionResponse,
@@ -1424,7 +1430,52 @@ export const indexingStatus3 = {
   get: get27,
 }
 
+export const get28 = oc
+  .route({
+    inputStructure: 'detailed',
+    method: 'GET',
+    operationId: 'getDatasetsByDatasetIdKnowledgeFsUpgradesByJobId',
+    path: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}',
+    tags: ['console'],
+  })
+  .input(z.object({ params: zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath }))
+  .output(zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse)
+
 export const post19 = oc
+  .route({
+    inputStructure: 'detailed',
+    method: 'POST',
+    operationId: 'postDatasetsByDatasetIdKnowledgeFsUpgradesByJobId',
+    path: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}',
+    successStatus: 202,
+    tags: ['console'],
+  })
+  .input(z.object({ params: zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath }))
+  .output(zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse)
+
+export const byJobId2 = {
+  get: get28,
+  post: post19,
+}
+
+export const post20 = oc
+  .route({
+    inputStructure: 'detailed',
+    method: 'POST',
+    operationId: 'postDatasetsByDatasetIdKnowledgeFsUpgrades',
+    path: '/datasets/{dataset_id}/knowledge-fs-upgrades',
+    successStatus: 202,
+    tags: ['console'],
+  })
+  .input(z.object({ params: zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath }))
+  .output(zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse)
+
+export const knowledgeFsUpgrades = {
+  post: post20,
+  byJobId: byJobId2,
+}
+
+export const post21 = oc
   .route({
     inputStructure: 'detailed',
     method: 'POST',
@@ -1437,7 +1488,7 @@ export const post19 = oc
   .output(zPostDatasetsByDatasetIdMetadataBuiltInByActionResponse)
 
 export const byAction4 = {
-  post: post19,
+  post: post21,
 }
 
 export const builtIn2 = {
@@ -1477,7 +1528,7 @@ export const byMetadataId = {
   patch: patch10,
 }
 
-export const get28 = oc
+export const get29 = oc
   .route({
     inputStructure: 'detailed',
     method: 'GET',
@@ -1488,7 +1539,7 @@ export const get28 = oc
   .input(z.object({ params: zGetDatasetsByDatasetIdMetadataPath }))
   .output(zGetDatasetsByDatasetIdMetadataResponse)
 
-export const post20 = oc
+export const post22 = oc
   .route({
     inputStructure: 'detailed',
     method: 'POST',
@@ -1506,13 +1557,13 @@ export const post20 = oc
   .output(zPostDatasetsByDatasetIdMetadataResponse)
 
 export const metadata4 = {
-  get: get28,
-  post: post20,
+  get: get29,
+  post: post22,
   builtIn: builtIn2,
   byMetadataId,
 }
 
-export const get29 = oc
+export const get30 = oc
   .route({
     inputStructure: 'detailed',
     method: 'GET',
@@ -1524,7 +1575,7 @@ export const get29 = oc
   .output(zGetDatasetsByDatasetIdNotionSyncResponse)
 
 export const sync2 = {
-  get: get29,
+  get: get30,
 }
 
 export const notion2 = {
@@ -1534,7 +1585,7 @@ export const notion2 = {
 /**
  * Get dataset permission user list
  */
-export const get30 = oc
+export const get31 = oc
   .route({
     description: 'Get dataset permission user list',
     inputStructure: 'detailed',
@@ -1547,13 +1598,13 @@ export const get30 = oc
   .output(zGetDatasetsByDatasetIdPermissionPartUsersResponse)
 
 export const permissionPartUsers = {
-  get: get30,
+  get: get31,
 }
 
 /**
  * Get dataset query history
  */
-export const get31 = oc
+export const get32 = oc
   .route({
     description: 'Get dataset query history',
     inputStructure: 'detailed',
@@ -1566,13 +1617,13 @@ export const get31 = oc
   .output(zGetDatasetsByDatasetIdQueriesResponse)
 
 export const queries = {
-  get: get31,
+  get: get32,
 }
 
 /**
  * Get applications related to dataset
  */
-export const get32 = oc
+export const get33 = oc
   .route({
     description: 'Get applications related to dataset',
     inputStructure: 'detailed',
@@ -1585,13 +1636,13 @@ export const get32 = oc
   .output(zGetDatasetsByDatasetIdRelatedAppsResponse)
 
 export const relatedApps = {
-  get: get32,
+  get: get33,
 }
 
 /**
  * retry document
  */
-export const post21 = oc
+export const post23 = oc
   .route({
     inputStructure: 'detailed',
     method: 'POST',
@@ -1610,13 +1661,13 @@ export const post21 = oc
   .output(zPostDatasetsByDatasetIdRetryResponse)
 
 export const retry = {
-  post: post21,
+  post: post23,
 }
 
 /**
  * Check if dataset is in use
  */
-export const get33 = oc
+export const get34 = oc
   .route({
     description: 'Check if dataset is in use',
     inputStructure: 'detailed',
@@ -1629,7 +1680,7 @@ export const get33 = oc
   .output(zGetDatasetsByDatasetIdUseCheckResponse)
 
 export const useCheck2 = {
-  get: get33,
+  get: get34,
 }
 
 export const delete9 = oc
@@ -1647,7 +1698,7 @@ export const delete9 = oc
 /**
  * Get dataset details
  */
-export const get34 = oc
+export const get35 = oc
   .route({
     description: 'Get dataset details',
     inputStructure: 'detailed',
@@ -1676,7 +1727,7 @@ export const patch11 = oc
 
 export const byDatasetId = {
   delete: delete9,
-  get: get34,
+  get: get35,
   patch: patch11,
   apiKeys: apiKeys2,
   autoDisableLogs,
@@ -1686,6 +1737,7 @@ export const byDatasetId = {
   externalHitTesting,
   hitTesting,
   indexingStatus: indexingStatus3,
+  knowledgeFsUpgrades,
   metadata: metadata4,
   notion: notion2,
   permissionPartUsers,
@@ -1723,7 +1775,7 @@ export const byApiKeyId2 = {
  *
  * Get all API keys for a dataset
  */
-export const get35 = oc
+export const get36 = oc
   .route({
     description: 'Get all API keys for a dataset',
     inputStructure: 'detailed',
@@ -1741,7 +1793,7 @@ export const get35 = oc
  *
  * Create a new API key for a dataset
  */
-export const post22 = oc
+export const post24 = oc
   .route({
     description: 'Create a new API key for a dataset',
     inputStructure: 'detailed',
@@ -1756,8 +1808,8 @@ export const post22 = oc
   .output(zPostDatasetsByResourceIdApiKeysResponse)
 
 export const apiKeys3 = {
-  get: get35,
-  post: post22,
+  get: get36,
+  post: post24,
   byApiKeyId: byApiKeyId2,
 }
 
@@ -1768,7 +1820,7 @@ export const byResourceId = {
 /**
  * Get list of datasets
  */
-export const get36 = oc
+export const get37 = oc
   .route({
     description: 'Get list of datasets',
     inputStructure: 'detailed',
@@ -1783,7 +1835,7 @@ export const get36 = oc
 /**
  * Create a new dataset
  */
-export const post23 = oc
+export const post25 = oc
   .route({
     description: 'Create a new dataset',
     inputStructure: 'detailed',
@@ -1797,8 +1849,8 @@ export const post23 = oc
   .output(zPostDatasetsResponse)
 
 export const datasets = {
-  get: get36,
-  post: post23,
+  get: get37,
+  post: post25,
   apiBaseInfo,
   apiKeys,
   batchImportStatus,

packages/contracts/generated/api/console/datasets/types.gen.ts

@@ -513,6 +513,27 @@ export type HitTestingResponse = {
   records: Array<HitTestingRecord>
 }
 
+export type KnowledgeFsUpgradeJobResponse = {
+  completed_at?: string | null
+  completed_documents: number
+  completed_sources: number
+  id: string
+  last_error_code?: string | null
+  last_error_message?: string | null
+  new_control_space_id?: string | null
+  old_dataset_id: string
+  snapshot_at: string
+  stage: KnowledgeFsUpgradeStage
+  status: KnowledgeFsUpgradeJobStatus
+  total_documents: number
+  total_sources: number
+}
+
+export type KnowledgeFsUpgradeRetryResponse = {
+  id: string
+  status?: 'queued'
+}
+
 export type DatasetMetadataListResponse = {
   built_in_field_enabled: boolean
   doc_metadata: Array<DatasetMetadataListItemResponse>
@@ -848,6 +869,17 @@ export type HitTestingRecord = {
   tsne_position: unknown | null
 }
 
+export type KnowledgeFsUpgradeStage =
+  | 'completed'
+  | 'creating_sources'
+  | 'finalizing'
+  | 'migrating_access'
+  | 'submitting_documents'
+  | 'validating'
+  | 'waiting_for_space'
+
+export type KnowledgeFsUpgradeJobStatus = 'failed' | 'queued' | 'running' | 'succeeded'
+
 export type DatasetMetadataListItemResponse = {
   count?: number
   id: string
@@ -2302,6 +2334,56 @@ export type GetDatasetsByDatasetIdIndexingStatusResponses = {
 export type GetDatasetsByDatasetIdIndexingStatusResponse =
   GetDatasetsByDatasetIdIndexingStatusResponses[keyof GetDatasetsByDatasetIdIndexingStatusResponses]
 
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesData = {
+  body?: never
+  path: {
+    dataset_id: string
+  }
+  query?: never
+  url: '/datasets/{dataset_id}/knowledge-fs-upgrades'
+}
+
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses = {
+  202: KnowledgeFsUpgradeJobResponse
+}
+
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesResponse =
+  PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses[keyof PostDatasetsByDatasetIdKnowledgeFsUpgradesResponses]
+
+export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdData = {
+  body?: never
+  path: {
+    dataset_id: string
+    job_id: string
+  }
+  query?: never
+  url: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}'
+}
+
+export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses = {
+  200: KnowledgeFsUpgradeJobResponse
+}
+
+export type GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse =
+  GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses[keyof GetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses]
+
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdData = {
+  body?: never
+  path: {
+    dataset_id: string
+    job_id: string
+  }
+  query?: never
+  url: '/datasets/{dataset_id}/knowledge-fs-upgrades/{job_id}'
+}
+
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses = {
+  202: KnowledgeFsUpgradeRetryResponse
+}
+
+export type PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse =
+  PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses[keyof PostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponses]
+
 export type GetDatasetsByDatasetIdMetadataData = {
   body?: never
   path: {

packages/contracts/generated/api/console/datasets/zod.gen.ts

@@ -244,6 +244,14 @@ export const zExternalHitTestingPayload = z.object({
   query: z.string(),
 })
 
+/**
+ * KnowledgeFSUpgradeRetryResponse
+ */
+export const zKnowledgeFsUpgradeRetryResponse = z.object({
+  id: z.string(),
+  status: z.literal('queued').optional().default('queued'),
+})
+
 /**
  * MetadataArgs
  */
@@ -754,6 +762,43 @@ export const zHitTestingQuery = z.object({
   content: z.string(),
 })
 
+/**
+ * KnowledgeFSUpgradeStage
+ */
+export const zKnowledgeFsUpgradeStage = z.enum([
+  'completed',
+  'creating_sources',
+  'finalizing',
+  'migrating_access',
+  'submitting_documents',
+  'validating',
+  'waiting_for_space',
+])
+
+/**
+ * KnowledgeFSUpgradeJobStatus
+ */
+export const zKnowledgeFsUpgradeJobStatus = z.enum(['failed', 'queued', 'running', 'succeeded'])
+
+/**
+ * KnowledgeFSUpgradeJobResponse
+ */
+export const zKnowledgeFsUpgradeJobResponse = z.object({
+  completed_at: z.iso.datetime().nullish(),
+  completed_documents: z.int().gte(0),
+  completed_sources: z.int().gte(0),
+  id: z.string(),
+  last_error_code: z.string().nullish(),
+  last_error_message: z.string().nullish(),
+  new_control_space_id: z.string().nullish(),
+  old_dataset_id: z.string(),
+  snapshot_at: z.iso.datetime(),
+  stage: zKnowledgeFsUpgradeStage,
+  status: zKnowledgeFsUpgradeJobStatus,
+  total_documents: z.int().gte(0),
+  total_sources: z.int().gte(0),
+})
+
 /**
  * DatasetMetadataListItemResponse
  */
@@ -2176,6 +2221,37 @@ export const zGetDatasetsByDatasetIdIndexingStatusPath = z.object({
  */
 export const zGetDatasetsByDatasetIdIndexingStatusResponse = zDocumentStatusListResponse
 
+export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesPath = z.object({
+  dataset_id: z.uuid(),
+})
+
+/**
+ * KnowledgeFS Dataset upgrade accepted
+ */
+export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesResponse = zKnowledgeFsUpgradeJobResponse
+
+export const zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath = z.object({
+  dataset_id: z.uuid(),
+  job_id: z.string(),
+})
+
+/**
+ * KnowledgeFS Dataset upgrade status
+ */
+export const zGetDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse =
+  zKnowledgeFsUpgradeJobResponse
+
+export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdPath = z.object({
+  dataset_id: z.uuid(),
+  job_id: z.string(),
+})
+
+/**
+ * KnowledgeFS Dataset upgrade retry accepted
+ */
+export const zPostDatasetsByDatasetIdKnowledgeFsUpgradesByJobIdResponse =
+  zKnowledgeFsUpgradeRetryResponse
+
 export const zGetDatasetsByDatasetIdMetadataPath = z.object({
   dataset_id: z.uuid(),
 })

web/app/components/datasets/settings/form/components/knowledge-fs-upgrade.tsx

@@ -0,0 +1,155 @@
+'use client'
+
+import type { KnowledgeFsUpgradeJobResponse } from '@dify/contracts/api/console/datasets/types.gen'
+import { Button } from '@langgenius/dify-ui/button'
+import { useMutation, useQuery } from '@tanstack/react-query'
+import { useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { consoleQuery } from '@/service/client'
+
+type Props = {
+  datasetId: string
+  disabled: boolean
+}
+
+const POLL_INTERVAL = 2_000
+
+const getErrorMessage = (error: unknown) => (error instanceof Error ? error.message : String(error))
+
+const UpgradeProgress = ({
+  datasetId,
+  initialJob,
+}: {
+  datasetId: string
+  initialJob: KnowledgeFsUpgradeJobResponse
+}) => {
+  const { t } = useTranslation()
+  const [retryError, setRetryError] = useState<string>()
+  const jobContract = consoleQuery.datasets.byDatasetId.knowledgeFsUpgrades.byJobId
+  const jobInput = {
+    params: {
+      dataset_id: datasetId,
+      job_id: initialJob.id,
+    },
+  }
+  const {
+    data: job,
+    error: statusError,
+    refetch,
+  } = useQuery({
+    ...jobContract.get.queryOptions({ input: jobInput }),
+    initialData: initialJob,
+    refetchInterval: (query) => {
+      const status = query.state.data?.status
+      return status === 'queued' || status === 'running' ? POLL_INTERVAL : false
+    },
+  })
+  const retryMutation = useMutation({
+    ...jobContract.post.mutationOptions(),
+    onMutate: () => setRetryError(undefined),
+    onSuccess: () => void refetch(),
+    onError: (error) => setRetryError(getErrorMessage(error)),
+  })
+
+  const completed = job.completed_documents + job.completed_sources
+  const total = job.total_documents + job.total_sources
+  const progress = total > 0 ? Math.round((completed / total) * 100) : 0
+  const statusLabel = {
+    queued: t(($) => $['newKnowledge.documentStatus.queued'], { ns: 'dataset' }),
+    running: t(($) => $['newKnowledge.documentStatus.processing'], { ns: 'dataset' }),
+    succeeded: t(($) => $['newKnowledge.processingTaskState.succeeded'], { ns: 'dataset' }),
+    failed: t(($) => $['newKnowledge.documentStatus.failed'], { ns: 'dataset' }),
+  }[job.status]
+  const errorMessage =
+    retryError ||
+    job.last_error_message ||
+    job.last_error_code ||
+    (statusError && getErrorMessage(statusError))
+
+  return (
+    <div className="flex flex-col gap-y-3 rounded-xl border border-components-panel-border bg-components-panel-bg p-4">
+      <div className="flex items-center justify-between gap-x-4">
+        <div className="text-sm font-medium text-text-primary">
+          {statusLabel}
+          <span className="ml-2 text-text-tertiary">{job.stage}</span>
+        </div>
+        <div className="text-xs text-text-secondary">{progress}%</div>
+      </div>
+      <div
+        className="h-1.5 overflow-hidden rounded-full bg-components-progress-bar-bg"
+        role="progressbar"
+        aria-valuemin={0}
+        aria-valuemax={100}
+        aria-valuenow={progress}
+      >
+        <div
+          className="h-full rounded-full bg-components-progress-bar-progress-solid"
+          style={{ width: `${progress}%` }}
+        />
+      </div>
+      <div className="flex gap-x-5 text-xs text-text-secondary">
+        <span>
+          {t(($) => $['newKnowledge.documents'], { ns: 'dataset' })}: {job.completed_documents}/
+          {job.total_documents}
+        </span>
+        <span>
+          {t(($) => $['newKnowledge.sources'], { ns: 'dataset' })}: {job.completed_sources}/
+          {job.total_sources}
+        </span>
+      </div>
+      {errorMessage && (
+        <div className="text-xs break-words text-text-destructive">
+          {t(($) => $.error, { ns: 'common' })}: {errorMessage}
+        </div>
+      )}
+      {job.status === 'failed' && (
+        <div>
+          <Button
+            size="small"
+            variant="secondary"
+            loading={retryMutation.isPending}
+            onClick={() => retryMutation.mutate(jobInput)}
+          >
+            {t(($) => $.retry, { ns: 'dataset' })}
+          </Button>
+        </div>
+      )}
+    </div>
+  )
+}
+
+const KnowledgeFSUpgrade = ({ datasetId, disabled }: Props) => {
+  const { t } = useTranslation()
+  const [job, setJob] = useState<KnowledgeFsUpgradeJobResponse>()
+  const [startError, setStartError] = useState<string>()
+  const upgradeContract = consoleQuery.datasets.byDatasetId.knowledgeFsUpgrades
+  const upgradeInput = { params: { dataset_id: datasetId } }
+  const startMutation = useMutation({
+    ...upgradeContract.post.mutationOptions(),
+    onMutate: () => setStartError(undefined),
+    onSuccess: setJob,
+    onError: (error) => setStartError(getErrorMessage(error)),
+  })
+
+  if (job) return <UpgradeProgress datasetId={datasetId} initialJob={job} />
+
+  return (
+    <div className="flex flex-col items-start gap-y-2">
+      <Button
+        variant="secondary"
+        loading={startMutation.isPending}
+        disabled={disabled || startMutation.isPending}
+        onClick={() => startMutation.mutate(upgradeInput)}
+      >
+        {t(($) => $['upgradeBtn.encourageShort'], { ns: 'billing' })}
+      </Button>
+      {startError && (
+        <div className="text-xs break-words text-text-destructive">
+          {t(($) => $.error, { ns: 'common' })}: {startError}
+        </div>
+      )}
+    </div>
+  )
+}
+
+export default KnowledgeFSUpgrade

web/app/components/datasets/settings/form/index.tsx

@@ -5,6 +5,7 @@ import Divider from '@/app/components/base/divider'
 import BasicInfoSection from './components/basic-info-section'
 import ExternalKnowledgeSection from './components/external-knowledge-section'
 import IndexingSection from './components/indexing-section'
+import KnowledgeFSUpgrade from './components/knowledge-fs-upgrade'
 import { useFormState } from './hooks/use-form-state'
 
 const Form = () => {
@@ -119,6 +120,12 @@ const Form = () => {
 
       <Divider type="horizontal" className="my-1 h-px bg-divider-subtle" />
 
+      {!isExternalProvider && currentDataset && (
+        <KnowledgeFSUpgrade datasetId={currentDataset.id} disabled={readonly} />
+      )}
+
+      <Divider type="horizontal" className="my-1 h-px bg-divider-subtle" />
+
       {/* Save Button */}
       <div className="flex gap-x-1">
         <div className="flex h-7 w-45 shrink-0 items-center pt-1" />