From f8438704a67162a2456aeeabd2212be6476d6de9 Mon Sep 17 00:00:00 2001 From: yyh Date: Fri, 23 Jan 2026 15:11:04 +0800 Subject: [PATCH] refactor(app-asset): migrate file upload to presigned URL and batch upload - Replace FormData file upload with presigned URL two-step upload - Add batch-upload contract for folder uploads (reduces N+M to 1+M requests) - Remove deprecated createFile contract and useCreateAppAssetFile hook - Remove checksum field from AppAssetNode and AppAssetTreeView types - Add upload-to-presigned-url utility for direct storage uploads --- .../skill/hooks/use-create-operations.ts | 116 ++++++------- .../workflow/skill/hooks/use-file-drop.ts | 11 +- .../skill/hooks/use-inline-create-node.ts | 9 +- .../workflow/skill/utils/tree-utils.ts | 1 - web/contract/console/app-asset.ts | 36 +++-- web/contract/router.ts | 6 +- web/service/upload-to-presigned-url.ts | 44 +++++ web/service/use-app-asset.ts | 152 ++++++++++++------ web/types/app-asset.ts | 57 ++++++- 9 files changed, 286 insertions(+), 146 deletions(-) create mode 100644 web/service/upload-to-presigned-url.ts diff --git a/web/app/components/workflow/skill/hooks/use-create-operations.ts b/web/app/components/workflow/skill/hooks/use-create-operations.ts index f9d82a8bb0..c7cd9e70df 100644 --- a/web/app/components/workflow/skill/hooks/use-create-operations.ts +++ b/web/app/components/workflow/skill/hooks/use-create-operations.ts @@ -1,15 +1,15 @@ 'use client' -// Handles file/folder creation and upload operations - import type { StoreApi } from 'zustand' import type { SkillEditorSliceShape } from '@/app/components/workflow/store/workflow/skill-editor/types' +import type { BatchUploadNodeInput } from '@/types/app-asset' import { useCallback, useRef } from 'react' import { useTranslation } from 'react-i18next' import Toast from '@/app/components/base/toast' import { - useCreateAppAssetFile, + useBatchUpload, useCreateAppAssetFolder, + useUploadFileWithPresignedUrl, } from '@/service/use-app-asset' type UseCreateOperationsOptions = { @@ -34,7 +34,8 @@ export function useCreateOperations({ const folderInputRef = useRef(null) const createFolder = useCreateAppAssetFolder() - const createFile = useCreateAppAssetFile() + const uploadFile = useUploadFileWithPresignedUrl() + const batchUpload = useBatchUpload() const handleNewFile = useCallback(() => { storeApi.getState().startCreateNode('file', parentId) @@ -56,9 +57,8 @@ export function useCreateOperations({ try { await Promise.all( files.map(file => - createFile.mutateAsync({ + uploadFile.mutateAsync({ appId, - name: file.name, file, parentId, }), @@ -80,7 +80,7 @@ export function useCreateOperations({ e.target.value = '' onClose() } - }, [appId, createFile, onClose, parentId, t]) + }, [appId, uploadFile, onClose, parentId, t]) const handleFolderChange = useCallback(async (e: React.ChangeEvent) => { const files = Array.from(e.target.files || []) @@ -90,78 +90,52 @@ export function useCreateOperations({ } try { - const folders = new Set() + const fileMap = new Map() + const tree: BatchUploadNodeInput[] = [] + const folderMap = new Map() for (const file of files) { const relativePath = getRelativePath(file) - const parts = relativePath.split('/') + fileMap.set(relativePath, file) - if (parts.length > 1) { - let folderPath = '' - for (let i = 0; i < parts.length - 1; i++) { - folderPath = folderPath ? `${folderPath}/${parts[i]}` : parts[i] - folders.add(folderPath) + const parts = relativePath.split('/') + let currentLevel = tree + let currentPath = '' + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + const isLastPart = i === parts.length - 1 + currentPath = currentPath ? `${currentPath}/${part}` : part + + if (isLastPart) { + currentLevel.push({ + name: part, + node_type: 'file', + size: file.size, + }) + } + else { + let folder = folderMap.get(currentPath) + if (!folder) { + folder = { + name: part, + node_type: 'folder', + children: [], + } + folderMap.set(currentPath, folder) + currentLevel.push(folder) + } + currentLevel = folder.children! } } } - const sortedFolders = Array.from(folders).sort((a, b) => { - return a.split('/').length - b.split('/').length + await batchUpload.mutateAsync({ + appId, + tree, + files: fileMap, }) - const folderIdMap = new Map() - folderIdMap.set('', parentId) - - const foldersByDepth = new Map() - for (const folderPath of sortedFolders) { - const depth = folderPath.split('/').length - const group = foldersByDepth.get(depth) - if (group) - group.push(folderPath) - else - foldersByDepth.set(depth, [folderPath]) - } - - for (const [, foldersAtDepth] of foldersByDepth) { - const createdFolders = await Promise.all( - foldersAtDepth.map(async (folderPath) => { - const parts = folderPath.split('/') - const folderName = parts[parts.length - 1] - const parentPath = parts.slice(0, -1).join('/') - const parentFolderId = folderIdMap.get(parentPath) ?? parentId - - const result = await createFolder.mutateAsync({ - appId, - payload: { - name: folderName, - parent_id: parentFolderId, - }, - }) - - return { folderPath, id: result.id } - }), - ) - - for (const { folderPath, id } of createdFolders) - folderIdMap.set(folderPath, id) - } - - await Promise.all( - files.map((file) => { - const relativePath = getRelativePath(file) - const parts = relativePath.split('/') - const parentPath = parts.length > 1 ? parts.slice(0, -1).join('/') : '' - const targetParentId = folderIdMap.get(parentPath) ?? parentId - - return createFile.mutateAsync({ - appId, - name: file.name, - file, - parentId: targetParentId, - }) - }), - ) - Toast.notify({ type: 'success', message: t('skillSidebar.menu.folderUploaded'), @@ -177,12 +151,12 @@ export function useCreateOperations({ e.target.value = '' onClose() } - }, [appId, createFile, createFolder, onClose, parentId, t]) + }, [appId, batchUpload, onClose, t]) return { fileInputRef, folderInputRef, - isCreating: createFile.isPending || createFolder.isPending, + isCreating: uploadFile.isPending || createFolder.isPending || batchUpload.isPending, handleNewFile, handleNewFolder, handleFileChange, diff --git a/web/app/components/workflow/skill/hooks/use-file-drop.ts b/web/app/components/workflow/skill/hooks/use-file-drop.ts index 96927c64c4..10e8e133f7 100644 --- a/web/app/components/workflow/skill/hooks/use-file-drop.ts +++ b/web/app/components/workflow/skill/hooks/use-file-drop.ts @@ -8,7 +8,7 @@ import { useTranslation } from 'react-i18next' import { useStore as useAppStore } from '@/app/components/app/store' import Toast from '@/app/components/base/toast' import { useWorkflowStore } from '@/app/components/workflow/store' -import { useCreateAppAssetFile } from '@/service/use-app-asset' +import { useUploadFileWithPresignedUrl } from '@/service/use-app-asset' import { ROOT_ID } from '../constants' type FileDropTarget = { @@ -21,7 +21,7 @@ export function useFileDrop() { const appDetail = useAppStore(s => s.appDetail) const appId = appDetail?.id || '' const storeApi = useWorkflowStore() - const createFile = useCreateAppAssetFile() + const uploadFile = useUploadFileWithPresignedUrl() const handleDragOver = useCallback((e: React.DragEvent, target: FileDropTarget) => { e.preventDefault() @@ -80,9 +80,8 @@ export function useFileDrop() { try { await Promise.all( files.map(file => - createFile.mutateAsync({ + uploadFile.mutateAsync({ appId, - name: file.name, file, parentId: targetFolderId, }), @@ -100,12 +99,12 @@ export function useFileDrop() { message: t('skillSidebar.menu.uploadError'), }) } - }, [appId, createFile, t, storeApi]) + }, [appId, uploadFile, t, storeApi]) return { handleDragOver, handleDragLeave, handleDrop, - isUploading: createFile.isPending, + isUploading: uploadFile.isPending, } } diff --git a/web/app/components/workflow/skill/hooks/use-inline-create-node.ts b/web/app/components/workflow/skill/hooks/use-inline-create-node.ts index 5037d31b1b..ec63c6d31d 100644 --- a/web/app/components/workflow/skill/hooks/use-inline-create-node.ts +++ b/web/app/components/workflow/skill/hooks/use-inline-create-node.ts @@ -8,9 +8,9 @@ import { useStore as useAppStore } from '@/app/components/app/store' import Toast from '@/app/components/base/toast' import { useStore, useWorkflowStore } from '@/app/components/workflow/store' import { - useCreateAppAssetFile, useCreateAppAssetFolder, useRenameAppAssetNode, + useUploadFileWithPresignedUrl, } from '@/service/use-app-asset' import { getFileExtension, isTextLikeFile } from '../utils/file-utils' import { createDraftTreeNode, insertDraftTreeNode } from '../utils/tree-utils' @@ -35,7 +35,7 @@ export function useInlineCreateNode({ const pendingCreateNode = useStore(s => s.pendingCreateNode) const storeApi = useWorkflowStore() - const createFile = useCreateAppAssetFile() + const uploadFile = useUploadFileWithPresignedUrl() const createFolder = useCreateAppAssetFolder() const renameNode = useRenameAppAssetNode() @@ -79,9 +79,8 @@ export function useInlineCreateNode({ else { const emptyBlob = new Blob([''], { type: 'text/plain' }) const file = new File([emptyBlob], trimmedName) - const createdFile = await createFile.mutateAsync({ + const createdFile = await uploadFile.mutateAsync({ appId, - name: trimmedName, file, parentId: pendingCreateParentId, }) @@ -123,7 +122,7 @@ export function useInlineCreateNode({ }) }, [ appId, - createFile, + uploadFile, createFolder, pendingCreateId, pendingCreateParentId, diff --git a/web/app/components/workflow/skill/utils/tree-utils.ts b/web/app/components/workflow/skill/utils/tree-utils.ts index 9836a4632a..59dd02425e 100644 --- a/web/app/components/workflow/skill/utils/tree-utils.ts +++ b/web/app/components/workflow/skill/utils/tree-utils.ts @@ -161,7 +161,6 @@ export function createDraftTreeNode(options: DraftTreeNodeOptions): AppAssetTree path: '', extension: '', size: 0, - checksum: '', children: [], } } diff --git a/web/contract/console/app-asset.ts b/web/contract/console/app-asset.ts index e63eeada8d..24779183ec 100644 --- a/web/contract/console/app-asset.ts +++ b/web/contract/console/app-asset.ts @@ -4,7 +4,11 @@ import type { AppAssetNode, AppAssetPublishResponse, AppAssetTreeResponse, + BatchUploadPayload, + BatchUploadResponse, CreateFolderPayload, + FileUploadUrlResponse, + GetFileUploadUrlPayload, MoveNodePayload, RenameNodePayload, ReorderNodePayload, @@ -33,16 +37,6 @@ export const createFolderContract = base }>()) .output(type()) -export const createFileContract = base - .route({ - path: '/apps/{appId}/assets/files', - method: 'POST', - }) - .input(type<{ - params: { appId: string } - }>()) - .output(type()) - export const getFileContentContract = base .route({ path: '/apps/{appId}/assets/files/{nodeId}', @@ -126,3 +120,25 @@ export const publishContract = base params: { appId: string } }>()) .output(type()) + +export const getFileUploadUrlContract = base + .route({ + path: '/apps/{appId}/assets/files/upload', + method: 'POST', + }) + .input(type<{ + params: { appId: string } + body: GetFileUploadUrlPayload + }>()) + .output(type()) + +export const batchUploadContract = base + .route({ + path: '/apps/{appId}/assets/batch-upload', + method: 'POST', + }) + .input(type<{ + params: { appId: string } + body: BatchUploadPayload + }>()) + .output(type()) diff --git a/web/contract/router.ts b/web/contract/router.ts index bd9bca235c..aad6bd1b0a 100644 --- a/web/contract/router.ts +++ b/web/contract/router.ts @@ -1,10 +1,11 @@ import type { InferContractRouterInputs } from '@orpc/contract' import { - createFileContract, + batchUploadContract, createFolderContract, deleteNodeContract, getFileContentContract, getFileDownloadUrlContract, + getFileUploadUrlContract, moveNodeContract, publishContract, renameNodeContract, @@ -52,7 +53,6 @@ export const consoleRouterContract = { appAsset: { tree: treeContract, createFolder: createFolderContract, - createFile: createFileContract, getFileContent: getFileContentContract, getFileDownloadUrl: getFileDownloadUrlContract, updateFileContent: updateFileContentContract, @@ -61,6 +61,8 @@ export const consoleRouterContract = { moveNode: moveNodeContract, reorderNode: reorderNodeContract, publish: publishContract, + getFileUploadUrl: getFileUploadUrlContract, + batchUpload: batchUploadContract, }, } diff --git a/web/service/upload-to-presigned-url.ts b/web/service/upload-to-presigned-url.ts new file mode 100644 index 0000000000..71433c76c8 --- /dev/null +++ b/web/service/upload-to-presigned-url.ts @@ -0,0 +1,44 @@ +type UploadToPresignedUrlOptions = { + file: File + uploadUrl: string + onProgress?: (progress: number) => void + signal?: AbortSignal +} + +export async function uploadToPresignedUrl({ + file, + uploadUrl, + onProgress, + signal, +}: UploadToPresignedUrlOptions): Promise { + return new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest() + + if (signal) { + signal.addEventListener('abort', () => { + xhr.abort() + reject(new DOMException('Upload aborted', 'AbortError')) + }) + } + + xhr.open('PUT', uploadUrl) + xhr.setRequestHeader('Content-Type', file.type || 'application/octet-stream') + + xhr.upload.onprogress = (e) => { + if (e.lengthComputable && onProgress) + onProgress(Math.round((e.loaded / e.total) * 100)) + } + + xhr.onload = () => { + if (xhr.status >= 200 && xhr.status < 300) + resolve() + else + reject(new Error(`Upload failed with status ${xhr.status}`)) + } + + xhr.onerror = () => reject(new Error('Upload network error')) + xhr.ontimeout = () => reject(new Error('Upload timeout')) + + xhr.send(file) + }) +} diff --git a/web/service/use-app-asset.ts b/web/service/use-app-asset.ts index 1de386e790..9de6453632 100644 --- a/web/service/use-app-asset.ts +++ b/web/service/use-app-asset.ts @@ -1,7 +1,10 @@ import type { AppAssetNode, AppAssetTreeResponse, + BatchUploadNodeInput, + BatchUploadNodeOutput, CreateFolderPayload, + GetFileUploadUrlPayload, MoveNodePayload, RenameNodePayload, ReorderNodePayload, @@ -14,6 +17,7 @@ import { } from '@tanstack/react-query' import { consoleClient, consoleQuery } from '@/service/client' import { upload } from './base' +import { uploadToPresignedUrl } from './upload-to-presigned-url' type UseGetAppAssetTreeOptions = { select?: (data: AppAssetTreeResponse) => TData @@ -49,53 +53,6 @@ export const useCreateAppAssetFolder = () => { }) } -export const useCreateAppAssetFile = () => { - const queryClient = useQueryClient() - return useMutation({ - mutationKey: consoleQuery.appAsset.createFile.mutationKey(), - mutationFn: async ({ - appId, - name, - file, - parentId, - onProgress, - }: { - appId: string - name: string - file: File - parentId?: string | null - onProgress?: (progress: number) => void - }): Promise => { - const formData = new FormData() - formData.append('name', name) - formData.append('file', file) - if (parentId) - formData.append('parent_id', parentId) - - const xhr = new XMLHttpRequest() - return upload( - { - xhr, - data: formData, - onprogress: onProgress - ? (e) => { - if (e.lengthComputable) - onProgress(Math.round((e.loaded / e.total) * 100)) - } - : undefined, - }, - false, - `/apps/${appId}/assets/files`, - ) as Promise - }, - onSuccess: (_, variables) => { - queryClient.invalidateQueries({ - queryKey: consoleQuery.appAsset.tree.queryKey({ input: { params: { appId: variables.appId } } }), - }) - }, - }) -} - export const useGetAppAssetFileContent = (appId: string, nodeId: string, options?: { enabled?: boolean }) => { return useQuery({ queryKey: consoleQuery.appAsset.getFileContent.queryKey({ input: { params: { appId, nodeId } } }), @@ -310,3 +267,104 @@ export const usePublishAppAssets = () => { }, }) } + +export const useUploadFileWithPresignedUrl = () => { + const queryClient = useQueryClient() + return useMutation({ + mutationKey: consoleQuery.appAsset.getFileUploadUrl.mutationKey(), + mutationFn: async ({ + appId, + file, + parentId, + onProgress, + }: { + appId: string + file: File + parentId?: string | null + onProgress?: (progress: number) => void + }): Promise => { + const payload: GetFileUploadUrlPayload = { + name: file.name, + size: file.size, + parent_id: parentId, + } + + const { node, upload_url } = await consoleClient.appAsset.getFileUploadUrl({ + params: { appId }, + body: payload, + }) + + await uploadToPresignedUrl({ + file, + uploadUrl: upload_url, + onProgress, + }) + + return node + }, + onSuccess: (_, variables) => { + queryClient.invalidateQueries({ + queryKey: consoleQuery.appAsset.tree.queryKey({ input: { params: { appId: variables.appId } } }), + }) + }, + }) +} + +export const useBatchUpload = () => { + const queryClient = useQueryClient() + return useMutation({ + mutationKey: consoleQuery.appAsset.batchUpload.mutationKey(), + mutationFn: async ({ + appId, + tree, + files, + onProgress, + }: { + appId: string + tree: BatchUploadNodeInput[] + files: Map + onProgress?: (uploaded: number, total: number) => void + }): Promise => { + const response = await consoleClient.appAsset.batchUpload({ + params: { appId }, + body: { children: tree }, + }) + + const uploadTasks: Array<{ path: string, file: File, url: string }> = [] + + const extractUploads = (nodes: BatchUploadNodeOutput[], pathPrefix: string = '') => { + for (const node of nodes) { + const currentPath = pathPrefix ? `${pathPrefix}/${node.name}` : node.name + if (node.upload_url) { + const file = files.get(currentPath) + if (file) + uploadTasks.push({ path: currentPath, file, url: node.upload_url }) + } + if (node.children && node.children.length > 0) + extractUploads(node.children, currentPath) + } + } + + extractUploads(response.children) + + let completed = 0 + const total = uploadTasks.length + + await Promise.all( + uploadTasks.map(async (task) => { + await uploadToPresignedUrl({ + file: task.file, + uploadUrl: task.url, + }) + completed++ + onProgress?.(completed, total) + }), + ) + }, + onSuccess: (_, variables) => { + queryClient.invalidateQueries({ + queryKey: consoleQuery.appAsset.tree.queryKey({ input: { params: { appId: variables.appId } } }), + }) + }, + }) +} diff --git a/web/types/app-asset.ts b/web/types/app-asset.ts index 8fcb9aad12..01def5c400 100644 --- a/web/types/app-asset.ts +++ b/web/types/app-asset.ts @@ -29,8 +29,6 @@ export type AppAssetNode = { extension: string /** File size in bytes, 0 for folders */ size: number - /** SHA-256 checksum of file content, empty for folders */ - checksum: string } /** @@ -50,8 +48,6 @@ export type AppAssetTreeView = { extension: string /** File size in bytes */ size: number - /** SHA-256 checksum */ - checksum: string /** Child nodes (for folders) */ children: AppAssetTreeView[] } @@ -138,3 +134,56 @@ export type ReorderNodePayload = { /** Place after this node ID, null for first position */ after_node_id: string | null } + +/** + * Request payload for getting file upload URL + */ +export type GetFileUploadUrlPayload = { + name: string + size: number + parent_id?: string | null +} + +/** + * Response for file upload URL request + */ +export type FileUploadUrlResponse = { + node: AppAssetNode + upload_url: string +} + +/** + * Input node structure for batch upload + */ +export type BatchUploadNodeInput = { + name: string + node_type: AssetNodeType + size?: number + children?: BatchUploadNodeInput[] +} + +/** + * Output node structure from batch upload (with IDs and upload URLs) + */ +export type BatchUploadNodeOutput = { + id: string + name: string + node_type: AssetNodeType + size: number + children: BatchUploadNodeOutput[] + upload_url?: string +} + +/** + * Request payload for batch upload + */ +export type BatchUploadPayload = { + children: BatchUploadNodeInput[] +} + +/** + * Response for batch upload request + */ +export type BatchUploadResponse = { + children: BatchUploadNodeOutput[] +}