From 8a6dd1fa672dd06fbad5e6ab9b78d94c6c588964 Mon Sep 17 00:00:00 2001 From: RobinZhiBin Date: Wed, 15 Jul 2026 09:51:20 +0800 Subject: [PATCH] fix(wiki): stabilize concurrent raw-material uploads and PostgreSQL-compatible IDs Give raw-material uploads a dedicated five-minute timeout and process file-picker and drag/drop uploads through a shared two-worker queue, so constrained uplinks no longer abort multipart requests at the global 30-second deadline. Switch wiki processing jobs and page citations to application-assigned IDs: the PostgreSQL/Kingbase migrations define plain BIGINT primary keys without identity defaults, so database-generated keys fail on insert. --- .../job/model/WikiProcessingJobEntity.java | 2 +- .../wiki/model/WikiPageCitationEntity.java | 2 +- .../model/WikiProcessingJobEntityTest.java | 19 +++++++++++++ .../model/WikiPageCitationEntityTest.java | 19 +++++++++++++ .../src/api/__tests__/wikiUpload.test.ts | 23 ++++++++++++++++ mateclaw-ui/src/api/index.ts | 2 ++ .../src/utils/__tests__/wikiUpload.test.ts | 27 +++++++++++++++++++ mateclaw-ui/src/utils/wikiUpload.ts | 23 ++++++++++++++++ .../Wiki/components/RawMaterialPanel.vue | 12 ++++++--- 9 files changed, 124 insertions(+), 5 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/wiki/job/model/WikiProcessingJobEntityTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/wiki/model/WikiPageCitationEntityTest.java create mode 100644 mateclaw-ui/src/api/__tests__/wikiUpload.test.ts create mode 100644 mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts create mode 100644 mateclaw-ui/src/utils/wikiUpload.ts diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java index 7be65cca..60bbf378 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java @@ -13,7 +13,7 @@ import java.time.LocalDateTime; @TableName("mate_wiki_processing_job") public class WikiProcessingJobEntity { - @TableId(type = IdType.AUTO) + @TableId(type = IdType.ASSIGN_ID) private Long id; private Long kbId; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java index 8de827ad..f645654e 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java @@ -13,7 +13,7 @@ import java.time.LocalDateTime; @TableName("mate_wiki_page_citation") public class WikiPageCitationEntity { - @TableId(type = IdType.AUTO) + @TableId(type = IdType.ASSIGN_ID) private Long id; private Long pageId; diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/job/model/WikiProcessingJobEntityTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/job/model/WikiProcessingJobEntityTest.java new file mode 100644 index 00000000..1e79dcb3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/job/model/WikiProcessingJobEntityTest.java @@ -0,0 +1,19 @@ +package vip.mate.wiki.job.model; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class WikiProcessingJobEntityTest { + + @Test + void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException { + TableId tableId = WikiProcessingJobEntity.class + .getDeclaredField("id") + .getAnnotation(TableId.class); + + assertEquals(IdType.ASSIGN_ID, tableId.type()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/model/WikiPageCitationEntityTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/model/WikiPageCitationEntityTest.java new file mode 100644 index 00000000..f8ef13ec --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/model/WikiPageCitationEntityTest.java @@ -0,0 +1,19 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class WikiPageCitationEntityTest { + + @Test + void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException { + TableId tableId = WikiPageCitationEntity.class + .getDeclaredField("id") + .getAnnotation(TableId.class); + + assertEquals(IdType.ASSIGN_ID, tableId.type()); + } +} diff --git a/mateclaw-ui/src/api/__tests__/wikiUpload.test.ts b/mateclaw-ui/src/api/__tests__/wikiUpload.test.ts new file mode 100644 index 00000000..9b63dfe7 --- /dev/null +++ b/mateclaw-ui/src/api/__tests__/wikiUpload.test.ts @@ -0,0 +1,23 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { http, wikiApi } from '@/api' + +describe('wikiApi.uploadRaw', () => { + afterEach(() => { + vi.restoreAllMocks() + }) + + it('uses the dedicated five-minute upload timeout instead of the global request timeout', async () => { + const post = vi.spyOn(http, 'post').mockResolvedValue({ data: { id: 'raw-1' } }) + const formData = new FormData() + formData.append('file', new File(['fixture'], 'fixture.txt', { type: 'text/plain' })) + + await wikiApi.uploadRaw(42, formData) + + expect(post).toHaveBeenCalledWith( + '/wiki/knowledge-bases/42/raw/upload', + formData, + expect.objectContaining({ timeout: 300_000 }), + ) + }) +}) diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 62f6e7fa..25d65b70 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -1,5 +1,6 @@ import axios from 'axios' import { handleAuthFailure, updateTokenFromHeader } from '@/utils/auth' +import { WIKI_UPLOAD_TIMEOUT_MS } from '@/utils/wikiUpload' import type { ApprovalGrant, ApprovalGrantPage, @@ -840,6 +841,7 @@ export const wikiApi = { uploadRaw: (kbId: number, formData: FormData, onProgress?: (pct: number) => void) => http.post(`/wiki/knowledge-bases/${kbId}/raw/upload`, formData, { headers: { 'Content-Type': 'multipart/form-data' }, + timeout: WIKI_UPLOAD_TIMEOUT_MS, onUploadProgress: onProgress ? (e) => { if (e.total) onProgress(Math.round((e.loaded / e.total) * 100)) } : undefined, diff --git a/mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts b/mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts new file mode 100644 index 00000000..6462d241 --- /dev/null +++ b/mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from 'vitest' + +import { processWithConcurrency } from '@/utils/wikiUpload' + +describe('processWithConcurrency', () => { + it('never runs more than the configured number of workers', async () => { + let active = 0 + let maxActive = 0 + const completed: number[] = [] + + await processWithConcurrency([1, 2, 3, 4, 5], async (item) => { + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise(resolve => setTimeout(resolve, 5)) + completed.push(item) + active -= 1 + }, 2) + + expect(maxActive).toBe(2) + expect(completed.sort()).toEqual([1, 2, 3, 4, 5]) + }) + + it('rejects invalid concurrency instead of silently skipping work', async () => { + await expect(processWithConcurrency([1], async () => undefined, 0)) + .rejects.toThrow('concurrency must be a positive integer') + }) +}) diff --git a/mateclaw-ui/src/utils/wikiUpload.ts b/mateclaw-ui/src/utils/wikiUpload.ts new file mode 100644 index 00000000..c7b6b9f3 --- /dev/null +++ b/mateclaw-ui/src/utils/wikiUpload.ts @@ -0,0 +1,23 @@ +export const WIKI_UPLOAD_TIMEOUT_MS = 5 * 60 * 1000 + +export const WIKI_UPLOAD_CONCURRENCY = 2 + +export async function processWithConcurrency( + items: readonly T[], + worker: (item: T, index: number) => Promise, + concurrency = WIKI_UPLOAD_CONCURRENCY, +): Promise { + if (!Number.isInteger(concurrency) || concurrency < 1) { + throw new RangeError('concurrency must be a positive integer') + } + + const iterator = items.entries() + async function runWorker() { + for (const [index, item] of iterator) { + await worker(item, index) + } + } + + const workerCount = Math.min(concurrency, items.length) + await Promise.all(Array.from({ length: workerCount }, () => runWorker())) +} diff --git a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue index 96e62dcc..8f966405 100644 --- a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue +++ b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue @@ -382,6 +382,7 @@ import { Download } from '@element-plus/icons-vue' import { useWikiStore } from '@/stores/useWikiStore' import { useWorkspaceStore } from '@/stores/useWorkspaceStore' import { wikiApi } from '@/api/index' +import { processWithConcurrency } from '@/utils/wikiUpload' import JobStageBar from './JobStageBar.vue' import type { WikiProcessingJob } from '@/composables/useWikiJobPoller' @@ -831,7 +832,10 @@ const { isDragging, onDragEnter, onDragLeave, onDrop: handleDrop } = useFileDrop async function uploadDroppedFiles(event: DragEvent) { if (!event.dataTransfer?.files || !store.currentKB) return const kbId = store.currentKB.id - await Promise.all(Array.from(event.dataTransfer.files).map(f => uploadFile(kbId, f))) + await processWithConcurrency( + Array.from(event.dataTransfer.files), + file => uploadFile(kbId, file), + ) } // ─── Optimistic upload items ────────────────────────────────────────────────── @@ -886,8 +890,10 @@ async function handleFileSelect(event: Event) { const input = event.target as HTMLInputElement if (!input.files || !store.currentKB) return const kbId = store.currentKB.id - // Upload all files concurrently - await Promise.all(Array.from(input.files).map(f => uploadFile(kbId, f))) + await processWithConcurrency( + Array.from(input.files), + file => uploadFile(kbId, file), + ) input.value = '' }