fix(wiki): stabilize concurrent raw-material uploads and PostgreSQL-compatible IDs

Give raw-material uploads a dedicated five-minute timeout and process file-picker and drag/drop uploads through a shared two-worker queue, so constrained uplinks no longer abort multipart requests at the global 30-second deadline.

Switch wiki processing jobs and page citations to application-assigned IDs: the PostgreSQL/Kingbase migrations define plain BIGINT primary keys without identity defaults, so database-generated keys fail on insert.
This commit is contained in:
matevip 2026-07-15 14:27:14 +08:00
parent e2c3cfd5b4
commit cd0360ff3f
9 changed files with 124 additions and 5 deletions

View File

@ -13,7 +13,7 @@ import java.time.LocalDateTime;
@TableName("mate_wiki_processing_job")
public class WikiProcessingJobEntity {
@TableId(type = IdType.AUTO)
@TableId(type = IdType.ASSIGN_ID)
private Long id;
private Long kbId;

View File

@ -13,7 +13,7 @@ import java.time.LocalDateTime;
@TableName("mate_wiki_page_citation")
public class WikiPageCitationEntity {
@TableId(type = IdType.AUTO)
@TableId(type = IdType.ASSIGN_ID)
private Long id;
private Long pageId;

View File

@ -0,0 +1,19 @@
package vip.mate.wiki.job.model;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class WikiProcessingJobEntityTest {
@Test
void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException {
TableId tableId = WikiProcessingJobEntity.class
.getDeclaredField("id")
.getAnnotation(TableId.class);
assertEquals(IdType.ASSIGN_ID, tableId.type());
}
}

View File

@ -0,0 +1,19 @@
package vip.mate.wiki.model;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.annotation.TableId;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
class WikiPageCitationEntityTest {
@Test
void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException {
TableId tableId = WikiPageCitationEntity.class
.getDeclaredField("id")
.getAnnotation(TableId.class);
assertEquals(IdType.ASSIGN_ID, tableId.type());
}
}

View File

@ -0,0 +1,23 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import { http, wikiApi } from '@/api'
describe('wikiApi.uploadRaw', () => {
afterEach(() => {
vi.restoreAllMocks()
})
it('uses the dedicated five-minute upload timeout instead of the global request timeout', async () => {
const post = vi.spyOn(http, 'post').mockResolvedValue({ data: { id: 'raw-1' } })
const formData = new FormData()
formData.append('file', new File(['fixture'], 'fixture.txt', { type: 'text/plain' }))
await wikiApi.uploadRaw(42, formData)
expect(post).toHaveBeenCalledWith(
'/wiki/knowledge-bases/42/raw/upload',
formData,
expect.objectContaining({ timeout: 300_000 }),
)
})
})

View File

@ -1,5 +1,6 @@
import axios from 'axios'
import { handleAuthFailure, updateTokenFromHeader } from '@/utils/auth'
import { WIKI_UPLOAD_TIMEOUT_MS } from '@/utils/wikiUpload'
import type {
ApprovalGrant,
ApprovalGrantPage,
@ -840,6 +841,7 @@ export const wikiApi = {
uploadRaw: (kbId: number, formData: FormData, onProgress?: (pct: number) => void) =>
http.post(`/wiki/knowledge-bases/${kbId}/raw/upload`, formData, {
headers: { 'Content-Type': 'multipart/form-data' },
timeout: WIKI_UPLOAD_TIMEOUT_MS,
onUploadProgress: onProgress
? (e) => { if (e.total) onProgress(Math.round((e.loaded / e.total) * 100)) }
: undefined,

View File

@ -0,0 +1,27 @@
import { describe, expect, it } from 'vitest'
import { processWithConcurrency } from '@/utils/wikiUpload'
describe('processWithConcurrency', () => {
it('never runs more than the configured number of workers', async () => {
let active = 0
let maxActive = 0
const completed: number[] = []
await processWithConcurrency([1, 2, 3, 4, 5], async (item) => {
active += 1
maxActive = Math.max(maxActive, active)
await new Promise(resolve => setTimeout(resolve, 5))
completed.push(item)
active -= 1
}, 2)
expect(maxActive).toBe(2)
expect(completed.sort()).toEqual([1, 2, 3, 4, 5])
})
it('rejects invalid concurrency instead of silently skipping work', async () => {
await expect(processWithConcurrency([1], async () => undefined, 0))
.rejects.toThrow('concurrency must be a positive integer')
})
})

View File

@ -0,0 +1,23 @@
export const WIKI_UPLOAD_TIMEOUT_MS = 5 * 60 * 1000
export const WIKI_UPLOAD_CONCURRENCY = 2
export async function processWithConcurrency<T>(
items: readonly T[],
worker: (item: T, index: number) => Promise<void>,
concurrency = WIKI_UPLOAD_CONCURRENCY,
): Promise<void> {
if (!Number.isInteger(concurrency) || concurrency < 1) {
throw new RangeError('concurrency must be a positive integer')
}
const iterator = items.entries()
async function runWorker() {
for (const [index, item] of iterator) {
await worker(item, index)
}
}
const workerCount = Math.min(concurrency, items.length)
await Promise.all(Array.from({ length: workerCount }, () => runWorker()))
}

View File

@ -382,6 +382,7 @@ import { Download } from '@element-plus/icons-vue'
import { useWikiStore } from '@/stores/useWikiStore'
import { useWorkspaceStore } from '@/stores/useWorkspaceStore'
import { wikiApi } from '@/api/index'
import { processWithConcurrency } from '@/utils/wikiUpload'
import JobStageBar from './JobStageBar.vue'
import type { WikiProcessingJob } from '@/composables/useWikiJobPoller'
@ -831,7 +832,10 @@ const { isDragging, onDragEnter, onDragLeave, onDrop: handleDrop } = useFileDrop
async function uploadDroppedFiles(event: DragEvent) {
if (!event.dataTransfer?.files || !store.currentKB) return
const kbId = store.currentKB.id
await Promise.all(Array.from(event.dataTransfer.files).map(f => uploadFile(kbId, f)))
await processWithConcurrency(
Array.from(event.dataTransfer.files),
file => uploadFile(kbId, file),
)
}
// Optimistic upload items
@ -886,8 +890,10 @@ async function handleFileSelect(event: Event) {
const input = event.target as HTMLInputElement
if (!input.files || !store.currentKB) return
const kbId = store.currentKB.id
// Upload all files concurrently
await Promise.all(Array.from(input.files).map(f => uploadFile(kbId, f)))
await processWithConcurrency(
Array.from(input.files),
file => uploadFile(kbId, file),
)
input.value = ''
}