mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(wiki): stabilize concurrent raw-material uploads and PostgreSQL-compatible IDs
Give raw-material uploads a dedicated five-minute timeout and process file-picker and drag/drop uploads through a shared two-worker queue, so constrained uplinks no longer abort multipart requests at the global 30-second deadline. Switch wiki processing jobs and page citations to application-assigned IDs: the PostgreSQL/Kingbase migrations define plain BIGINT primary keys without identity defaults, so database-generated keys fail on insert.
This commit is contained in:
parent
e82adfed42
commit
8a6dd1fa67
@ -13,7 +13,7 @@ import java.time.LocalDateTime;
|
||||
@TableName("mate_wiki_processing_job")
|
||||
public class WikiProcessingJobEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
@TableId(type = IdType.ASSIGN_ID)
|
||||
private Long id;
|
||||
|
||||
private Long kbId;
|
||||
|
||||
@ -13,7 +13,7 @@ import java.time.LocalDateTime;
|
||||
@TableName("mate_wiki_page_citation")
|
||||
public class WikiPageCitationEntity {
|
||||
|
||||
@TableId(type = IdType.AUTO)
|
||||
@TableId(type = IdType.ASSIGN_ID)
|
||||
private Long id;
|
||||
|
||||
private Long pageId;
|
||||
|
||||
@ -0,0 +1,19 @@
|
||||
package vip.mate.wiki.job.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class WikiProcessingJobEntityTest {
|
||||
|
||||
@Test
|
||||
void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException {
|
||||
TableId tableId = WikiProcessingJobEntity.class
|
||||
.getDeclaredField("id")
|
||||
.getAnnotation(TableId.class);
|
||||
|
||||
assertEquals(IdType.ASSIGN_ID, tableId.type());
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
package vip.mate.wiki.model;
|
||||
|
||||
import com.baomidou.mybatisplus.annotation.IdType;
|
||||
import com.baomidou.mybatisplus.annotation.TableId;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
|
||||
class WikiPageCitationEntityTest {
|
||||
|
||||
@Test
|
||||
void postgresCompatiblePrimaryKeyUsesApplicationAssignedId() throws NoSuchFieldException {
|
||||
TableId tableId = WikiPageCitationEntity.class
|
||||
.getDeclaredField("id")
|
||||
.getAnnotation(TableId.class);
|
||||
|
||||
assertEquals(IdType.ASSIGN_ID, tableId.type());
|
||||
}
|
||||
}
|
||||
23
mateclaw-ui/src/api/__tests__/wikiUpload.test.ts
Normal file
23
mateclaw-ui/src/api/__tests__/wikiUpload.test.ts
Normal file
@ -0,0 +1,23 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { http, wikiApi } from '@/api'
|
||||
|
||||
describe('wikiApi.uploadRaw', () => {
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('uses the dedicated five-minute upload timeout instead of the global request timeout', async () => {
|
||||
const post = vi.spyOn(http, 'post').mockResolvedValue({ data: { id: 'raw-1' } })
|
||||
const formData = new FormData()
|
||||
formData.append('file', new File(['fixture'], 'fixture.txt', { type: 'text/plain' }))
|
||||
|
||||
await wikiApi.uploadRaw(42, formData)
|
||||
|
||||
expect(post).toHaveBeenCalledWith(
|
||||
'/wiki/knowledge-bases/42/raw/upload',
|
||||
formData,
|
||||
expect.objectContaining({ timeout: 300_000 }),
|
||||
)
|
||||
})
|
||||
})
|
||||
@ -1,5 +1,6 @@
|
||||
import axios from 'axios'
|
||||
import { handleAuthFailure, updateTokenFromHeader } from '@/utils/auth'
|
||||
import { WIKI_UPLOAD_TIMEOUT_MS } from '@/utils/wikiUpload'
|
||||
import type {
|
||||
ApprovalGrant,
|
||||
ApprovalGrantPage,
|
||||
@ -840,6 +841,7 @@ export const wikiApi = {
|
||||
uploadRaw: (kbId: number, formData: FormData, onProgress?: (pct: number) => void) =>
|
||||
http.post(`/wiki/knowledge-bases/${kbId}/raw/upload`, formData, {
|
||||
headers: { 'Content-Type': 'multipart/form-data' },
|
||||
timeout: WIKI_UPLOAD_TIMEOUT_MS,
|
||||
onUploadProgress: onProgress
|
||||
? (e) => { if (e.total) onProgress(Math.round((e.loaded / e.total) * 100)) }
|
||||
: undefined,
|
||||
|
||||
27
mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts
Normal file
27
mateclaw-ui/src/utils/__tests__/wikiUpload.test.ts
Normal file
@ -0,0 +1,27 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { processWithConcurrency } from '@/utils/wikiUpload'
|
||||
|
||||
describe('processWithConcurrency', () => {
|
||||
it('never runs more than the configured number of workers', async () => {
|
||||
let active = 0
|
||||
let maxActive = 0
|
||||
const completed: number[] = []
|
||||
|
||||
await processWithConcurrency([1, 2, 3, 4, 5], async (item) => {
|
||||
active += 1
|
||||
maxActive = Math.max(maxActive, active)
|
||||
await new Promise(resolve => setTimeout(resolve, 5))
|
||||
completed.push(item)
|
||||
active -= 1
|
||||
}, 2)
|
||||
|
||||
expect(maxActive).toBe(2)
|
||||
expect(completed.sort()).toEqual([1, 2, 3, 4, 5])
|
||||
})
|
||||
|
||||
it('rejects invalid concurrency instead of silently skipping work', async () => {
|
||||
await expect(processWithConcurrency([1], async () => undefined, 0))
|
||||
.rejects.toThrow('concurrency must be a positive integer')
|
||||
})
|
||||
})
|
||||
23
mateclaw-ui/src/utils/wikiUpload.ts
Normal file
23
mateclaw-ui/src/utils/wikiUpload.ts
Normal file
@ -0,0 +1,23 @@
|
||||
export const WIKI_UPLOAD_TIMEOUT_MS = 5 * 60 * 1000
|
||||
|
||||
export const WIKI_UPLOAD_CONCURRENCY = 2
|
||||
|
||||
export async function processWithConcurrency<T>(
|
||||
items: readonly T[],
|
||||
worker: (item: T, index: number) => Promise<void>,
|
||||
concurrency = WIKI_UPLOAD_CONCURRENCY,
|
||||
): Promise<void> {
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1) {
|
||||
throw new RangeError('concurrency must be a positive integer')
|
||||
}
|
||||
|
||||
const iterator = items.entries()
|
||||
async function runWorker() {
|
||||
for (const [index, item] of iterator) {
|
||||
await worker(item, index)
|
||||
}
|
||||
}
|
||||
|
||||
const workerCount = Math.min(concurrency, items.length)
|
||||
await Promise.all(Array.from({ length: workerCount }, () => runWorker()))
|
||||
}
|
||||
@ -382,6 +382,7 @@ import { Download } from '@element-plus/icons-vue'
|
||||
import { useWikiStore } from '@/stores/useWikiStore'
|
||||
import { useWorkspaceStore } from '@/stores/useWorkspaceStore'
|
||||
import { wikiApi } from '@/api/index'
|
||||
import { processWithConcurrency } from '@/utils/wikiUpload'
|
||||
import JobStageBar from './JobStageBar.vue'
|
||||
import type { WikiProcessingJob } from '@/composables/useWikiJobPoller'
|
||||
|
||||
@ -831,7 +832,10 @@ const { isDragging, onDragEnter, onDragLeave, onDrop: handleDrop } = useFileDrop
|
||||
async function uploadDroppedFiles(event: DragEvent) {
|
||||
if (!event.dataTransfer?.files || !store.currentKB) return
|
||||
const kbId = store.currentKB.id
|
||||
await Promise.all(Array.from(event.dataTransfer.files).map(f => uploadFile(kbId, f)))
|
||||
await processWithConcurrency(
|
||||
Array.from(event.dataTransfer.files),
|
||||
file => uploadFile(kbId, file),
|
||||
)
|
||||
}
|
||||
|
||||
// ─── Optimistic upload items ──────────────────────────────────────────────────
|
||||
@ -886,8 +890,10 @@ async function handleFileSelect(event: Event) {
|
||||
const input = event.target as HTMLInputElement
|
||||
if (!input.files || !store.currentKB) return
|
||||
const kbId = store.currentKB.id
|
||||
// Upload all files concurrently
|
||||
await Promise.all(Array.from(input.files).map(f => uploadFile(kbId, f)))
|
||||
await processWithConcurrency(
|
||||
Array.from(input.files),
|
||||
file => uploadFile(kbId, file),
|
||||
)
|
||||
input.value = ''
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user