From 4700d0312d0aaa37b33761f33967ed6e26d25c3c Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 19 Apr 2026 19:36:13 +0800 Subject: [PATCH] fix(wiki): deduplicate raw material uploads across all processing statuses MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: addFile()/addText() hash dedup only matched rows with status=completed, so the same file uploaded while in partial/pending/ processing/failed status would create a duplicate row. Fix: - Remove .eq(processingStatus, "completed") from dedup queries — match any non-deleted row with the same content hash in the KB - On dedup hit: completed/pending/processing → return as-is; partial/failed → trigger reprocess (partial enters resume branch) - Clean up the newly uploaded temp file when dedup discards it - Frontend: uploadRawFile/addRawText check for existing id in the list before unshift to prevent visual duplicates Test: WikiRawMaterialDedupTest — 10 cases covering all 5 statuses, reprocess triggers for partial/failed, no-op for others, insert only when no match. --- .../wiki/service/WikiRawMaterialService.java | 43 +++++- .../service/WikiRawMaterialDedupTest.java | 138 ++++++++++++++++++ mateclaw-ui/src/stores/useWikiStore.ts | 15 +- 3 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/wiki/service/WikiRawMaterialDedupTest.java diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index 010a53e5..66d32080 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -88,16 +88,14 @@ public class WikiRawMaterialService { public WikiRawMaterialEntity addText(Long kbId, String title, String content) { String hash = computeHash(content); - // 去重：相同 hash 且已处理过的材料直接返回 + // Dedup: reuse any existing row with the same hash in this KB (any status) WikiRawMaterialEntity existing = rawMapper.selectOne( new LambdaQueryWrapper() .eq(WikiRawMaterialEntity::getKbId, kbId) .eq(WikiRawMaterialEntity::getContentHash, hash) - .eq(WikiRawMaterialEntity::getProcessingStatus, "completed") .last("LIMIT 1")); if (existing != null) { - log.info("[Wiki] Duplicate text detected (hash={}), returning existing id={}", hash, existing.getId()); - return existing; + return handleDuplicate(existing); } WikiRawMaterialEntity entity = new WikiRawMaterialEntity(); @@ -142,17 +140,17 @@ public class WikiRawMaterialService { log.warn("[Wiki] Could not compute file hash for dedup: {}", e.getMessage()); } - // 去重：相同 hash 且已处理过的材料直接返回已有记录 + // Dedup: reuse any existing row with the same hash in this KB (any status) if (entity.getContentHash() != null) { WikiRawMaterialEntity existing = rawMapper.selectOne( new LambdaQueryWrapper() .eq(WikiRawMaterialEntity::getKbId, kbId) .eq(WikiRawMaterialEntity::getContentHash, entity.getContentHash()) - .eq(WikiRawMaterialEntity::getProcessingStatus, "completed") .last("LIMIT 1")); if (existing != null) { - log.info("[Wiki] Duplicate file detected (hash={}), returning existing id={}", entity.getContentHash(), existing.getId()); - return existing; + // Clean up the newly uploaded file — we won't use it + cleanupFile(sourcePath); + return handleDuplicate(existing); } } @@ -334,6 +332,35 @@ public class WikiRawMaterialService { return entity.getOriginalContent(); } + /** + * Handle a duplicate upload: decide what to do based on the existing row's status. + * - completed → return as-is (no reprocessing needed) + * - partial / failed → reprocess (partial enters resume branch) + * - pending / processing → return as-is (already queued or running) + */ + private WikiRawMaterialEntity handleDuplicate(WikiRawMaterialEntity existing) { + String prevStatus = existing.getProcessingStatus(); + log.info("[Wiki] Duplicate file detected, reusing id={}, prevStatus={}", existing.getId(), prevStatus); + + if ("partial".equals(prevStatus) || "failed".equals(prevStatus)) { + reprocess(existing.getId()); + } + // completed / pending / processing → return as-is + return existing; + } + + /** + * Delete a file from disk if it exists (cleanup for dedup-discarded uploads). + */ + private void cleanupFile(String path) { + if (path == null) return; + try { + java.nio.file.Files.deleteIfExists(java.nio.file.Paths.get(path)); + } catch (Exception e) { + log.warn("[Wiki] Failed to clean up duplicate upload file {}: {}", path, e.getMessage()); + } + } + private String computeHash(String content) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiRawMaterialDedupTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiRawMaterialDedupTest.java new file mode 100644 index 00000000..73407fe1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiRawMaterialDedupTest.java @@ -0,0 +1,138 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.tool.builtin.DocumentExtractTool; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.repository.WikiRawMaterialMapper; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Verifies that addText / addFile deduplication works across all processing + * statuses, not just 'completed'. A duplicate upload should always reuse + * the existing row — never insert a second one. + */ +class WikiRawMaterialDedupTest { + + private WikiRawMaterialMapper rawMapper; + private WikiKnowledgeBaseService kbService; + private ApplicationEventPublisher eventPublisher; + private WikiChunkService chunkService; + private WikiRawMaterialService service; + + private static final Long KB_ID = 1L; + + @BeforeEach + void setUp() { + rawMapper = mock(WikiRawMaterialMapper.class); + kbService = mock(WikiKnowledgeBaseService.class); + eventPublisher = mock(ApplicationEventPublisher.class); + chunkService = mock(WikiChunkService.class); + WikiProperties props = new WikiProperties(); + props.setAutoProcessOnUpload(false); // don't fire events during test + DocumentExtractTool docTool = mock(DocumentExtractTool.class); + + service = new WikiRawMaterialService(rawMapper, kbService, props, eventPublisher, docTool, chunkService); + } + + private WikiRawMaterialEntity existingRow(Long id, String status) { + WikiRawMaterialEntity e = new WikiRawMaterialEntity(); + e.setId(id); + e.setKbId(KB_ID); + e.setTitle("test.pdf"); + e.setProcessingStatus(status); + e.setContentHash("abc123"); + return e; + } + + // ==================== addText dedup ==================== + + @ParameterizedTest(name = "addText dedup when existing status = {0}") + @ValueSource(strings = {"completed", "partial", "failed", "pending", "processing"}) + @DisplayName("addText: same hash should reuse existing row regardless of status") + void addText_dedup_all_statuses(String status) { + WikiRawMaterialEntity existing = existingRow(42L, status); + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + // reprocess() calls selectById internally for partial/failed + when(rawMapper.selectById(42L)).thenReturn(existing); + + WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content"); + + assertEquals(42L, result.getId(), "should return existing id"); + // Should never insert a new row + verify(rawMapper, never()).insert(any(WikiRawMaterialEntity.class)); + } + + @Test + @DisplayName("addText: reprocess triggered for partial status") + void addText_dedup_partial_triggers_reprocess() { + WikiRawMaterialEntity existing = existingRow(42L, "partial"); + // First call: dedup lookup returns existing + // Second call (inside reprocess): selectById returns existing + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + when(rawMapper.selectById(42L)).thenReturn(existing); + + service.addText(KB_ID, "test", "some content"); + + // reprocess changes status to pending and fires event + assertEquals("pending", existing.getProcessingStatus()); + verify(eventPublisher).publishEvent(any()); + } + + @Test + @DisplayName("addText: reprocess triggered for failed status") + void addText_dedup_failed_triggers_reprocess() { + WikiRawMaterialEntity existing = existingRow(42L, "failed"); + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + when(rawMapper.selectById(42L)).thenReturn(existing); + + service.addText(KB_ID, "test", "some content"); + + assertEquals("pending", existing.getProcessingStatus()); + verify(eventPublisher).publishEvent(any()); + } + + @Test + @DisplayName("addText: completed status returns as-is, no reprocess") + void addText_dedup_completed_no_reprocess() { + WikiRawMaterialEntity existing = existingRow(42L, "completed"); + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + + WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content"); + + assertEquals("completed", result.getProcessingStatus()); + verify(eventPublisher, never()).publishEvent(any()); + } + + @Test + @DisplayName("addText: pending/processing status returns as-is, no reprocess") + void addText_dedup_pending_no_reprocess() { + WikiRawMaterialEntity existing = existingRow(42L, "pending"); + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing); + + WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content"); + + assertEquals("pending", result.getProcessingStatus()); + verify(eventPublisher, never()).publishEvent(any()); + } + + @Test + @DisplayName("addText: no existing row inserts new record") + void addText_no_duplicate_inserts() { + when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + + service.addText(KB_ID, "test", "brand new content"); + + verify(rawMapper).insert(any(WikiRawMaterialEntity.class)); + verify(kbService).incrementRawCount(KB_ID); + } +} diff --git a/mateclaw-ui/src/stores/useWikiStore.ts b/mateclaw-ui/src/stores/useWikiStore.ts index afd37f30..d6697d67 100644 --- a/mateclaw-ui/src/stores/useWikiStore.ts +++ b/mateclaw-ui/src/stores/useWikiStore.ts @@ -108,7 +108,12 @@ export const useWikiStore = defineStore('wiki', () => { async function addRawText(kbId: number, title: string, content: string) { const res: any = await wikiApi.addRawText(kbId, { title, content }) const raw = res.data || res - rawMaterials.value.unshift(raw) + const existingIdx = rawMaterials.value.findIndex(r => r.id === raw.id) + if (existingIdx >= 0) { + rawMaterials.value[existingIdx] = raw + } else { + rawMaterials.value.unshift(raw) + } return raw } @@ -117,7 +122,13 @@ export const useWikiStore = defineStore('wiki', () => { formData.append('file', file) const res: any = await wikiApi.uploadRaw(kbId, formData) const raw = res.data || res - rawMaterials.value.unshift(raw) + // Dedup: if backend returned an existing record, replace it in the list instead of adding a duplicate + const existingIdx = rawMaterials.value.findIndex(r => r.id === raw.id) + if (existingIdx >= 0) { + rawMaterials.value[existingIdx] = raw + } else { + rawMaterials.value.unshift(raw) + } return raw }