mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(wiki): deduplicate raw material uploads across all processing statuses
Root cause: addFile()/addText() hash dedup only matched rows with status=completed, so the same file uploaded while in partial/pending/ processing/failed status would create a duplicate row. Fix: - Remove .eq(processingStatus, "completed") from dedup queries — match any non-deleted row with the same content hash in the KB - On dedup hit: completed/pending/processing → return as-is; partial/failed → trigger reprocess (partial enters resume branch) - Clean up the newly uploaded temp file when dedup discards it - Frontend: uploadRawFile/addRawText check for existing id in the list before unshift to prevent visual duplicates Test: WikiRawMaterialDedupTest — 10 cases covering all 5 statuses, reprocess triggers for partial/failed, no-op for others, insert only when no match.
This commit is contained in:
matevip
parent
d9dfd602f9
commit
4700d0312d
@ -88,16 +88,14 @@ public class WikiRawMaterialService {
|
||||
public WikiRawMaterialEntity addText(Long kbId, String title, String content) {
|
||||
String hash = computeHash(content);
|
||||
|
||||
// 去重：相同 hash 且已处理过的材料直接返回
|
||||
// Dedup: reuse any existing row with the same hash in this KB (any status)
|
||||
WikiRawMaterialEntity existing = rawMapper.selectOne(
|
||||
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||
.eq(WikiRawMaterialEntity::getContentHash, hash)
|
||||
.eq(WikiRawMaterialEntity::getProcessingStatus, "completed")
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
log.info("[Wiki] Duplicate text detected (hash={}), returning existing id={}", hash, existing.getId());
|
||||
return existing;
|
||||
return handleDuplicate(existing);
|
||||
}
|
||||
|
||||
WikiRawMaterialEntity entity = new WikiRawMaterialEntity();
|
||||
@ -142,17 +140,17 @@ public class WikiRawMaterialService {
|
||||
log.warn("[Wiki] Could not compute file hash for dedup: {}", e.getMessage());
|
||||
}
|
||||
|
||||
// 去重：相同 hash 且已处理过的材料直接返回已有记录
|
||||
// Dedup: reuse any existing row with the same hash in this KB (any status)
|
||||
if (entity.getContentHash() != null) {
|
||||
WikiRawMaterialEntity existing = rawMapper.selectOne(
|
||||
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||
.eq(WikiRawMaterialEntity::getContentHash, entity.getContentHash())
|
||||
.eq(WikiRawMaterialEntity::getProcessingStatus, "completed")
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
log.info("[Wiki] Duplicate file detected (hash={}), returning existing id={}", entity.getContentHash(), existing.getId());
|
||||
return existing;
|
||||
// Clean up the newly uploaded file — we won't use it
|
||||
cleanupFile(sourcePath);
|
||||
return handleDuplicate(existing);
|
||||
}
|
||||
}
|
||||
|
||||
@ -334,6 +332,35 @@ public class WikiRawMaterialService {
|
||||
return entity.getOriginalContent();
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a duplicate upload: decide what to do based on the existing row's status.
|
||||
* - completed → return as-is (no reprocessing needed)
|
||||
* - partial / failed → reprocess (partial enters resume branch)
|
||||
* - pending / processing → return as-is (already queued or running)
|
||||
*/
|
||||
private WikiRawMaterialEntity handleDuplicate(WikiRawMaterialEntity existing) {
|
||||
String prevStatus = existing.getProcessingStatus();
|
||||
log.info("[Wiki] Duplicate file detected, reusing id={}, prevStatus={}", existing.getId(), prevStatus);
|
||||
|
||||
if ("partial".equals(prevStatus) || "failed".equals(prevStatus)) {
|
||||
reprocess(existing.getId());
|
||||
}
|
||||
// completed / pending / processing → return as-is
|
||||
return existing;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a file from disk if it exists (cleanup for dedup-discarded uploads).
|
||||
*/
|
||||
private void cleanupFile(String path) {
|
||||
if (path == null) return;
|
||||
try {
|
||||
java.nio.file.Files.deleteIfExists(java.nio.file.Paths.get(path));
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Failed to clean up duplicate upload file {}: {}", path, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private String computeHash(String content) {
|
||||
try {
|
||||
MessageDigest digest = MessageDigest.getInstance("SHA-256");
|
||||
|
||||
@ -0,0 +1,138 @@
|
||||
package vip.mate.wiki.service;
|
||||
|
||||
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.DisplayName;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.params.ParameterizedTest;
|
||||
import org.junit.jupiter.params.provider.ValueSource;
|
||||
import org.springframework.context.ApplicationEventPublisher;
|
||||
import vip.mate.tool.builtin.DocumentExtractTool;
|
||||
import vip.mate.wiki.WikiProperties;
|
||||
import vip.mate.wiki.model.WikiRawMaterialEntity;
|
||||
import vip.mate.wiki.repository.WikiRawMaterialMapper;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
/**
|
||||
* Verifies that addText / addFile deduplication works across all processing
|
||||
* statuses, not just 'completed'. A duplicate upload should always reuse
|
||||
* the existing row — never insert a second one.
|
||||
*/
|
||||
class WikiRawMaterialDedupTest {
|
||||
|
||||
private WikiRawMaterialMapper rawMapper;
|
||||
private WikiKnowledgeBaseService kbService;
|
||||
private ApplicationEventPublisher eventPublisher;
|
||||
private WikiChunkService chunkService;
|
||||
private WikiRawMaterialService service;
|
||||
|
||||
private static final Long KB_ID = 1L;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
rawMapper = mock(WikiRawMaterialMapper.class);
|
||||
kbService = mock(WikiKnowledgeBaseService.class);
|
||||
eventPublisher = mock(ApplicationEventPublisher.class);
|
||||
chunkService = mock(WikiChunkService.class);
|
||||
WikiProperties props = new WikiProperties();
|
||||
props.setAutoProcessOnUpload(false); // don't fire events during test
|
||||
DocumentExtractTool docTool = mock(DocumentExtractTool.class);
|
||||
|
||||
service = new WikiRawMaterialService(rawMapper, kbService, props, eventPublisher, docTool, chunkService);
|
||||
}
|
||||
|
||||
private WikiRawMaterialEntity existingRow(Long id, String status) {
|
||||
WikiRawMaterialEntity e = new WikiRawMaterialEntity();
|
||||
e.setId(id);
|
||||
e.setKbId(KB_ID);
|
||||
e.setTitle("test.pdf");
|
||||
e.setProcessingStatus(status);
|
||||
e.setContentHash("abc123");
|
||||
return e;
|
||||
}
|
||||
|
||||
// ==================== addText dedup ====================
|
||||
|
||||
@ParameterizedTest(name = "addText dedup when existing status = {0}")
|
||||
@ValueSource(strings = {"completed", "partial", "failed", "pending", "processing"})
|
||||
@DisplayName("addText: same hash should reuse existing row regardless of status")
|
||||
void addText_dedup_all_statuses(String status) {
|
||||
WikiRawMaterialEntity existing = existingRow(42L, status);
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing);
|
||||
// reprocess() calls selectById internally for partial/failed
|
||||
when(rawMapper.selectById(42L)).thenReturn(existing);
|
||||
|
||||
WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content");
|
||||
|
||||
assertEquals(42L, result.getId(), "should return existing id");
|
||||
// Should never insert a new row
|
||||
verify(rawMapper, never()).insert(any(WikiRawMaterialEntity.class));
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("addText: reprocess triggered for partial status")
|
||||
void addText_dedup_partial_triggers_reprocess() {
|
||||
WikiRawMaterialEntity existing = existingRow(42L, "partial");
|
||||
// First call: dedup lookup returns existing
|
||||
// Second call (inside reprocess): selectById returns existing
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing);
|
||||
when(rawMapper.selectById(42L)).thenReturn(existing);
|
||||
|
||||
service.addText(KB_ID, "test", "some content");
|
||||
|
||||
// reprocess changes status to pending and fires event
|
||||
assertEquals("pending", existing.getProcessingStatus());
|
||||
verify(eventPublisher).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("addText: reprocess triggered for failed status")
|
||||
void addText_dedup_failed_triggers_reprocess() {
|
||||
WikiRawMaterialEntity existing = existingRow(42L, "failed");
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing);
|
||||
when(rawMapper.selectById(42L)).thenReturn(existing);
|
||||
|
||||
service.addText(KB_ID, "test", "some content");
|
||||
|
||||
assertEquals("pending", existing.getProcessingStatus());
|
||||
verify(eventPublisher).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("addText: completed status returns as-is, no reprocess")
|
||||
void addText_dedup_completed_no_reprocess() {
|
||||
WikiRawMaterialEntity existing = existingRow(42L, "completed");
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing);
|
||||
|
||||
WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content");
|
||||
|
||||
assertEquals("completed", result.getProcessingStatus());
|
||||
verify(eventPublisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("addText: pending/processing status returns as-is, no reprocess")
|
||||
void addText_dedup_pending_no_reprocess() {
|
||||
WikiRawMaterialEntity existing = existingRow(42L, "pending");
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(existing);
|
||||
|
||||
WikiRawMaterialEntity result = service.addText(KB_ID, "test", "some content");
|
||||
|
||||
assertEquals("pending", result.getProcessingStatus());
|
||||
verify(eventPublisher, never()).publishEvent(any());
|
||||
}
|
||||
|
||||
@Test
|
||||
@DisplayName("addText: no existing row inserts new record")
|
||||
void addText_no_duplicate_inserts() {
|
||||
when(rawMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null);
|
||||
|
||||
service.addText(KB_ID, "test", "brand new content");
|
||||
|
||||
verify(rawMapper).insert(any(WikiRawMaterialEntity.class));
|
||||
verify(kbService).incrementRawCount(KB_ID);
|
||||
}
|
||||
}
|
||||
@ -108,7 +108,12 @@ export const useWikiStore = defineStore('wiki', () => {
|
||||
async function addRawText(kbId: number, title: string, content: string) {
|
||||
const res: any = await wikiApi.addRawText(kbId, { title, content })
|
||||
const raw = res.data || res
|
||||
rawMaterials.value.unshift(raw)
|
||||
const existingIdx = rawMaterials.value.findIndex(r => r.id === raw.id)
|
||||
if (existingIdx >= 0) {
|
||||
rawMaterials.value[existingIdx] = raw
|
||||
} else {
|
||||
rawMaterials.value.unshift(raw)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
@ -117,7 +122,13 @@ export const useWikiStore = defineStore('wiki', () => {
|
||||
formData.append('file', file)
|
||||
const res: any = await wikiApi.uploadRaw(kbId, formData)
|
||||
const raw = res.data || res
|
||||
rawMaterials.value.unshift(raw)
|
||||
// Dedup: if backend returned an existing record, replace it in the list instead of adding a duplicate
|
||||
const existingIdx = rawMaterials.value.findIndex(r => r.id === raw.id)
|
||||
if (existingIdx >= 0) {
|
||||
rawMaterials.value[existingIdx] = raw
|
||||
} else {
|
||||
rawMaterials.value.unshift(raw)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user