From f3a335f4f6cd532c9a93a3a30f35c6447b462606 Mon Sep 17 00:00:00 2001 From: matevip Date: Sun, 31 May 2026 07:57:50 +0800 Subject: [PATCH] fix(wiki): re-ingest modified text files via content-hash change detection --- .../service/WikiDirectoryScanService.java | 46 ++++++++++--------- .../wiki/service/WikiRawMaterialService.java | 26 +++++++++++ .../WikiSourceWatcherServiceE2ETest.java | 8 ++++ 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java index 5c0f5fbd..fe475225 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java @@ -137,32 +137,36 @@ public class WikiDirectoryScanService { String fileName = file.getFileName().toString(); String ext = getExtension(fileName); - // 基于 sourcePath 去重 + if (TEXT_EXTENSIONS.contains(ext)) { + // Text files: dedup by content hash, so an unchanged file is + // skipped while a modified file (new hash) is re-ingested. + String content = Files.readString(file, StandardCharsets.UTF_8); + boolean fresh = rawService.ingestTextFileFromScan(kbId, fileName, absolutePath, content); + if (fresh) { + added++; + } else { + skipped++; + } + continue; + } + + // Binary files: dedup by source path. Re-ingest on content change + // is not detected here (hashing large binaries each scan is + // expensive) — modified binaries should be re-uploaded explicitly. WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath); if (existing != null) { skipped++; continue; } - - if (TEXT_EXTENSIONS.contains(ext)) { - // 文本文件：读取内容；记录 source path 以便重复扫描去重 - String content = Files.readString(file, StandardCharsets.UTF_8); - WikiRawMaterialEntity textRaw = rawService.addText(kbId, fileName, content); - if (textRaw != null) { - rawService.updateSourcePath(textRaw.getId(), absolutePath); - } - } else { - // 二进制文件：直接引用原始路径，不复制 - String sourceType = switch (ext) { - case "pdf" -> "pdf"; - case "docx", "doc" -> "docx"; - case "pptx", "ppt" -> "pptx"; - case "xlsx", "xls" -> "xlsx"; - case "html", "htm" -> "html"; - default -> "text"; - }; - rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file)); - } + String sourceType = switch (ext) { + case "pdf" -> "pdf"; + case "docx", "doc" -> "docx"; + case "pptx", "ppt" -> "pptx"; + case "xlsx", "xls" -> "xlsx"; + case "html", "htm" -> "html"; + default -> "text"; + }; + rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file)); added++; } catch (Exception e) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index 33d75ddc..0f7595f5 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -86,6 +86,32 @@ public class WikiRawMaterialService { .eq(WikiRawMaterialEntity::getSourcePath, sourcePath)); } + /** + * Import a text file discovered by a directory scan, detecting content + * changes by hash: unchanged content (a raw with the same hash already + * exists) is a no-op, while changed content creates a new raw and triggers + * processing — so a modified file is re-ingested rather than silently + * skipped. The originating path is recorded for diagnostics. + * + * @return {@code true} when the file was newly ingested (new or changed + * content), {@code false} when skipped as unchanged + */ + public boolean ingestTextFileFromScan(Long kbId, String fileName, String absolutePath, String content) { + String hash = computeHash(content); + WikiRawMaterialEntity sameContent = rawMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiRawMaterialEntity::getKbId, kbId) + .eq(WikiRawMaterialEntity::getContentHash, hash) + .last("LIMIT 1")); + // addText dedups internally by hash, so this reuses sameContent when + // unchanged and inserts + triggers processing when the content differs. + WikiRawMaterialEntity raw = addText(kbId, fileName, content); + if (raw != null) { + updateSourcePath(raw.getId(), absolutePath); + } + return sameContent == null; + } + /** * Record the originating file path on a raw material via a partial update, * so a later directory re-scan can dedup it by source path. Used for diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java index 5575026d..58dca587 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java @@ -54,6 +54,14 @@ class WikiSourceWatcherServiceE2ETest { Files.writeString(sourceDir.resolve("note-c.md"), "# Note C\n\ncontent c"); int secondAdded = watcherService.runScanCycle(); assertEquals(1, secondAdded, "only the newly added file should ingest"); + + // Re-scanning with no changes ingests nothing. + assertEquals(0, watcherService.runScanCycle(), "unchanged files must not re-ingest"); + + // Modifying an existing file's content re-ingests it (content hash changed). + Files.writeString(sourceDir.resolve("note-a.md"), "# Note A\n\nEDITED content a"); + int afterEdit = watcherService.runScanCycle(); + assertEquals(1, afterEdit, "a modified file must be re-ingested"); } @Test