diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java index d381591d..a7e985ad 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java @@ -165,14 +165,9 @@ public class WikiDirectoryScanService { continue; } - // Binary files: dedup by source path. Re-ingest on content change - // is not detected here (hashing large binaries each scan is - // expensive) — modified binaries should be re-uploaded explicitly. - WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath); - if (existing != null) { - skipped++; - continue; - } + // Binary files: dedup by content hash too, so a modified file is + // re-ingested. The unchanged case reads the file once to hash it; + // only a new/changed file is read again to import. String sourceType = switch (ext) { case "pdf" -> "pdf"; case "docx", "doc" -> "docx"; @@ -181,8 +176,13 @@ public class WikiDirectoryScanService { case "html", "htm" -> "html"; default -> "text"; }; - rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file)); - added++; + boolean freshBinary = rawService.ingestBinaryFileFromScan( + kbId, fileName, sourceType, absolutePath, Files.size(file)); + if (freshBinary) { + added++; + } else { + skipped++; + } } catch (Exception e) { errors.add("Failed to import: " + file.getFileName() + " (" + e.getMessage() + ")"); diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index 087ff385..6f3b8d4a 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -115,6 +115,37 @@ public class WikiRawMaterialService { return sameContent == null; } + /** + * Import a binary file discovered by a directory scan, detecting content + * changes by hashing the bytes: unchanged content (a raw with the same hash + * exists) is skipped, while changed content is re-ingested via + * {@link #addFile}. The unchanged case reads the file once; only a + * new/changed file is read again by addFile. + * + * @return {@code true} when newly ingested, {@code false} when unchanged + */ + public boolean ingestBinaryFileFromScan(Long kbId, String title, String sourceType, + String absolutePath, long fileSize) { + String hash = null; + try { + hash = computeHashOfBytes(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(absolutePath))); + } catch (Exception e) { + log.warn("[Wiki] Could not hash file for change detection: {}", e.getMessage()); + } + if (hash != null) { + WikiRawMaterialEntity sameContent = rawMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiRawMaterialEntity::getKbId, kbId) + .eq(WikiRawMaterialEntity::getContentHash, hash) + .last("LIMIT 1")); + if (sameContent != null) { + return false; // unchanged — addFile not called, avoids a second read + } + } + addFile(kbId, title, sourceType, absolutePath, fileSize); + return true; + } + /** * Record the originating file path on a raw material via a partial update, * so a later directory re-scan can dedup it by source path. Used for diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java index 86173ee0..d669117b 100644 --- a/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java @@ -87,6 +87,21 @@ class WikiSourceWatcherServiceE2ETest { "the escaping symlink should be reported as skipped"); } + @Test + void modifiedBinaryFile_isReingested(@TempDir Path sourceDir) throws java.io.IOException { + Path pdf = sourceDir.resolve("doc.pdf"); + Files.write(pdf, "PDF-VERSION-ONE-bytes".getBytes()); + long kb = SEQ.incrementAndGet(); + + assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added()); + // Unchanged binary re-scan ingests nothing. + assertEquals(0, scanService.scanDirectory(kb, sourceDir.toString()).added()); + // Changed bytes -> different content hash -> re-ingested. + Files.write(pdf, "PDF-VERSION-TWO-different-bytes".getBytes()); + assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added(), + "a modified binary file must be re-ingested"); + } + @Test void kbsWithoutSourceDirectory_areSkipped() { // A KB with no source directory must not cause errors in the cycle.