opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(wiki): re-ingest modified binary files via content-hash detection

Browse Source
This commit is contained in:
matevip 2026-05-31 07:58:10 +08:00
parent f0cbfc913e
commit aac04cdfa4
3 changed files with 56 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiDirectoryScanService.java
View File

@ -165,14 +165,9 @@ public class WikiDirectoryScanService {
continue;
}
// Binary files: dedup by source path. Re-ingest on content change
// is not detected here (hashing large binaries each scan is
// expensive) — modified binaries should be re-uploaded explicitly.
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
if (existing != null) {
skipped++;
continue;
}
// Binary files: dedup by content hash too, so a modified file is
// re-ingested. The unchanged case reads the file once to hash it;
// only a new/changed file is read again to import.
String sourceType = switch (ext) {
case "pdf" -> "pdf";
case "docx", "doc" -> "docx";
@ -181,8 +176,13 @@ public class WikiDirectoryScanService {
case "html", "htm" -> "html";
default -> "text";
};
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
added++;
boolean freshBinary = rawService.ingestBinaryFileFromScan(
kbId, fileName, sourceType, absolutePath, Files.size(file));
if (freshBinary) {
added++;
} else {
skipped++;
}
} catch (Exception e) {
errors.add("Failed to import: " + file.getFileName() + " (" + e.getMessage() + ")");

31
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java
View File

@ -115,6 +115,37 @@ public class WikiRawMaterialService {
return sameContent == null;
}
/**
* Import a binary file discovered by a directory scan, detecting content
* changes by hashing the bytes: unchanged content (a raw with the same hash
* exists) is skipped, while changed content is re-ingested via
* {@link #addFile}. The unchanged case reads the file once; only a
* new/changed file is read again by addFile.
*
* @return {@code true} when newly ingested, {@code false} when unchanged
*/
public boolean ingestBinaryFileFromScan(Long kbId, String title, String sourceType,
String absolutePath, long fileSize) {
String hash = null;
try {
hash = computeHashOfBytes(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(absolutePath)));
} catch (Exception e) {
log.warn("[Wiki] Could not hash file for change detection: {}", e.getMessage());
}
if (hash != null) {
WikiRawMaterialEntity sameContent = rawMapper.selectOne(
new LambdaQueryWrapper<WikiRawMaterialEntity>()
.eq(WikiRawMaterialEntity::getKbId, kbId)
.eq(WikiRawMaterialEntity::getContentHash, hash)
.last("LIMIT 1"));
if (sameContent != null) {
return false; // unchanged — addFile not called, avoids a second read
}
}
addFile(kbId, title, sourceType, absolutePath, fileSize);
return true;
}
/**
* Record the originating file path on a raw material via a partial update,
* so a later directory re-scan can dedup it by source path. Used for

15
mateclaw-server/src/test/java/vip/mate/wiki/service/WikiSourceWatcherServiceE2ETest.java
View File

@ -87,6 +87,21 @@ class WikiSourceWatcherServiceE2ETest {
"the escaping symlink should be reported as skipped");
}
@Test
void modifiedBinaryFile_isReingested(@TempDir Path sourceDir) throws java.io.IOException {
Path pdf = sourceDir.resolve("doc.pdf");
Files.write(pdf, "PDF-VERSION-ONE-bytes".getBytes());
long kb = SEQ.incrementAndGet();
assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added());
// Unchanged binary re-scan ingests nothing.
assertEquals(0, scanService.scanDirectory(kb, sourceDir.toString()).added());
// Changed bytes -> different content hash -> re-ingested.
Files.write(pdf, "PDF-VERSION-TWO-different-bytes".getBytes());
assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added(),
"a modified binary file must be re-ingested");
}
@Test
void kbsWithoutSourceDirectory_areSkipped() {
// A KB with no source directory must not cause errors in the cycle.