mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 11:58:34 +08:00
fix(wiki): re-ingest modified binary files via content-hash detection
This commit is contained in:
parent
f0cbfc913e
commit
aac04cdfa4
@ -165,14 +165,9 @@ public class WikiDirectoryScanService {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Binary files: dedup by source path. Re-ingest on content change
|
// Binary files: dedup by content hash too, so a modified file is
|
||||||
// is not detected here (hashing large binaries each scan is
|
// re-ingested. The unchanged case reads the file once to hash it;
|
||||||
// expensive) — modified binaries should be re-uploaded explicitly.
|
// only a new/changed file is read again to import.
|
||||||
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
|
|
||||||
if (existing != null) {
|
|
||||||
skipped++;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
String sourceType = switch (ext) {
|
String sourceType = switch (ext) {
|
||||||
case "pdf" -> "pdf";
|
case "pdf" -> "pdf";
|
||||||
case "docx", "doc" -> "docx";
|
case "docx", "doc" -> "docx";
|
||||||
@ -181,8 +176,13 @@ public class WikiDirectoryScanService {
|
|||||||
case "html", "htm" -> "html";
|
case "html", "htm" -> "html";
|
||||||
default -> "text";
|
default -> "text";
|
||||||
};
|
};
|
||||||
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
|
boolean freshBinary = rawService.ingestBinaryFileFromScan(
|
||||||
added++;
|
kbId, fileName, sourceType, absolutePath, Files.size(file));
|
||||||
|
if (freshBinary) {
|
||||||
|
added++;
|
||||||
|
} else {
|
||||||
|
skipped++;
|
||||||
|
}
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
errors.add("Failed to import: " + file.getFileName() + " (" + e.getMessage() + ")");
|
errors.add("Failed to import: " + file.getFileName() + " (" + e.getMessage() + ")");
|
||||||
|
|||||||
@ -115,6 +115,37 @@ public class WikiRawMaterialService {
|
|||||||
return sameContent == null;
|
return sameContent == null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Import a binary file discovered by a directory scan, detecting content
|
||||||
|
* changes by hashing the bytes: unchanged content (a raw with the same hash
|
||||||
|
* exists) is skipped, while changed content is re-ingested via
|
||||||
|
* {@link #addFile}. The unchanged case reads the file once; only a
|
||||||
|
* new/changed file is read again by addFile.
|
||||||
|
*
|
||||||
|
* @return {@code true} when newly ingested, {@code false} when unchanged
|
||||||
|
*/
|
||||||
|
public boolean ingestBinaryFileFromScan(Long kbId, String title, String sourceType,
|
||||||
|
String absolutePath, long fileSize) {
|
||||||
|
String hash = null;
|
||||||
|
try {
|
||||||
|
hash = computeHashOfBytes(java.nio.file.Files.readAllBytes(java.nio.file.Paths.get(absolutePath)));
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("[Wiki] Could not hash file for change detection: {}", e.getMessage());
|
||||||
|
}
|
||||||
|
if (hash != null) {
|
||||||
|
WikiRawMaterialEntity sameContent = rawMapper.selectOne(
|
||||||
|
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||||
|
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||||
|
.eq(WikiRawMaterialEntity::getContentHash, hash)
|
||||||
|
.last("LIMIT 1"));
|
||||||
|
if (sameContent != null) {
|
||||||
|
return false; // unchanged — addFile not called, avoids a second read
|
||||||
|
}
|
||||||
|
}
|
||||||
|
addFile(kbId, title, sourceType, absolutePath, fileSize);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record the originating file path on a raw material via a partial update,
|
* Record the originating file path on a raw material via a partial update,
|
||||||
* so a later directory re-scan can dedup it by source path. Used for
|
* so a later directory re-scan can dedup it by source path. Used for
|
||||||
|
|||||||
@ -87,6 +87,21 @@ class WikiSourceWatcherServiceE2ETest {
|
|||||||
"the escaping symlink should be reported as skipped");
|
"the escaping symlink should be reported as skipped");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void modifiedBinaryFile_isReingested(@TempDir Path sourceDir) throws java.io.IOException {
|
||||||
|
Path pdf = sourceDir.resolve("doc.pdf");
|
||||||
|
Files.write(pdf, "PDF-VERSION-ONE-bytes".getBytes());
|
||||||
|
long kb = SEQ.incrementAndGet();
|
||||||
|
|
||||||
|
assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added());
|
||||||
|
// Unchanged binary re-scan ingests nothing.
|
||||||
|
assertEquals(0, scanService.scanDirectory(kb, sourceDir.toString()).added());
|
||||||
|
// Changed bytes -> different content hash -> re-ingested.
|
||||||
|
Files.write(pdf, "PDF-VERSION-TWO-different-bytes".getBytes());
|
||||||
|
assertEquals(1, scanService.scanDirectory(kb, sourceDir.toString()).added(),
|
||||||
|
"a modified binary file must be re-ingested");
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void kbsWithoutSourceDirectory_areSkipped() {
|
void kbsWithoutSourceDirectory_areSkipped() {
|
||||||
// A KB with no source directory must not cause errors in the cycle.
|
// A KB with no source directory must not cause errors in the cycle.
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user