mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(wiki): re-ingest modified text files via content-hash change detection
This commit is contained in:
parent
4c56df2ed3
commit
f3a335f4f6
@ -137,32 +137,36 @@ public class WikiDirectoryScanService {
|
||||
String fileName = file.getFileName().toString();
|
||||
String ext = getExtension(fileName);
|
||||
|
||||
// 基于 sourcePath 去重
|
||||
if (TEXT_EXTENSIONS.contains(ext)) {
|
||||
// Text files: dedup by content hash, so an unchanged file is
|
||||
// skipped while a modified file (new hash) is re-ingested.
|
||||
String content = Files.readString(file, StandardCharsets.UTF_8);
|
||||
boolean fresh = rawService.ingestTextFileFromScan(kbId, fileName, absolutePath, content);
|
||||
if (fresh) {
|
||||
added++;
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Binary files: dedup by source path. Re-ingest on content change
|
||||
// is not detected here (hashing large binaries each scan is
|
||||
// expensive) — modified binaries should be re-uploaded explicitly.
|
||||
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
|
||||
if (existing != null) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TEXT_EXTENSIONS.contains(ext)) {
|
||||
// 文本文件:读取内容;记录 source path 以便重复扫描去重
|
||||
String content = Files.readString(file, StandardCharsets.UTF_8);
|
||||
WikiRawMaterialEntity textRaw = rawService.addText(kbId, fileName, content);
|
||||
if (textRaw != null) {
|
||||
rawService.updateSourcePath(textRaw.getId(), absolutePath);
|
||||
}
|
||||
} else {
|
||||
// 二进制文件:直接引用原始路径,不复制
|
||||
String sourceType = switch (ext) {
|
||||
case "pdf" -> "pdf";
|
||||
case "docx", "doc" -> "docx";
|
||||
case "pptx", "ppt" -> "pptx";
|
||||
case "xlsx", "xls" -> "xlsx";
|
||||
case "html", "htm" -> "html";
|
||||
default -> "text";
|
||||
};
|
||||
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
|
||||
}
|
||||
String sourceType = switch (ext) {
|
||||
case "pdf" -> "pdf";
|
||||
case "docx", "doc" -> "docx";
|
||||
case "pptx", "ppt" -> "pptx";
|
||||
case "xlsx", "xls" -> "xlsx";
|
||||
case "html", "htm" -> "html";
|
||||
default -> "text";
|
||||
};
|
||||
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
|
||||
added++;
|
||||
|
||||
} catch (Exception e) {
|
||||
|
||||
@ -86,6 +86,32 @@ public class WikiRawMaterialService {
|
||||
.eq(WikiRawMaterialEntity::getSourcePath, sourcePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a text file discovered by a directory scan, detecting content
|
||||
* changes by hash: unchanged content (a raw with the same hash already
|
||||
* exists) is a no-op, while changed content creates a new raw and triggers
|
||||
* processing — so a modified file is re-ingested rather than silently
|
||||
* skipped. The originating path is recorded for diagnostics.
|
||||
*
|
||||
* @return {@code true} when the file was newly ingested (new or changed
|
||||
* content), {@code false} when skipped as unchanged
|
||||
*/
|
||||
public boolean ingestTextFileFromScan(Long kbId, String fileName, String absolutePath, String content) {
|
||||
String hash = computeHash(content);
|
||||
WikiRawMaterialEntity sameContent = rawMapper.selectOne(
|
||||
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||
.eq(WikiRawMaterialEntity::getContentHash, hash)
|
||||
.last("LIMIT 1"));
|
||||
// addText dedups internally by hash, so this reuses sameContent when
|
||||
// unchanged and inserts + triggers processing when the content differs.
|
||||
WikiRawMaterialEntity raw = addText(kbId, fileName, content);
|
||||
if (raw != null) {
|
||||
updateSourcePath(raw.getId(), absolutePath);
|
||||
}
|
||||
return sameContent == null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Record the originating file path on a raw material via a partial update,
|
||||
* so a later directory re-scan can dedup it by source path. Used for
|
||||
|
||||
@ -54,6 +54,14 @@ class WikiSourceWatcherServiceE2ETest {
|
||||
Files.writeString(sourceDir.resolve("note-c.md"), "# Note C\n\ncontent c");
|
||||
int secondAdded = watcherService.runScanCycle();
|
||||
assertEquals(1, secondAdded, "only the newly added file should ingest");
|
||||
|
||||
// Re-scanning with no changes ingests nothing.
|
||||
assertEquals(0, watcherService.runScanCycle(), "unchanged files must not re-ingest");
|
||||
|
||||
// Modifying an existing file's content re-ingests it (content hash changed).
|
||||
Files.writeString(sourceDir.resolve("note-a.md"), "# Note A\n\nEDITED content a");
|
||||
int afterEdit = watcherService.runScanCycle();
|
||||
assertEquals(1, afterEdit, "a modified file must be re-ingested");
|
||||
}
|
||||
|
||||
@Test
|
||||
|
||||
Loading…
Reference in New Issue
Block a user