mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-16 04:18:17 +08:00
fix(wiki): re-ingest modified text files via content-hash change detection
This commit is contained in:
parent
4c56df2ed3
commit
f3a335f4f6
@ -137,32 +137,36 @@ public class WikiDirectoryScanService {
|
|||||||
String fileName = file.getFileName().toString();
|
String fileName = file.getFileName().toString();
|
||||||
String ext = getExtension(fileName);
|
String ext = getExtension(fileName);
|
||||||
|
|
||||||
// 基于 sourcePath 去重
|
if (TEXT_EXTENSIONS.contains(ext)) {
|
||||||
|
// Text files: dedup by content hash, so an unchanged file is
|
||||||
|
// skipped while a modified file (new hash) is re-ingested.
|
||||||
|
String content = Files.readString(file, StandardCharsets.UTF_8);
|
||||||
|
boolean fresh = rawService.ingestTextFileFromScan(kbId, fileName, absolutePath, content);
|
||||||
|
if (fresh) {
|
||||||
|
added++;
|
||||||
|
} else {
|
||||||
|
skipped++;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Binary files: dedup by source path. Re-ingest on content change
|
||||||
|
// is not detected here (hashing large binaries each scan is
|
||||||
|
// expensive) — modified binaries should be re-uploaded explicitly.
|
||||||
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
|
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
|
||||||
if (existing != null) {
|
if (existing != null) {
|
||||||
skipped++;
|
skipped++;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
String sourceType = switch (ext) {
|
||||||
if (TEXT_EXTENSIONS.contains(ext)) {
|
case "pdf" -> "pdf";
|
||||||
// 文本文件:读取内容;记录 source path 以便重复扫描去重
|
case "docx", "doc" -> "docx";
|
||||||
String content = Files.readString(file, StandardCharsets.UTF_8);
|
case "pptx", "ppt" -> "pptx";
|
||||||
WikiRawMaterialEntity textRaw = rawService.addText(kbId, fileName, content);
|
case "xlsx", "xls" -> "xlsx";
|
||||||
if (textRaw != null) {
|
case "html", "htm" -> "html";
|
||||||
rawService.updateSourcePath(textRaw.getId(), absolutePath);
|
default -> "text";
|
||||||
}
|
};
|
||||||
} else {
|
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
|
||||||
// 二进制文件:直接引用原始路径,不复制
|
|
||||||
String sourceType = switch (ext) {
|
|
||||||
case "pdf" -> "pdf";
|
|
||||||
case "docx", "doc" -> "docx";
|
|
||||||
case "pptx", "ppt" -> "pptx";
|
|
||||||
case "xlsx", "xls" -> "xlsx";
|
|
||||||
case "html", "htm" -> "html";
|
|
||||||
default -> "text";
|
|
||||||
};
|
|
||||||
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
|
|
||||||
}
|
|
||||||
added++;
|
added++;
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
@ -86,6 +86,32 @@ public class WikiRawMaterialService {
|
|||||||
.eq(WikiRawMaterialEntity::getSourcePath, sourcePath));
|
.eq(WikiRawMaterialEntity::getSourcePath, sourcePath));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Import a text file discovered by a directory scan, detecting content
|
||||||
|
* changes by hash: unchanged content (a raw with the same hash already
|
||||||
|
* exists) is a no-op, while changed content creates a new raw and triggers
|
||||||
|
* processing — so a modified file is re-ingested rather than silently
|
||||||
|
* skipped. The originating path is recorded for diagnostics.
|
||||||
|
*
|
||||||
|
* @return {@code true} when the file was newly ingested (new or changed
|
||||||
|
* content), {@code false} when skipped as unchanged
|
||||||
|
*/
|
||||||
|
public boolean ingestTextFileFromScan(Long kbId, String fileName, String absolutePath, String content) {
|
||||||
|
String hash = computeHash(content);
|
||||||
|
WikiRawMaterialEntity sameContent = rawMapper.selectOne(
|
||||||
|
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||||
|
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||||
|
.eq(WikiRawMaterialEntity::getContentHash, hash)
|
||||||
|
.last("LIMIT 1"));
|
||||||
|
// addText dedups internally by hash, so this reuses sameContent when
|
||||||
|
// unchanged and inserts + triggers processing when the content differs.
|
||||||
|
WikiRawMaterialEntity raw = addText(kbId, fileName, content);
|
||||||
|
if (raw != null) {
|
||||||
|
updateSourcePath(raw.getId(), absolutePath);
|
||||||
|
}
|
||||||
|
return sameContent == null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Record the originating file path on a raw material via a partial update,
|
* Record the originating file path on a raw material via a partial update,
|
||||||
* so a later directory re-scan can dedup it by source path. Used for
|
* so a later directory re-scan can dedup it by source path. Used for
|
||||||
|
|||||||
@ -54,6 +54,14 @@ class WikiSourceWatcherServiceE2ETest {
|
|||||||
Files.writeString(sourceDir.resolve("note-c.md"), "# Note C\n\ncontent c");
|
Files.writeString(sourceDir.resolve("note-c.md"), "# Note C\n\ncontent c");
|
||||||
int secondAdded = watcherService.runScanCycle();
|
int secondAdded = watcherService.runScanCycle();
|
||||||
assertEquals(1, secondAdded, "only the newly added file should ingest");
|
assertEquals(1, secondAdded, "only the newly added file should ingest");
|
||||||
|
|
||||||
|
// Re-scanning with no changes ingests nothing.
|
||||||
|
assertEquals(0, watcherService.runScanCycle(), "unchanged files must not re-ingest");
|
||||||
|
|
||||||
|
// Modifying an existing file's content re-ingests it (content hash changed).
|
||||||
|
Files.writeString(sourceDir.resolve("note-a.md"), "# Note A\n\nEDITED content a");
|
||||||
|
int afterEdit = watcherService.runScanCycle();
|
||||||
|
assertEquals(1, afterEdit, "a modified file must be re-ingested");
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user