fix(wiki): re-ingest modified text files via content-hash change detection

This commit is contained in:
matevip 2026-05-31 07:57:50 +08:00
parent 4c56df2ed3
commit f3a335f4f6
3 changed files with 59 additions and 21 deletions

View File

@ -137,32 +137,36 @@ public class WikiDirectoryScanService {
String fileName = file.getFileName().toString();
String ext = getExtension(fileName);
// 基于 sourcePath 去重
if (TEXT_EXTENSIONS.contains(ext)) {
// Text files: dedup by content hash, so an unchanged file is
// skipped while a modified file (new hash) is re-ingested.
String content = Files.readString(file, StandardCharsets.UTF_8);
boolean fresh = rawService.ingestTextFileFromScan(kbId, fileName, absolutePath, content);
if (fresh) {
added++;
} else {
skipped++;
}
continue;
}
// Binary files: dedup by source path. Re-ingest on content change
// is not detected here (hashing large binaries each scan is
// expensive) modified binaries should be re-uploaded explicitly.
WikiRawMaterialEntity existing = rawService.findBySourcePath(kbId, absolutePath);
if (existing != null) {
skipped++;
continue;
}
if (TEXT_EXTENSIONS.contains(ext)) {
// 文本文件读取内容记录 source path 以便重复扫描去重
String content = Files.readString(file, StandardCharsets.UTF_8);
WikiRawMaterialEntity textRaw = rawService.addText(kbId, fileName, content);
if (textRaw != null) {
rawService.updateSourcePath(textRaw.getId(), absolutePath);
}
} else {
// 二进制文件直接引用原始路径不复制
String sourceType = switch (ext) {
case "pdf" -> "pdf";
case "docx", "doc" -> "docx";
case "pptx", "ppt" -> "pptx";
case "xlsx", "xls" -> "xlsx";
case "html", "htm" -> "html";
default -> "text";
};
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
}
String sourceType = switch (ext) {
case "pdf" -> "pdf";
case "docx", "doc" -> "docx";
case "pptx", "ppt" -> "pptx";
case "xlsx", "xls" -> "xlsx";
case "html", "htm" -> "html";
default -> "text";
};
rawService.addFile(kbId, fileName, sourceType, absolutePath, Files.size(file));
added++;
} catch (Exception e) {

View File

@ -86,6 +86,32 @@ public class WikiRawMaterialService {
.eq(WikiRawMaterialEntity::getSourcePath, sourcePath));
}
/**
* Import a text file discovered by a directory scan, detecting content
* changes by hash: unchanged content (a raw with the same hash already
* exists) is a no-op, while changed content creates a new raw and triggers
* processing so a modified file is re-ingested rather than silently
* skipped. The originating path is recorded for diagnostics.
*
* @return {@code true} when the file was newly ingested (new or changed
* content), {@code false} when skipped as unchanged
*/
public boolean ingestTextFileFromScan(Long kbId, String fileName, String absolutePath, String content) {
String hash = computeHash(content);
WikiRawMaterialEntity sameContent = rawMapper.selectOne(
new LambdaQueryWrapper<WikiRawMaterialEntity>()
.eq(WikiRawMaterialEntity::getKbId, kbId)
.eq(WikiRawMaterialEntity::getContentHash, hash)
.last("LIMIT 1"));
// addText dedups internally by hash, so this reuses sameContent when
// unchanged and inserts + triggers processing when the content differs.
WikiRawMaterialEntity raw = addText(kbId, fileName, content);
if (raw != null) {
updateSourcePath(raw.getId(), absolutePath);
}
return sameContent == null;
}
/**
* Record the originating file path on a raw material via a partial update,
* so a later directory re-scan can dedup it by source path. Used for

View File

@ -54,6 +54,14 @@ class WikiSourceWatcherServiceE2ETest {
Files.writeString(sourceDir.resolve("note-c.md"), "# Note C\n\ncontent c");
int secondAdded = watcherService.runScanCycle();
assertEquals(1, secondAdded, "only the newly added file should ingest");
// Re-scanning with no changes ingests nothing.
assertEquals(0, watcherService.runScanCycle(), "unchanged files must not re-ingest");
// Modifying an existing file's content re-ingests it (content hash changed).
Files.writeString(sourceDir.resolve("note-a.md"), "# Note A\n\nEDITED content a");
int afterEdit = watcherService.runScanCycle();
assertEquals(1, afterEdit, "a modified file must be re-ingested");
}
@Test