package vip.mate.wiki.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import vip.mate.wiki.dto.WikiChunkDraft; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.repository.WikiChunkMapper; import java.nio.charset.StandardCharsets; import java.security.MessageDigest; import java.util.*; /** * Wiki chunk 服务 *

* RFC-013 最小切片:chunk 持久化 + hash 级增量对账。 *

* * @author MateClaw Team */ @Slf4j @Service @RequiredArgsConstructor public class WikiChunkService { private final WikiChunkMapper chunkMapper; /** * 将文本切片列表持久化为 chunk 记录。 *

* 如果该 rawId 已有 chunk 记录,走 {@link #reconcile} 增量对账;否则全量插入。 * * @param kbId 知识库 ID * @param rawId 原始材料 ID * @param chunks 切分后的文本列表(有序) * @param offsets 每个 chunk 对应的 [startOffset, endOffset] 数组 * @return 持久化后的 chunk ID 列表(与 chunks 同序) */ @Transactional public List persistChunks(Long kbId, Long rawId, List chunks, List offsets) { List existing = listByRawId(rawId); if (existing.isEmpty()) { // 全量插入 return insertAll(kbId, rawId, chunks, offsets); } // 增量对账 return reconcile(kbId, rawId, chunks, offsets, existing); } /** * RFC-051 PR-1a: persist chunks along with their structural metadata * (page number, token count, header breadcrumb, source section). *

* Behaves like {@link #persistChunks(Long, Long, List, List)} — full insert * when no existing chunks for the raw, otherwise hash-based reconcile so * unchanged chunks keep their embeddings. Reconcile updates the metadata * columns on retained rows so re-running ingest with a richer normalizer * can fill in fields that were null on the previous pass. * * @param drafts chunks-to-persist with metadata (ordered, ordinal == index) * @return persisted chunk IDs (same order as {@code drafts}) */ @Transactional public List persistChunks(Long kbId, Long rawId, List drafts) { List existing = listByRawId(rawId); if (existing.isEmpty()) { return insertAllDrafts(kbId, rawId, drafts); } return reconcileDrafts(kbId, rawId, drafts, existing); } private List insertAllDrafts(Long kbId, Long rawId, List drafts) { List ids = new ArrayList<>(drafts.size()); for (int i = 0; i < drafts.size(); i++) { WikiChunkDraft draft = drafts.get(i); String hash = computeHash(draft.content()); WikiChunkEntity entity = buildEntityFromDraft(kbId, rawId, i, draft, hash); chunkMapper.insert(entity); ids.add(entity.getId()); } log.info("[WikiChunk] Inserted {} drafts for raw={}", drafts.size(), rawId); return ids; } private List reconcileDrafts(Long kbId, Long rawId, List drafts, List existing) { Map oldByOrdinal = new HashMap<>(); for (WikiChunkEntity e : existing) { oldByOrdinal.put(e.getOrdinal(), e); } List resultIds = new ArrayList<>(drafts.size()); Set retainedIds = new HashSet<>(); int retained = 0, rebuilt = 0; for (int i = 0; i < drafts.size(); i++) { WikiChunkDraft draft = drafts.get(i); String hash = computeHash(draft.content()); WikiChunkEntity old = oldByOrdinal.get(i); if (old != null && hash.equals(old.getContentHash())) { // Same content: keep existing row (and its embedding). Refresh offsets and // metadata so a re-run with a smarter normalizer can fill gaps. boolean changed = false; if (!Objects.equals(old.getStartOffset(), draft.startOffset())) { old.setStartOffset(draft.startOffset()); changed = true; } if (!Objects.equals(old.getEndOffset(), draft.endOffset())) { old.setEndOffset(draft.endOffset()); changed = true; } if (!Objects.equals(old.getPageNumber(), draft.pageNumber())) { old.setPageNumber(draft.pageNumber()); changed = true; } if (!Objects.equals(old.getTokenCount(), draft.tokenCount())) { old.setTokenCount(draft.tokenCount()); changed = true; } if (!Objects.equals(old.getHeaderBreadcrumb(), draft.headerBreadcrumb())) { old.setHeaderBreadcrumb(draft.headerBreadcrumb()); changed = true; } if (!Objects.equals(old.getSourceSection(), draft.sourceSection())) { old.setSourceSection(draft.sourceSection()); changed = true; } if (changed) { chunkMapper.updateById(old); } resultIds.add(old.getId()); retainedIds.add(old.getId()); retained++; } else { WikiChunkEntity entity = buildEntityFromDraft(kbId, rawId, i, draft, hash); chunkMapper.insert(entity); resultIds.add(entity.getId()); rebuilt++; } } int deleted = 0; for (WikiChunkEntity old : existing) { if (!retainedIds.contains(old.getId()) && !resultIds.contains(old.getId())) { chunkMapper.deleteById(old.getId()); deleted++; } } log.info("[WikiChunk] Reconciled drafts raw={}: retained={}, rebuilt={}, deleted={}", rawId, retained, rebuilt, deleted); return resultIds; } private WikiChunkEntity buildEntityFromDraft(Long kbId, Long rawId, int ordinal, WikiChunkDraft draft, String hash) { WikiChunkEntity entity = new WikiChunkEntity(); entity.setKbId(kbId); entity.setRawId(rawId); entity.setOrdinal(ordinal); entity.setContent(draft.content()); entity.setCharCount(draft.content().length()); entity.setStartOffset(draft.startOffset()); entity.setEndOffset(draft.endOffset()); entity.setContentHash(hash); entity.setPageNumber(draft.pageNumber()); entity.setTokenCount(draft.tokenCount()); entity.setHeaderBreadcrumb(draft.headerBreadcrumb()); entity.setSourceSection(draft.sourceSection()); return entity; } /** * 增量对账:比对 hash,保留不变的 chunk(保护未来的 embedding),重建变化的。 * * @return 对账后的 chunk ID 列表(与 chunks 同序) */ private List reconcile(Long kbId, Long rawId, List chunks, List offsets, List existing) { // 旧 chunk 按 ordinal 索引 Map oldByOrdinal = new HashMap<>(); for (WikiChunkEntity e : existing) { oldByOrdinal.put(e.getOrdinal(), e); } List resultIds = new ArrayList<>(chunks.size()); Set retainedIds = new HashSet<>(); int retained = 0, rebuilt = 0; for (int i = 0; i < chunks.size(); i++) { String text = chunks.get(i); String hash = computeHash(text); int[] offset = offsets.get(i); WikiChunkEntity old = oldByOrdinal.get(i); if (old != null && hash.equals(old.getContentHash())) { // hash 相同 → 保留(embedding 等附加数据不丢) // 但更新 offset(材料可能在其他位置变了导致偏移变化) if (!old.getStartOffset().equals(offset[0]) || !old.getEndOffset().equals(offset[1])) { old.setStartOffset(offset[0]); old.setEndOffset(offset[1]); chunkMapper.updateById(old); } resultIds.add(old.getId()); retainedIds.add(old.getId()); retained++; } else { // hash 不同或 ordinal 超出旧范围 → 新建 WikiChunkEntity entity = buildEntity(kbId, rawId, i, text, hash, offset); chunkMapper.insert(entity); resultIds.add(entity.getId()); rebuilt++; } } // 删除多余的旧 chunk(数量缩减的情况) int deleted = 0; for (WikiChunkEntity old : existing) { if (!retainedIds.contains(old.getId()) && !resultIds.contains(old.getId())) { chunkMapper.deleteById(old.getId()); deleted++; } } log.info("[WikiChunk] Reconciled raw={}: retained={}, rebuilt={}, deleted={}", rawId, retained, rebuilt, deleted); return resultIds; } /** * 全量插入 */ private List insertAll(Long kbId, Long rawId, List chunks, List offsets) { List ids = new ArrayList<>(chunks.size()); for (int i = 0; i < chunks.size(); i++) { String text = chunks.get(i); String hash = computeHash(text); int[] offset = offsets.get(i); WikiChunkEntity entity = buildEntity(kbId, rawId, i, text, hash, offset); chunkMapper.insert(entity); ids.add(entity.getId()); } log.info("[WikiChunk] Inserted {} chunks for raw={}", chunks.size(), rawId); return ids; } public List listByRawId(Long rawId) { return chunkMapper.selectList( new LambdaQueryWrapper() .eq(WikiChunkEntity::getRawId, rawId) .orderByAsc(WikiChunkEntity::getOrdinal)); } public List listByKbId(Long kbId) { return chunkMapper.selectList( new LambdaQueryWrapper() .eq(WikiChunkEntity::getKbId, kbId) .orderByAsc(WikiChunkEntity::getRawId) .orderByAsc(WikiChunkEntity::getOrdinal)); } @Transactional public void deleteByRawId(Long rawId) { int deleted = chunkMapper.delete( new LambdaQueryWrapper() .eq(WikiChunkEntity::getRawId, rawId)); if (deleted > 0) { log.info("[WikiChunk] Deleted {} chunks for raw={}", deleted, rawId); } } @Transactional public void deleteByKbId(Long kbId) { int deleted = chunkMapper.delete( new LambdaQueryWrapper() .eq(WikiChunkEntity::getKbId, kbId)); if (deleted > 0) { log.info("[WikiChunk] Deleted {} chunks for kbId={}", deleted, kbId); } } // ==================== Helpers ==================== private WikiChunkEntity buildEntity(Long kbId, Long rawId, int ordinal, String text, String hash, int[] offset) { WikiChunkEntity entity = new WikiChunkEntity(); entity.setKbId(kbId); entity.setRawId(rawId); entity.setOrdinal(ordinal); entity.setContent(text); entity.setCharCount(text.length()); entity.setStartOffset(offset[0]); entity.setEndOffset(offset[1]); entity.setContentHash(hash); return entity; } private String computeHash(String content) { try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); byte[] hash = digest.digest(content.getBytes(StandardCharsets.UTF_8)); return HexFormat.of().formatHex(hash); } catch (Exception e) { log.warn("[WikiChunk] Hash computation failed: {}", e.getMessage()); return "HASH_ERROR_" + content.length(); } } }