+ * This carries everything {@link vip.mate.wiki.service.WikiChunkService} needs + * to persist a chunk row, including the new metadata columns added in V39: + * {@code page_number}, {@code token_count}, {@code header_breadcrumb}, + * {@code source_section}. The structural fields are nullable because not every + * source format yields each piece of metadata (e.g. plain text has no page + * number; HTML has no slide id). + *
+ * No production callers in PR-1a — wiring lands in PR-1c. The type is added + * here so the persistence overload in PR-1a can be unit-tested independently. + */ +public record WikiChunkDraft( + String content, + int startOffset, + int endOffset, + Integer pageNumber, + Integer tokenCount, + String headerBreadcrumb, + String sourceSection +) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiChunkTokenBackfillJob.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiChunkTokenBackfillJob.java new file mode 100644 index 00000000..de3f556a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiChunkTokenBackfillJob.java @@ -0,0 +1,77 @@ +package vip.mate.wiki.job; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.repository.WikiChunkMapper; + +import java.util.List; + +/** + * RFC-051 PR-1a: low-frequency backfill job that fills the new + * {@code token_count} column on existing chunks. Runs in the background after + * the V39 migration adds the column with all-NULL data. + *
+ * Strategy: + *
+ * stepModels[step] -> wikiDefaultModelId -> system default + *+ * + * The frontend has long written {@code wikiDefaultModelId} into the KB config + * JSON, but no Java code consumed it before this strategy existed. + */ +@Slf4j +@Component +@Order(2) +@RequiredArgsConstructor +public class KbDefaultModelStrategy implements WikiStepModelStrategy { + + private final ObjectMapper objectMapper; + + @Override + public boolean supports(WikiJobStep step) { return true; } + + @Override + public Long selectModelId(WikiProcessingJobEntity job, WikiKnowledgeBaseEntity kb, WikiJobStep step) { + if (kb == null || kb.getConfigContent() == null) return null; + try { + WikiKbConfig config = objectMapper.readValue(kb.getConfigContent(), WikiKbConfig.class); + return config.getWikiDefaultModelId(); + } catch (Exception e) { + log.debug("[KbDefaultModelStrategy] Failed to parse KB config: {}", e.getMessage()); + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java index 88404096..aef31ce6 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java @@ -51,6 +51,18 @@ public class WikiChunkEntity { /** RFC-011：生成该 embedding 的模型名称（切模型时需全量重嵌） */ private String embeddingModel; + /** RFC-051: source page number (PDF/PPTX) when known; null otherwise. */ + private Integer pageNumber; + + /** RFC-051: estimated token count; null until populated by chunker or backfill job. */ + private Integer tokenCount; + + /** RFC-051: header path leading to this chunk, e.g. "Intro / Setup / Linux". */ + private String headerBreadcrumb; + + /** RFC-051: short identifier of the source section (slide id, sheet name, heading). */ + private String sourceSection; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiChunkService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiChunkService.java index a29af317..c1286145 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiChunkService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiChunkService.java @@ -5,6 +5,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import vip.mate.wiki.dto.WikiChunkDraft; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.repository.WikiChunkMapper; @@ -55,6 +56,125 @@ public class WikiChunkService { return reconcile(kbId, rawId, chunks, offsets, existing); } + /** + * RFC-051 PR-1a: persist chunks along with their structural metadata + * (page number, token count, header breadcrumb, source section). + *
+ * Behaves like {@link #persistChunks(Long, Long, List, List)} — full insert
+ * when no existing chunks for the raw, otherwise hash-based reconcile so
+ * unchanged chunks keep their embeddings. Reconcile updates the metadata
+ * columns on retained rows so re-running ingest with a richer normalizer
+ * can fill in fields that were null on the previous pass.
+ *
+ * @param drafts chunks-to-persist with metadata (ordered, ordinal == index)
+ * @return persisted chunk IDs (same order as {@code drafts})
+ */
+ @Transactional
+ public List
+ * RFC-051: this method no longer touches {@code contentHash}. The previous
+ * behavior overwrote the original-upload hash with an extracted-text hash,
+ * which broke upload dedup (re-uploading the same file would compute a hash
+ * over raw bytes but find a row whose hash had been replaced with extracted
+ * text). The {@code contentHash} field is now an immutable identity for the
+ * uploaded artifact; downstream short-circuiting uses {@code lastProcessedHash}.
+ */
@Transactional
- public void updateExtractedText(Long id, String extractedText, String contentHash) {
+ public void updateExtractedText(Long id, String extractedText) {
WikiRawMaterialEntity entity = rawMapper.selectById(id);
if (entity == null) return;
entity.setExtractedText(extractedText);
- entity.setContentHash(contentHash);
rawMapper.updateById(entity);
}
@@ -316,8 +328,8 @@ public class WikiRawMaterialService {
log.warn("[Wiki] Extracted text truncated at {} chars for: {} (full document may be larger)",
text.length(), entity.getSourcePath());
} else {
- // 完整提取结果：缓存以避免重复提取
- updateExtractedText(entity.getId(), text, computeHash(text));
+ // Full extraction: cache to avoid re-extracting on subsequent calls.
+ updateExtractedText(entity.getId(), text);
}
log.info("[Wiki] Extracted text from {}: {} chars, method={}, truncated={}",
entity.getSourcePath(), text.length(), json.getStr("method"), truncated);
@@ -401,4 +413,19 @@ public class WikiRawMaterialService {
return null;
}
}
+
+ /**
+ * SHA-256 over raw bytes. Used for file uploads so that PDF/Office binaries
+ * produce a stable identity hash regardless of UTF-8 round-tripping.
+ */
+ private String computeHashOfBytes(byte[] bytes) {
+ try {
+ MessageDigest digest = MessageDigest.getInstance("SHA-256");
+ byte[] hash = digest.digest(bytes);
+ return HexFormat.of().formatHex(hash);
+ } catch (Exception e) {
+ log.warn("[Wiki] Failed to compute byte hash: {}", e.getMessage());
+ return null;
+ }
+ }
}
diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V39__rfc051_chunk_metadata.sql b/mateclaw-server/src/main/resources/db/migration/h2/V39__rfc051_chunk_metadata.sql
new file mode 100644
index 00000000..d6aaacbe
--- /dev/null
+++ b/mateclaw-server/src/main/resources/db/migration/h2/V39__rfc051_chunk_metadata.sql
@@ -0,0 +1,7 @@
+-- V39: RFC-051 PR-1a — chunk structural metadata.
+-- All columns are nullable; NULL means "unknown" for legacy chunks. token_count
+-- backfill happens asynchronously via WikiChunkTokenBackfillJob.
+ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS page_number INT DEFAULT NULL;
+ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS token_count INT DEFAULT NULL;
+ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS header_breadcrumb VARCHAR(1024) DEFAULT NULL;
+ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS source_section VARCHAR(512) DEFAULT NULL;
diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V39__rfc051_chunk_metadata.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V39__rfc051_chunk_metadata.sql
new file mode 100644
index 00000000..5730847d
--- /dev/null
+++ b/mateclaw-server/src/main/resources/db/migration/mysql/V39__rfc051_chunk_metadata.sql
@@ -0,0 +1,17 @@
+-- V39: RFC-051 PR-1a — chunk structural metadata.
+-- MySQL lacks `ADD COLUMN IF NOT EXISTS`; use INFORMATION_SCHEMA guard instead.
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'page_number');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN page_number INT DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'token_count');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN token_count INT DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'header_breadcrumb');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN header_breadcrumb VARCHAR(1024) DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'source_section');
+SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN source_section VARCHAR(512) DEFAULT NULL', 'SELECT 1');
+PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt;