diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index 63d7d46e..adcabd47 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -201,7 +201,7 @@ public class WikiController { item.put("title", raw.getTitle()); item.put("sourceType", raw.getSourceType()); item.put("processingStatus", raw.getProcessingStatus()); - item.put("processingDetail", raw.getProcessingDetail()); + item.put("errorMessage", raw.getErrorMessage()); item.put("progressPhase", raw.getProgressPhase()); item.put("progressDone", raw.getProgressDone()); item.put("progressTotal", raw.getProgressTotal()); diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 28fb2681..9eea8336 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -84,6 +84,15 @@ public class WikiProcessingService { * slug 注册为 winner,后到的 chunk 看到 winner 后会把内容写入 winner 对应的 page。 */ final ConcurrentHashMap slugClaims = new ConcurrentHashMap<>(); + /** + * Per-run merge dedup set: slugs that have already been successfully merged during + * this raw material processing run. Prevents the same page from being merged N times + * (once per chunk) when the document repeatedly references the same concept. + *

+ * Merge is skipped (not just decremented from count) when a slug is already present. + * Uses ConcurrentHashMap as a concurrent set via putIfAbsent. + */ + final ConcurrentHashMap mergedSlugs = new ConcurrentHashMap<>(); } private final ConcurrentHashMap progressCounters = new ConcurrentHashMap<>(); @@ -566,6 +575,14 @@ public class WikiProcessingService { ProgressCounter pc = progressCounters.get(rawId); // ─── 阶段 A:路由 ─── + // Rebuild existingPagesIndex fresh at route time so pages created by earlier chunks + // in this run are visible. This prevents the route from scheduling "create" for a + // concept that was already created by a previous chunk (which would be caught by + // savePageContent and converted to update, but wastes a merge LLM call). + // listSummaries uses a 5-min TTL cache that is evicted on every create/update, so + // this picks up changes from sequential chunks without an extra DB hit when nothing changed. + String freshIndex = buildExistingPagesIndex(kbId); + String routeSystem = PromptLoader.loadPrompt("wiki/route-system"); String routeUserTemplate = PromptLoader.loadPrompt("wiki/route-user"); String documentMapSection = (documentMap != null && !documentMap.isBlank()) @@ -574,7 +591,7 @@ public class WikiProcessingService { String routeUser = routeUserTemplate .replace("{config}", configContent) .replace("{document_map_section}", documentMapSection) - .replace("{existing_pages}", existingPagesIndex) + .replace("{existing_pages}", freshIndex) .replace("{raw_title}", rawTitle) .replace("{raw_content}", textContent); Prompt routePrompt = new Prompt(List.of( @@ -662,8 +679,23 @@ public class WikiProcessingService { List> createFutures = new ArrayList<>(0); // kept for allOf join below // ─── 阶段 B-2:并行 merge ─── - List> mergeFutures = new ArrayList<>(updateSlugs.size()); + // Dedup: skip slugs already merged in this run to prevent N-version churn on + // high-frequency reference pages (e.g. a herb mentioned in every chapter gets v1, + // not v19). The first chunk that merges a slug wins; later chunks skip it and + // adjust the shared total counter so progress stays consistent. + List effectiveUpdateSlugs = new ArrayList<>(); for (String slug : updateSlugs) { + if (pc != null && pc.mergedSlugs.putIfAbsent(slug, Boolean.TRUE) != null) { + // Already merged in a previous chunk — remove from total so progress bar stays accurate + pc.total.decrementAndGet(); + log.debug("[Wiki] Phase B merge slug='{}' deduped (already merged this run), skipping", slug); + } else { + effectiveUpdateSlugs.add(slug); + } + } + + List> mergeFutures = new ArrayList<>(effectiveUpdateSlugs.size()); + for (String slug : effectiveUpdateSlugs) { final String mergeSlug = slug; mergeFutures.add(CompletableFuture.runAsync(() -> { try { @@ -1047,13 +1079,22 @@ public class WikiProcessingService { String configContent = kb.getConfigContent() != null ? kb.getConfigContent() : ""; String mergeSystem = PromptLoader.loadPrompt("wiki/merge-page-system"); + // Trim existing content to prevent context overflow on small models (qwen-turbo: 4096 tokens). + // Merging a 3000-char page + 30K chunk blows past the limit → truncated JSON → parse failure. + // 1800 chars ≈ ~600 tokens, leaving ample room for the chunk and response. + final int MAX_EXISTING_CHARS = 1800; + String rawExisting = existing.getContent() != null ? existing.getContent() : ""; + String trimmedExisting = rawExisting.length() > MAX_EXISTING_CHARS + ? rawExisting.substring(0, MAX_EXISTING_CHARS) + "\n...(内容已截断,请基于以上内容合并新信息)" + : rawExisting; + String mergeUserTemplate = PromptLoader.loadPrompt("wiki/merge-page-user"); String mergeUser = mergeUserTemplate .replace("{config}", configContent) .replace("{page_slug}", existing.getSlug() != null ? existing.getSlug() : slug) .replace("{page_title}", existing.getTitle() != null ? existing.getTitle() : "") .replace("{page_last_updated_by}", existing.getLastUpdatedBy() != null ? existing.getLastUpdatedBy() : "ai") - .replace("{page_content}", existing.getContent() != null ? existing.getContent() : "") + .replace("{page_content}", trimmedExisting) .replace("{raw_title}", raw.getTitle()) .replace("{raw_content}", chunkText); Prompt prompt = new Prompt(List.of( diff --git a/mateclaw-server/src/main/resources/prompts/wiki/route-system.txt b/mateclaw-server/src/main/resources/prompts/wiki/route-system.txt index 1dd19315..96174a5f 100644 --- a/mateclaw-server/src/main/resources/prompts/wiki/route-system.txt +++ b/mateclaw-server/src/main/resources/prompts/wiki/route-system.txt @@ -13,6 +13,13 @@ - 概念地图中出现的概念,若当前 chunk 只是一句话带过或完全未提及,**不要列入** - 同一概念不要跨 chunk 重复 create——如果已有页面索引里已有该概念,放入 update +## Update 列表约束(严格执行) + +- `update` 列表**最多 5 条**,超过不得超出 +- **只把本 chunk 中有至少 3 句以上实质新内容的页面列入 update** +- 仅被一两句话带过、或只作为"参见"引用的概念,**不列入 update** +- 如果符合条件的页面超过 5 个,选择本 chunk 中**覆盖深度最高的前 5 个** + ## 你不做什么 - ❌ **绝不输出 content 字段** —— 正文由后续的"单页生成助手"独立生成,你只决定结构 @@ -58,7 +65,7 @@ 字段说明: - `create`:完全新增的页面,**只给 metadata,不给 content** -- `update`:已存在但需要根据本材料合并更新的页面,**只列 slug 字符串数组** +- `update`:已存在但需要根据本材料合并更新的页面,**只列 slug 字符串数组,最多 5 条** - 两个数组合计通常至少 1 条(材料完全无价值——如空白页、纯目录——才允许全空) ## 关键纪律