From fa5d406118954f2a180d71ddc35633ae06787dd0 Mon Sep 17 00:00:00 2001 From: MIST <52695829+MISTLXC@users.noreply.github.com> Date: Fri, 3 Jul 2026 10:37:16 +0800 Subject: [PATCH] =?UTF-8?q?fix(wiki):=20agent=20=E9=80=9A=E8=BF=87=20wiki?= =?UTF-8?q?=5Fcreate=5Fpage=20=E5=86=99=E5=85=A5=E7=9A=84=E9=A1=B5?= =?UTF-8?q?=E9=9D=A2=E7=BC=BA=E5=B0=91=20raw/chunks/embeddings/citations?= =?UTF-8?q?=EF=BC=8C=E7=95=8C=E9=9D=A2=E6=97=A0=E6=B3=95=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E4=B8=8E=E6=93=8D=E4=BD=9C=20(#475)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## 背景 Agent 通过 `wiki_create_page` 工具写入知识库的报告、分析结果等页面，虽然能在"Wiki 页面"列表中看到，但： - **识别不到**：不出现在"原始材料"面板 - **不可操作**："查看引用"按钮消失（`sourceRawIds` 为空） - **不可处理**：无 raw 可 reprocess、无 chunks 导致语义检索查不到 - **不可下载**：无 raw 行，下载端点无数据 ## 根因 `wiki_create_page` 只调用 `WikiPageService.createPage()` 写了一张 `mate_wiki_page` 表（`sourceRawIds=null`），跳过了 UI 上传文本路径的全部"消化副产物"——raw material 创建、chunks 切片、embeddings 生成、citations 构建、lineage 血缘、`WikiPageCreatedEvent` 事件发布。 ## 解决方案 让 `wiki_create_page` 在落 page 之后，同步创建 raw material 并补齐全部消化副产物，但**不重跑 LLM 页面生成**（agent 已提供最终内容）。`WikiTool.wiki_create_page` 的同步返回（`ok / pageId / slug`）不受影响。 ### 改动（4 个文件，+190 / -1） | 文件 | 改动 | |---|---| | `WikiRawMaterialService.java` | 新增 `addAgentAuthored(kbId, title, content)`：创建 `sourceType="text"` 的 raw 行，状态置 `processing`（不发布 `WikiProcessingEvent`，避免触发 LLM 重消化），按 content hash 去重 | | `WikiProcessingService.java` | 新增 `linkAgentPageToRaw(pageId, kbId, rawId, rawTitle, pageType)`：编排 ①`mergeSourceLineage`（血缘）②`deriveKnowledgeLayer` ③`persistChunks`（切片）④`embedMissingChunks` + `embedPage` ⑤`buildCitationsAsync` ⑥发布 `WikiPageCreatedEvent` ⑦raw 置 `completed` + `lastProcessedHash`。每步独立 try/catch，单点失败不阻塞其它 | | `WikiTool.java` | `wiki_create_page` 在 `createPage` 后调用 `addAgentAuthored` + `linkAgentPageToRaw`。`processingService` 为可选注入（`@Autowired(required = false)`），测试上下文中为 null 时退化为旧行为 | | `WikiPageService.java` | `deleteExclusiveBySourceRawId` 新增跳过 `lastUpdatedBy = "ai"` 的页面：agent 直接创作的页面不应在 reprocess 时被自动清理 | ### 修复后效果 | 能力 | 修复前 | 修复后 | |---|---|---| | Wiki 页面列表可见 | ✅ | ✅ | | 原始材料面板可见（可识别） | ❌ | ✅ | | 下载按钮 | ❌ | ✅ | | 查看引用按钮（可操作） | ❌ 隐藏 | ✅ | | CitationDrawer 有内容 | ❌ 空 | ✅ | | 可 reprocess（可处理） | ❌ | ✅ | | 语义检索可命中 | ❌ | ✅ chunks+embeddings | | pipeline 触发器评估 | ❌ | ✅ | | 页面内容 = agent 原文 | ✅ | ✅（不重跑 LLM） | | UI "AI 生成"标记 | ✅ | ✅（`lastUpdatedBy = "ai"`） | ## 验证 全量 wiki 模块测试通过：`Tests run: 474, Failures: 0, Errors: 0, Skipped: 0` ## 风险 - `processingService` 为可选注入，测试/轻量上下文下退化为旧行为，**向后兼容无破坏** - 新增两处数据库写入（raw + chunks），每次 `wiki_create_page` 增加 1 条 raw + N 条 chunk 行 - reprocess 行为：reprocess 时 `lastUpdatedBy = "ai"` 的页面被保留不清理，LLM 管线会从 raw content 重新消化生成额外概念页 --- .../mate/wiki/service/WikiPageService.java | 2 +- .../wiki/service/WikiProcessingService.java | 105 ++++++++++++++++++ .../wiki/service/WikiRawMaterialService.java | 52 +++++++++ .../java/vip/mate/wiki/tool/WikiTool.java | 32 ++++++ 4 files changed, 190 insertions(+), 1 deletion(-) diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java index a10b0652..944e6c13 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java @@ -1200,7 +1200,7 @@ public class WikiPageService { List allPages = listByKbId(kbId); int deleted = 0; for (WikiPageEntity page : allPages) { - if ("manual".equals(page.getLastUpdatedBy())) continue; + if ("manual".equals(page.getLastUpdatedBy()) || "ai".equals(page.getLastUpdatedBy())) continue; // RFC-051 PR-2: never sweep system / locked pages, even when their // source raw is being reprocessed. if (isProtected(page)) continue; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 7b93612d..5fca7f82 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -1831,6 +1831,111 @@ public class WikiProcessingService { } } + /** + * Link an agent-authored page to its source raw material and populate the + * full set of ingest by-products (lineage, chunks, embeddings, citations, + * knowledge layer, pipeline-trigger event) so the page is a first-class + * citizen — searchable, citation-traceable, downloadable, reprocess-able — + * WITHOUT re-running the LLM page-generation pipeline (the agent already + * supplied the final page content). + * + *

Called by {@code WikiTool.wiki_create_page} right after {@code createPage}. + * Each step is individually try/caught so a failure in one (e.g. embedding + * provider down) cannot block the others — the page still lands with lineage + * and citations even if embeddings are deferred. The raw is flipped to + * {@code completed} at the end so it shows as a successfully processed + * material in the Raw Material panel. + * + * @param pageId the newly created page id + * @param kbId the KB id + * @param rawId the agent-authored raw material id (from {@code addAgentAuthored}) + * @param rawTitle the raw material title (for the lineage snapshot) + * @param pageType the page's pageType (may be null; used for layer + event) + */ + public void linkAgentPageToRaw(Long pageId, Long kbId, Long rawId, String rawTitle, String pageType) { + // 1. Lineage: page -> raw (dual-writes sourceRawIds + sourceEntries so + // the "View Citations" button appears and raw-delete cascade works). + try { + pageService.mergeSourceLineage(pageId, rawId, rawTitle); + } catch (Exception e) { + log.warn("[Wiki] Agent-page lineage failed for page={}, raw={}: {}", pageId, rawId, e.getMessage()); + } + + // Knowledge layer (fact/experience) from the pageType profile, matching + // afterPagePersisted so agent pages join the KB's layer classification. + try { + deriveKnowledgeLayer(pageId, kbId, pageType); + } catch (Exception e) { + log.warn("[Wiki] Agent-page knowledge layer failed for page={}: {}", pageId, e.getMessage()); + } + + // 2. Chunks from the raw's text — reuse the standard splitter so semantic + // search and citations work exactly like an uploaded text file. + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw == null) { + log.warn("[Wiki] Agent-page link skipped: raw={} not found", rawId); + return; + } + String textContent = rawService.getTextContent(raw); + if (textContent != null && !textContent.isBlank()) { + try { + List chunksWithOffset = splitIntoChunksWithOffsets(textContent); + List chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList(); + List offsets = chunksWithOffset.stream() + .map(c -> new int[]{c.startOffset(), c.endOffset()}).toList(); + if (chunks.isEmpty()) { + chunkService.persistChunks(kbId, rawId, + List.of(textContent), List.of(new int[]{0, textContent.length()})); + } else { + chunkService.persistChunks(kbId, rawId, chunks, offsets); + } + } catch (Exception e) { + log.warn("[Wiki] Agent-page chunk persistence failed for page={}, raw={}: {}", + pageId, rawId, e.getMessage()); + } + } + + // 3. Embeddings (async-safe calls) — both chunk-level (for semantic + // search) and page-level (for hybrid retrieval). + try { + embeddingService.embedMissingChunks(kbId); + embeddingService.embedPage(pageId); + } catch (Exception e) { + log.warn("[Wiki] Agent-page embedding failed for kb={}, page={}: {}", kbId, pageId, e.getMessage()); + } + + // 4. Citations: page -> chunks of its source raw (feeds the CitationDrawer). + try { + citationService.buildCitationsAsync(pageId, kbId); + } catch (Exception e) { + log.warn("[Wiki] Agent-page citation build failed for page={}: {}", pageId, e.getMessage()); + } + + // 5. Pipeline trigger event (so pageType-count triggers get evaluated), + // matching afterPagePersisted's contract for ingest-created pages. + try { + if (eventPublisher != null && pageType != null && !pageType.isBlank()) { + eventPublisher.publishEvent(new WikiPageCreatedEvent(kbId, pageType, pageId)); + } + } catch (Exception e) { + log.warn("[Wiki] Agent-page event publish failed for page={}: {}", pageId, e.getMessage()); + } + + // 6. Flip raw to completed so it shows as a successfully processed + // material in the Raw Material panel, and stamp lastProcessedHash so + // the reprocess short-circuit (unchanged content -> skip) works if + // the user later reprocesses this raw through the normal pipeline. + try { + rawService.updateProcessingStatus(rawId, "completed", null, null); + rawService.setLastProcessedHash(rawId, raw.getContentHash()); + } catch (Exception e) { + log.warn("[Wiki] Agent-page raw status flip failed for raw={}: {}", rawId, e.getMessage()); + } + + log.info("[Wiki] Agent page linked: pageId={}, kbId={}, rawId={}, pageType={}", + pageId, kbId, rawId, pageType); + } + /** Stamp the page's knowledge layer (fact/experience) derived from its pageType profile. */ private void deriveKnowledgeLayer(Long pageId, Long kbId, String pageType) { if (pageTypeProfileService == null || pageType == null || pageType.isBlank()) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index a2d90254..0e278080 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -298,6 +298,58 @@ public class WikiRawMaterialService { return entity; } + /** + * Create a raw material record for agent-authored content WITHOUT triggering + * the LLM ingest pipeline. Used by {@code wiki_create_page} so an agent-written + * page gets a lineage anchor — it appears in the Raw Material panel, hosts + * chunks, supports the download button (text raws are served from the + * {@code original_content} column by the download endpoint), and can be + * reprocessed later — without re-running LLM page generation, since the + * agent has already produced the final page content. + * + *

The raw is left in {@code processing} status; the caller flips it to + * {@code completed} via {@link WikiProcessingService#linkAgentPageToRaw} + * once chunks + citations have landed. Dedup by content hash reuses an + * existing row when the agent writes the same content again (idempotent), + * mirroring {@link #addText}'s dedup semantics. + * + * @return the raw material entity (newly inserted or an existing same-content row) + */ + @Transactional + public WikiRawMaterialEntity addAgentAuthored(Long kbId, String title, String content) { + String hash = computeHash(content); + + // Dedup: reuse any existing row with the same hash in this KB (any status). + // An agent often re-writes the same report title in a conversation; stacking + // duplicate raws would pollute the Raw Material panel. + WikiRawMaterialEntity existing = rawMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiRawMaterialEntity::getKbId, kbId) + .eq(WikiRawMaterialEntity::getContentHash, hash) + .last("LIMIT 1")); + if (existing != null) { + return existing; + } + + WikiRawMaterialEntity entity = new WikiRawMaterialEntity(); + entity.setKbId(kbId); + entity.setTitle(title); + entity.setSourceType("text"); + entity.setOriginalContent(content); + entity.setFileSize((long) content.getBytes(StandardCharsets.UTF_8).length); + entity.setContentHash(hash); + // 'processing' rather than 'pending': this raw is claimed by the agent + // tool's own synchronous post-processing (linkAgentPageToRaw), so it + // must NOT be picked up by the async ingest listener (which would + // re-run LLM page generation). The caller flips it to 'completed'. + entity.setProcessingStatus("processing"); + rawMapper.insert(entity); + kbService.incrementRawCount(kbId); + + log.info("[Wiki] Agent-authored raw material added: id={}, kbId={}, title={}", entity.getId(), kbId, title); + return entity; + } + /** * Adds a file-type raw material (PDF / DOCX / image / ...). * diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index 526e3285..9e10133d 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -97,6 +97,17 @@ public class WikiTool { @Autowired(required = false) private vip.mate.wiki.profile.WikiPageTypeProfileService pageTypeProfileService; + /** + * Optional. When present, {@code wiki_create_page} also creates a raw + * material + chunks + embeddings + citations + lineage so the agent-written + * page is a first-class citizen (searchable / citation-traceable / + * downloadable / reprocess-able) identical to a UI-uploaded text file. + * Absent in lightweight test contexts — the page is still created, just + * without the ingest by-products (legacy behavior). + */ + @Autowired(required = false) + private WikiProcessingService processingService; + /** * Per-agent pageType permission gate. Mandatory: this is a security control, * so it is a required constructor dependency rather than an optional bean — @@ -509,6 +520,27 @@ public class WikiTool { WikiPageEntity page = pageService.createPage(kbId, slug, title, content, summary, null, pageType); log.info("[WikiTool] Created page: {} (slug={}, kbId={}, type={})", title, slug, kbId, pageType); + // Make the agent-written page a first-class citizen: create a raw + // material (so it shows in the Raw Material panel + supports the + // download button) and link page -> raw with chunks / embeddings / + // citations / lineage (so "View Citations", semantic search, and + // reprocess all work) — identical to a UI-uploaded text file, but + // without re-running LLM page generation (the content is already final). + // Skipped in lightweight test contexts where processingService is null. + if (processingService != null) { + try { + WikiRawMaterialEntity raw = rawService.addAgentAuthored(kbId, title, content); + processingService.linkAgentPageToRaw( + page.getId(), kbId, raw.getId(), raw.getTitle(), pageType); + } catch (Exception e) { + // The page itself is already persisted; by-product population + // is a best-effort enhancement, never a blocker for the tool + // contract (agent already has its pageId to return). + log.warn("[WikiTool] Agent-page by-product population failed for page={}: {}", + page.getId(), e.getMessage()); + } + } + return JSONUtil.createObj() .set("ok", true) .set("message", "Page created successfully")