mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
fix(wiki): agent 通过 wiki_create_page 写入的页面缺少 raw/chunks/embeddings/citations，界面无法识别与操作 (#475)
## 背景 Agent 通过 `wiki_create_page` 工具写入知识库的报告、分析结果等页面，虽然能在"Wiki 页面"列表中看到，但： - **识别不到**：不出现在"原始材料"面板 - **不可操作**："查看引用"按钮消失（`sourceRawIds` 为空） - **不可处理**：无 raw 可 reprocess、无 chunks 导致语义检索查不到 - **不可下载**：无 raw 行，下载端点无数据 ## 根因 `wiki_create_page` 只调用 `WikiPageService.createPage()` 写了一张 `mate_wiki_page` 表（`sourceRawIds=null`），跳过了 UI 上传文本路径的全部"消化副产物"——raw material 创建、chunks 切片、embeddings 生成、citations 构建、lineage 血缘、`WikiPageCreatedEvent` 事件发布。 ## 解决方案 让 `wiki_create_page` 在落 page 之后，同步创建 raw material 并补齐全部消化副产物，但**不重跑 LLM 页面生成**（agent 已提供最终内容）。`WikiTool.wiki_create_page` 的同步返回（`ok / pageId / slug`）不受影响。 ### 改动（4 个文件，+190 / -1） | 文件 | 改动 | |---|---| | `WikiRawMaterialService.java` | 新增 `addAgentAuthored(kbId, title, content)`：创建 `sourceType="text"` 的 raw 行，状态置 `processing`（不发布 `WikiProcessingEvent`，避免触发 LLM 重消化），按 content hash 去重 | | `WikiProcessingService.java` | 新增 `linkAgentPageToRaw(pageId, kbId, rawId, rawTitle, pageType)`：编排 ①`mergeSourceLineage`（血缘）②`deriveKnowledgeLayer` ③`persistChunks`（切片）④`embedMissingChunks` + `embedPage` ⑤`buildCitationsAsync` ⑥发布 `WikiPageCreatedEvent` ⑦raw 置 `completed` + `lastProcessedHash`。每步独立 try/catch，单点失败不阻塞其它 | | `WikiTool.java` | `wiki_create_page` 在 `createPage` 后调用 `addAgentAuthored` + `linkAgentPageToRaw`。`processingService` 为可选注入（`@Autowired(required = false)`），测试上下文中为 null 时退化为旧行为 | | `WikiPageService.java` | `deleteExclusiveBySourceRawId` 新增跳过 `lastUpdatedBy = "ai"` 的页面：agent 直接创作的页面不应在 reprocess 时被自动清理 | ### 修复后效果 | 能力 | 修复前 | 修复后 | |---|---|---| | Wiki 页面列表可见 | ✅ | ✅ | | 原始材料面板可见（可识别） | ❌ | ✅ | | 下载按钮 | ❌ | ✅ | | 查看引用按钮（可操作） | ❌ 隐藏 | ✅ | | CitationDrawer 有内容 | ❌ 空 | ✅ | | 可 reprocess（可处理） | ❌ | ✅ | | 语义检索可命中 | ❌ | ✅ chunks+embeddings | | pipeline 触发器评估 | ❌ | ✅ | | 页面内容 = agent 原文 | ✅ | ✅（不重跑 LLM） | | UI "AI 生成"标记 | ✅ | ✅（`lastUpdatedBy = "ai"`） | ## 验证 全量 wiki 模块测试通过：`Tests run: 474, Failures: 0, Errors: 0, Skipped: 0` ## 风险 - `processingService` 为可选注入，测试/轻量上下文下退化为旧行为，**向后兼容无破坏** - 新增两处数据库写入（raw + chunks），每次 `wiki_create_page` 增加 1 条 raw + N 条 chunk 行 - reprocess 行为：reprocess 时 `lastUpdatedBy = "ai"` 的页面被保留不清理，LLM 管线会从 raw content 重新消化生成额外概念页
This commit is contained in:
MIST
GitHub
parent
2e3dd071a5
commit
fa5d406118
@ -1200,7 +1200,7 @@ public class WikiPageService {
|
||||
List<WikiPageEntity> allPages = listByKbId(kbId);
|
||||
int deleted = 0;
|
||||
for (WikiPageEntity page : allPages) {
|
||||
if ("manual".equals(page.getLastUpdatedBy())) continue;
|
||||
if ("manual".equals(page.getLastUpdatedBy()) || "ai".equals(page.getLastUpdatedBy())) continue;
|
||||
// RFC-051 PR-2: never sweep system / locked pages, even when their
|
||||
// source raw is being reprocessed.
|
||||
if (isProtected(page)) continue;
|
||||
|
||||
@ -1831,6 +1831,111 @@ public class WikiProcessingService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Link an agent-authored page to its source raw material and populate the
|
||||
* full set of ingest by-products (lineage, chunks, embeddings, citations,
|
||||
* knowledge layer, pipeline-trigger event) so the page is a first-class
|
||||
* citizen — searchable, citation-traceable, downloadable, reprocess-able —
|
||||
* WITHOUT re-running the LLM page-generation pipeline (the agent already
|
||||
* supplied the final page content).
|
||||
*
|
||||
* <p>Called by {@code WikiTool.wiki_create_page} right after {@code createPage}.
|
||||
* Each step is individually try/caught so a failure in one (e.g. embedding
|
||||
* provider down) cannot block the others — the page still lands with lineage
|
||||
* and citations even if embeddings are deferred. The raw is flipped to
|
||||
* {@code completed} at the end so it shows as a successfully processed
|
||||
* material in the Raw Material panel.
|
||||
*
|
||||
* @param pageId the newly created page id
|
||||
* @param kbId the KB id
|
||||
* @param rawId the agent-authored raw material id (from {@code addAgentAuthored})
|
||||
* @param rawTitle the raw material title (for the lineage snapshot)
|
||||
* @param pageType the page's pageType (may be null; used for layer + event)
|
||||
*/
|
||||
public void linkAgentPageToRaw(Long pageId, Long kbId, Long rawId, String rawTitle, String pageType) {
|
||||
// 1. Lineage: page -> raw (dual-writes sourceRawIds + sourceEntries so
|
||||
// the "View Citations" button appears and raw-delete cascade works).
|
||||
try {
|
||||
pageService.mergeSourceLineage(pageId, rawId, rawTitle);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page lineage failed for page={}, raw={}: {}", pageId, rawId, e.getMessage());
|
||||
}
|
||||
|
||||
// Knowledge layer (fact/experience) from the pageType profile, matching
|
||||
// afterPagePersisted so agent pages join the KB's layer classification.
|
||||
try {
|
||||
deriveKnowledgeLayer(pageId, kbId, pageType);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page knowledge layer failed for page={}: {}", pageId, e.getMessage());
|
||||
}
|
||||
|
||||
// 2. Chunks from the raw's text — reuse the standard splitter so semantic
|
||||
// search and citations work exactly like an uploaded text file.
|
||||
WikiRawMaterialEntity raw = rawService.getById(rawId);
|
||||
if (raw == null) {
|
||||
log.warn("[Wiki] Agent-page link skipped: raw={} not found", rawId);
|
||||
return;
|
||||
}
|
||||
String textContent = rawService.getTextContent(raw);
|
||||
if (textContent != null && !textContent.isBlank()) {
|
||||
try {
|
||||
List<ChunkWithOffset> chunksWithOffset = splitIntoChunksWithOffsets(textContent);
|
||||
List<String> chunks = chunksWithOffset.stream().map(ChunkWithOffset::text).toList();
|
||||
List<int[]> offsets = chunksWithOffset.stream()
|
||||
.map(c -> new int[]{c.startOffset(), c.endOffset()}).toList();
|
||||
if (chunks.isEmpty()) {
|
||||
chunkService.persistChunks(kbId, rawId,
|
||||
List.of(textContent), List.of(new int[]{0, textContent.length()}));
|
||||
} else {
|
||||
chunkService.persistChunks(kbId, rawId, chunks, offsets);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page chunk persistence failed for page={}, raw={}: {}",
|
||||
pageId, rawId, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Embeddings (async-safe calls) — both chunk-level (for semantic
|
||||
// search) and page-level (for hybrid retrieval).
|
||||
try {
|
||||
embeddingService.embedMissingChunks(kbId);
|
||||
embeddingService.embedPage(pageId);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page embedding failed for kb={}, page={}: {}", kbId, pageId, e.getMessage());
|
||||
}
|
||||
|
||||
// 4. Citations: page -> chunks of its source raw (feeds the CitationDrawer).
|
||||
try {
|
||||
citationService.buildCitationsAsync(pageId, kbId);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page citation build failed for page={}: {}", pageId, e.getMessage());
|
||||
}
|
||||
|
||||
// 5. Pipeline trigger event (so pageType-count triggers get evaluated),
|
||||
// matching afterPagePersisted's contract for ingest-created pages.
|
||||
try {
|
||||
if (eventPublisher != null && pageType != null && !pageType.isBlank()) {
|
||||
eventPublisher.publishEvent(new WikiPageCreatedEvent(kbId, pageType, pageId));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page event publish failed for page={}: {}", pageId, e.getMessage());
|
||||
}
|
||||
|
||||
// 6. Flip raw to completed so it shows as a successfully processed
|
||||
// material in the Raw Material panel, and stamp lastProcessedHash so
|
||||
// the reprocess short-circuit (unchanged content -> skip) works if
|
||||
// the user later reprocesses this raw through the normal pipeline.
|
||||
try {
|
||||
rawService.updateProcessingStatus(rawId, "completed", null, null);
|
||||
rawService.setLastProcessedHash(rawId, raw.getContentHash());
|
||||
} catch (Exception e) {
|
||||
log.warn("[Wiki] Agent-page raw status flip failed for raw={}: {}", rawId, e.getMessage());
|
||||
}
|
||||
|
||||
log.info("[Wiki] Agent page linked: pageId={}, kbId={}, rawId={}, pageType={}",
|
||||
pageId, kbId, rawId, pageType);
|
||||
}
|
||||
|
||||
/** Stamp the page's knowledge layer (fact/experience) derived from its pageType profile. */
|
||||
private void deriveKnowledgeLayer(Long pageId, Long kbId, String pageType) {
|
||||
if (pageTypeProfileService == null || pageType == null || pageType.isBlank()) {
|
||||
|
||||
@ -298,6 +298,58 @@ public class WikiRawMaterialService {
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a raw material record for agent-authored content WITHOUT triggering
|
||||
* the LLM ingest pipeline. Used by {@code wiki_create_page} so an agent-written
|
||||
* page gets a lineage anchor — it appears in the Raw Material panel, hosts
|
||||
* chunks, supports the download button (text raws are served from the
|
||||
* {@code original_content} column by the download endpoint), and can be
|
||||
* reprocessed later — without re-running LLM page generation, since the
|
||||
* agent has already produced the final page content.
|
||||
*
|
||||
* <p>The raw is left in {@code processing} status; the caller flips it to
|
||||
* {@code completed} via {@link WikiProcessingService#linkAgentPageToRaw}
|
||||
* once chunks + citations have landed. Dedup by content hash reuses an
|
||||
* existing row when the agent writes the same content again (idempotent),
|
||||
* mirroring {@link #addText}'s dedup semantics.
|
||||
*
|
||||
* @return the raw material entity (newly inserted or an existing same-content row)
|
||||
*/
|
||||
@Transactional
|
||||
public WikiRawMaterialEntity addAgentAuthored(Long kbId, String title, String content) {
|
||||
String hash = computeHash(content);
|
||||
|
||||
// Dedup: reuse any existing row with the same hash in this KB (any status).
|
||||
// An agent often re-writes the same report title in a conversation; stacking
|
||||
// duplicate raws would pollute the Raw Material panel.
|
||||
WikiRawMaterialEntity existing = rawMapper.selectOne(
|
||||
new LambdaQueryWrapper<WikiRawMaterialEntity>()
|
||||
.eq(WikiRawMaterialEntity::getKbId, kbId)
|
||||
.eq(WikiRawMaterialEntity::getContentHash, hash)
|
||||
.last("LIMIT 1"));
|
||||
if (existing != null) {
|
||||
return existing;
|
||||
}
|
||||
|
||||
WikiRawMaterialEntity entity = new WikiRawMaterialEntity();
|
||||
entity.setKbId(kbId);
|
||||
entity.setTitle(title);
|
||||
entity.setSourceType("text");
|
||||
entity.setOriginalContent(content);
|
||||
entity.setFileSize((long) content.getBytes(StandardCharsets.UTF_8).length);
|
||||
entity.setContentHash(hash);
|
||||
// 'processing' rather than 'pending': this raw is claimed by the agent
|
||||
// tool's own synchronous post-processing (linkAgentPageToRaw), so it
|
||||
// must NOT be picked up by the async ingest listener (which would
|
||||
// re-run LLM page generation). The caller flips it to 'completed'.
|
||||
entity.setProcessingStatus("processing");
|
||||
rawMapper.insert(entity);
|
||||
kbService.incrementRawCount(kbId);
|
||||
|
||||
log.info("[Wiki] Agent-authored raw material added: id={}, kbId={}, title={}", entity.getId(), kbId, title);
|
||||
return entity;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds a file-type raw material (PDF / DOCX / image / ...).
|
||||
*
|
||||
|
||||
@ -97,6 +97,17 @@ public class WikiTool {
|
||||
@Autowired(required = false)
|
||||
private vip.mate.wiki.profile.WikiPageTypeProfileService pageTypeProfileService;
|
||||
|
||||
/**
|
||||
* Optional. When present, {@code wiki_create_page} also creates a raw
|
||||
* material + chunks + embeddings + citations + lineage so the agent-written
|
||||
* page is a first-class citizen (searchable / citation-traceable /
|
||||
* downloadable / reprocess-able) identical to a UI-uploaded text file.
|
||||
* Absent in lightweight test contexts — the page is still created, just
|
||||
* without the ingest by-products (legacy behavior).
|
||||
*/
|
||||
@Autowired(required = false)
|
||||
private WikiProcessingService processingService;
|
||||
|
||||
/**
|
||||
* Per-agent pageType permission gate. Mandatory: this is a security control,
|
||||
* so it is a required constructor dependency rather than an optional bean —
|
||||
@ -509,6 +520,27 @@ public class WikiTool {
|
||||
WikiPageEntity page = pageService.createPage(kbId, slug, title, content, summary, null, pageType);
|
||||
log.info("[WikiTool] Created page: {} (slug={}, kbId={}, type={})", title, slug, kbId, pageType);
|
||||
|
||||
// Make the agent-written page a first-class citizen: create a raw
|
||||
// material (so it shows in the Raw Material panel + supports the
|
||||
// download button) and link page -> raw with chunks / embeddings /
|
||||
// citations / lineage (so "View Citations", semantic search, and
|
||||
// reprocess all work) — identical to a UI-uploaded text file, but
|
||||
// without re-running LLM page generation (the content is already final).
|
||||
// Skipped in lightweight test contexts where processingService is null.
|
||||
if (processingService != null) {
|
||||
try {
|
||||
WikiRawMaterialEntity raw = rawService.addAgentAuthored(kbId, title, content);
|
||||
processingService.linkAgentPageToRaw(
|
||||
page.getId(), kbId, raw.getId(), raw.getTitle(), pageType);
|
||||
} catch (Exception e) {
|
||||
// The page itself is already persisted; by-product population
|
||||
// is a best-effort enhancement, never a blocker for the tool
|
||||
// contract (agent already has its pageId to return).
|
||||
log.warn("[WikiTool] Agent-page by-product population failed for page={}: {}",
|
||||
page.getId(), e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
return JSONUtil.createObj()
|
||||
.set("ok", true)
|
||||
.set("message", "Page created successfully")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user