diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java index 9500d1c8..43daf60b 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java @@ -43,8 +43,12 @@ public class WikiContextService { StringBuilder sb = new StringBuilder(); sb.append("\n\n## Wiki Knowledge Base\n\n"); - sb.append("You have access to structured wiki knowledge bases. "); - sb.append("Use `wiki_read_page` tool to read full page content when needed.\n\n"); + sb.append("You have access to structured wiki knowledge bases. The knowledge base is automatically resolved from your agentId.\n\n"); + sb.append("Wiki tools (only need agentId + slug or query, NO kbId needed):\n"); + sb.append("- `wiki_search_pages(agentId, query)` — full-text search across titles, summaries, and content\n"); + sb.append("- `wiki_read_page(agentId, slug)` — read full page content with source file info\n"); + sb.append("- `wiki_list_pages(agentId)` — list all pages with summaries\n"); + sb.append("- `wiki_trace_source(agentId, slug)` — find which original documents a page was generated from\n\n"); int totalChars = 0; int maxChars = properties.getMaxContextChars(); @@ -57,21 +61,34 @@ public class WikiContextService { if (kb.getDescription() != null && !kb.getDescription().isBlank()) { sb.append(" — ").append(kb.getDescription()); } - sb.append("\n\n"); - sb.append("Knowledge Base ID: `").append(kb.getId()).append("`\n\n"); - sb.append("Available pages:\n"); + sb.append(" (").append(pages.size()).append(" pages)\n\n"); - for (WikiPageEntity page : pages) { - String line = "- **[[" + page.getTitle() + "]]** (`" + page.getSlug() + "`): " - + (page.getSummary() != null ? page.getSummary() : "No summary") + "\n"; + // 大量页面时只列标题索引（紧凑模式），节省 prompt 空间 + boolean compact = pages.size() > 20; - if (totalChars + line.length() > maxChars) { - sb.append("- ... and more pages (use `wiki_list_pages` to see all)\n"); - break; + if (compact) { + sb.append("Page index (use `wiki_search_pages` to find relevant pages, `wiki_read_page` to read full content):\n"); + for (WikiPageEntity page : pages) { + String line = "- `" + page.getSlug() + "` — " + page.getTitle() + "\n"; + if (totalChars + line.length() > maxChars) { + sb.append("- ... and ").append(pages.size()).append(" total pages (use `wiki_list_pages` to see all)\n"); + break; + } + sb.append(line); + totalChars += line.length(); + } + } else { + sb.append("Available pages:\n"); + for (WikiPageEntity page : pages) { + String line = "- **[[" + page.getTitle() + "]]** (`" + page.getSlug() + "`): " + + (page.getSummary() != null ? page.getSummary() : "No summary") + "\n"; + if (totalChars + line.length() > maxChars) { + sb.append("- ... and more pages (use `wiki_list_pages` to see all)\n"); + break; + } + sb.append(line); + totalChars += line.length(); } - - sb.append(line); - totalChars += line.length(); } sb.append("\n"); diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java index 4af94861..1f5beed0 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java @@ -43,6 +43,16 @@ public class WikiPageService { return pages; } + /** + * 列出知识库所有页面（含 content，用于全文搜索） + */ + public List listByKbIdWithContent(Long kbId) { + return pageMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiPageEntity::getKbId, kbId) + .orderByAsc(WikiPageEntity::getTitle)); + } + /** * 列出页面摘要（用于上下文注入和 LLM 消化） */ diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 0fbe0d2a..e00a49fe 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -75,16 +75,26 @@ public class WikiProcessingService { } // Phase 2: LLM 消化 - int totalPages; + // result[0] = totalPages, result[1] = failedChunks, result[2] = totalChunks + int[] result; if (textContent.length() > properties.getMaxChunkSize()) { - totalPages = processInChunks(kb, raw, textContent); + result = processInChunks(kb, raw, textContent); } else { - totalPages = processChunk(kb, raw, textContent); + int pages = processChunk(kb, raw, textContent); + result = new int[]{pages, pages == 0 ? 1 : 0, 1}; } + int totalPages = result[0]; + int failedChunks = result[1]; + int totalChunks = result[2]; + // Phase 3: 更新状态和计数 if (totalPages == 0) { rawService.updateProcessingStatus(rawId, "failed", "No pages generated from LLM response"); + } else if (failedChunks > 0) { + // 部分成功：有些 chunk 失败但有些产出了页面 + rawService.updateProcessingStatus(rawId, "partial", + failedChunks + " of " + totalChunks + " chunks failed, " + totalPages + " pages generated"); } else { rawService.updateProcessingStatus(rawId, "completed", null); } @@ -120,13 +130,14 @@ public class WikiProcessingService { /** * 分块处理大文档 * - * @return 创建+更新的页面总数 + * @return int[3]: [totalPages, failedChunks, totalChunks] */ - private int processInChunks(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw, String text) { + private int[] processInChunks(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw, String text) { int chunkSize = properties.getMaxChunkSize(); int overlap = 500; // 块间重叠 int start = 0; int totalPages = 0; + int failedChunks = 0; int chunkIndex = 0; while (start < text.length()) { @@ -145,7 +156,17 @@ public class WikiProcessingService { String chunk = text.substring(start, end); log.info("[Wiki] Processing chunk {}: chars {}-{} of {}", chunkIndex, start, end, text.length()); - totalPages += processChunk(kb, raw, chunk); + try { + totalPages += processChunk(kb, raw, chunk); + } catch (Exception e) { + failedChunks++; + // content_filter 错误：标注后继续处理其他 chunk + if (e.getMessage() != null && e.getMessage().contains("content_filter")) { + log.warn("[Wiki] Chunk {} blocked by content filter, skipping", chunkIndex); + } else { + log.warn("[Wiki] Chunk {} failed: {}", chunkIndex, e.getMessage()); + } + } start = end - overlap; if (start < 0) start = 0; @@ -153,7 +174,7 @@ public class WikiProcessingService { if (start <= previousStart) start = end; chunkIndex++; } - return totalPages; + return new int[]{totalPages, failedChunks, chunkIndex}; } /** @@ -287,6 +308,8 @@ public class WikiProcessingService { if (response == null || response.isBlank()) return null; String cleaned = response.trim(); + + // 1. 剥离 markdown 代码块标记 if (cleaned.startsWith("```json")) { cleaned = cleaned.substring(7); } else if (cleaned.startsWith("```")) { @@ -297,9 +320,24 @@ public class WikiProcessingService { } cleaned = cleaned.trim(); + // 2. 清洗控制字符（保留 \n \r \t），防止 LLM 输出含不可见字符导致 JSON 解析失败 + cleaned = cleaned.replaceAll("[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]", ""); + + // 3. 第一次尝试直接解析 try { return objectMapper.readTree(cleaned); } catch (Exception e) { + // 4. 如果整体不是 JSON，尝试提取第一个 JSON 对象块（LLM 可能在 JSON 前后加了说明文字） + int jsonStart = cleaned.indexOf("{"); + int jsonEnd = cleaned.lastIndexOf("}"); + if (jsonStart >= 0 && jsonEnd > jsonStart) { + String extracted = cleaned.substring(jsonStart, jsonEnd + 1); + try { + return objectMapper.readTree(extracted); + } catch (Exception e2) { + log.warn("[Wiki] Failed to parse extracted JSON block: {}", e2.getMessage()); + } + } log.warn("[Wiki] Failed to parse JSON response: {}", e.getMessage()); return null; } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index a4840437..25d5484c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -3,6 +3,8 @@ package vip.mate.wiki.tool; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.tool.annotation.Tool; @@ -10,8 +12,10 @@ import org.springframework.ai.tool.annotation.ToolParam; import org.springframework.stereotype.Component; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.model.WikiRawMaterialEntity; import vip.mate.wiki.service.WikiKnowledgeBaseService; import vip.mate.wiki.service.WikiPageService; +import vip.mate.wiki.service.WikiRawMaterialService; import java.util.List; @@ -19,6 +23,7 @@ import java.util.List; * Wiki 知识库工具 *

* 供 Agent 在对话中按需读取 Wiki 页面内容。 + * kbId 通过 agentId 自动解析，LLM 无需传递。 * * @author MateClaw Team */ @@ -29,23 +34,25 @@ public class WikiTool { private final WikiPageService pageService; private final WikiKnowledgeBaseService kbService; + private final WikiRawMaterialService rawService; @Tool(description = """ 读取 Wiki 知识库中指定页面的完整内容。 当系统提示词中的 Wiki 页面摘要不够详细时，使用此工具获取完整内容。 - 返回 Markdown 格式的页面内容，包含 [[双向链接]]。 + 返回 Markdown 格式的页面内容，包含 [[双向链接]] 和来源原始文件信息。 """) public String wiki_read_page( @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "知识库 ID") Long kbId, @ToolParam(description = "页面标识符 (slug)") String slug) { - if (kbId == null || slug == null || slug.isBlank()) { - return error("kbId and slug are required"); + if (slug == null || slug.isBlank()) { + return error("slug is required"); } - String accessError = checkAccess(agentId, kbId); - if (accessError != null) return accessError; + Long kbId = resolveKbId(agentId); + if (kbId == null) { + return error("No wiki knowledge base found for this agent"); + } WikiPageEntity page = pageService.getBySlug(kbId, slug); if (page == null) { @@ -57,7 +64,8 @@ public class WikiTool { .set("slug", page.getSlug()) .set("version", page.getVersion()) .set("lastUpdatedBy", page.getLastUpdatedBy()) - .set("content", page.getContent()); + .set("content", page.getContent()) + .set("sourceFiles", resolveSourceFiles(page.getSourceRawIds())); return result.toString(); } @@ -66,16 +74,13 @@ public class WikiTool { 返回页面列表，包含标题、slug 和摘要。 """) public String wiki_list_pages( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "知识库 ID") Long kbId) { + @ToolParam(description = "当前 Agent 的 ID") Long agentId) { + Long kbId = resolveKbId(agentId); if (kbId == null) { - return error("kbId is required"); + return error("No wiki knowledge base found for this agent"); } - String accessError = checkAccess(agentId, kbId); - if (accessError != null) return accessError; - List pages = pageService.listSummaries(kbId); JSONArray arr = new JSONArray(); for (WikiPageEntity page : pages) { @@ -94,33 +99,41 @@ public class WikiTool { @Tool(description = """ 在 Wiki 知识库中搜索页面。 - 按关键词搜索页面标题和摘要，返回匹配的页面列表。 + 按关键词搜索页面标题、摘要和正文内容，返回匹配的页面列表及其来源文件。 """) public String wiki_search_pages( @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "知识库 ID") Long kbId, @ToolParam(description = "搜索关键词") String query) { - if (kbId == null || query == null || query.isBlank()) { - return error("kbId and query are required"); + if (query == null || query.isBlank()) { + return error("query is required"); } - String accessError = checkAccess(agentId, kbId); - if (accessError != null) return accessError; + Long kbId = resolveKbId(agentId); + if (kbId == null) { + return error("No wiki knowledge base found for this agent"); + } String queryLower = query.toLowerCase(); - List pages = pageService.listSummaries(kbId); - List matched = pages.stream() + List allPages = pageService.listByKbIdWithContent(kbId); + List matched = allPages.stream() .filter(p -> (p.getTitle() != null && p.getTitle().toLowerCase().contains(queryLower)) - || (p.getSummary() != null && p.getSummary().toLowerCase().contains(queryLower))) + || (p.getSummary() != null && p.getSummary().toLowerCase().contains(queryLower)) + || (p.getContent() != null && p.getContent().toLowerCase().contains(queryLower))) + .limit(20) .toList(); JSONArray arr = new JSONArray(); for (WikiPageEntity page : matched) { - arr.add(JSONUtil.createObj() + JSONObject obj = JSONUtil.createObj() .set("title", page.getTitle()) .set("slug", page.getSlug()) - .set("summary", page.getSummary())); + .set("summary", page.getSummary()) + .set("sourceFiles", resolveSourceFiles(page.getSourceRawIds())); + boolean titleMatch = page.getTitle() != null && page.getTitle().toLowerCase().contains(queryLower); + boolean contentMatch = page.getContent() != null && page.getContent().toLowerCase().contains(queryLower); + obj.set("matchIn", titleMatch ? "title" : contentMatch ? "content" : "summary"); + arr.add(obj); } return JSONUtil.createObj() @@ -131,24 +144,72 @@ public class WikiTool { .toString(); } + @Tool(description = """ + 追溯 Wiki 页面的来源原始文件。 + 查询指定页面是由哪些原始文档生成的，返回文件名、类型、路径等信息。 + 用于回答"这个内容出自哪篇文档"类的问题。 + """) + public String wiki_trace_source( + @ToolParam(description = "当前 Agent 的 ID") Long agentId, + @ToolParam(description = "页面标识符 (slug)") String slug) { + + if (slug == null || slug.isBlank()) { + return error("slug is required"); + } + + Long kbId = resolveKbId(agentId); + if (kbId == null) { + return error("No wiki knowledge base found for this agent"); + } + + WikiPageEntity page = pageService.getBySlug(kbId, slug); + if (page == null) { + return error("Page not found: " + slug); + } + + return JSONUtil.createObj() + .set("pageTitle", page.getTitle()) + .set("pageSlug", page.getSlug()) + .set("sourceFiles", resolveSourceFiles(page.getSourceRawIds())) + .toString(); + } + /** - * 校验 Agent 是否有权访问指定知识库 + * 通过 agentId 自动解析关联的知识库 ID + *

+ * 查找逻辑：Agent 专属 KB + 公共 KB（agent_id IS NULL），取第一个。 */ - private String checkAccess(Long agentId, Long kbId) { - WikiKnowledgeBaseEntity kb = kbService.getById(kbId); - if (kb == null) { - return error("Knowledge base not found: " + kbId); + private Long resolveKbId(Long agentId) { + List kbs = kbService.listByAgentId(agentId); + if (kbs.isEmpty()) { + // agentId 为 null 时也尝试查公共 KB + kbs = kbService.listAll(); } - // KB 绑定了 agent 时，必须提供匹配的 agentId - if (kb.getAgentId() != null) { - if (agentId == null) { - return error("Access denied: agentId is required for this knowledge base"); - } - if (!kb.getAgentId().equals(agentId)) { - return error("Access denied: knowledge base is not associated with this agent"); + return kbs.isEmpty() ? null : kbs.get(0).getId(); + } + + /** + * 将 sourceRawIds JSON 数组解析为原始文件信息列表 + */ + private JSONArray resolveSourceFiles(String sourceRawIdsJson) { + JSONArray result = new JSONArray(); + if (sourceRawIdsJson == null || sourceRawIdsJson.isBlank()) return result; + try { + List rawIds = new ObjectMapper().readValue(sourceRawIdsJson, new TypeReference>() {}); + for (Long rawId : rawIds) { + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw != null) { + result.add(JSONUtil.createObj() + .set("rawId", raw.getId()) + .set("title", raw.getTitle()) + .set("sourceType", raw.getSourceType()) + .set("sourcePath", raw.getSourcePath())); + } } + } catch (Exception e) { + log.warn("[WikiTool] Failed to resolve source files: {}", e.getMessage()); } - return null; + return result; } private String error(String message) { diff --git a/mateclaw-server/src/main/resources/db/tools-sync-mysql.sql b/mateclaw-server/src/main/resources/db/tools-sync-mysql.sql index 832c2732..ec6de9c0 100644 --- a/mateclaw-server/src/main/resources/db/tools-sync-mysql.sql +++ b/mateclaw-server/src/main/resources/db/tools-sync-mysql.sql @@ -59,5 +59,5 @@ VALUES (1000000016, 'SqlQueryTool', 'SQL 查询', '在外部数据源上执行 ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), builtin=VALUES(builtin), update_time=NOW(); INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) -VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、列出和搜索 Wiki 知识库中的结构化页面。支持 wiki_read_page、wiki_list_pages、wiki_search_pages 三个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0) +VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、搜索 Wiki 知识库中的结构化页面，并追溯原始来源文件。支持 wiki_read_page、wiki_list_pages、wiki_search_pages、wiki_trace_source 四个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), builtin=VALUES(builtin), update_time=NOW(); diff --git a/mateclaw-server/src/main/resources/db/tools-sync.sql b/mateclaw-server/src/main/resources/db/tools-sync.sql index 9f96e21f..33a8584c 100644 --- a/mateclaw-server/src/main/resources/db/tools-sync.sql +++ b/mateclaw-server/src/main/resources/db/tools-sync.sql @@ -63,4 +63,4 @@ VALUES (1000000016, 'SqlQueryTool', 'SQL 查询', '在外部数据源上执行 MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) KEY (id) -VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、列出和搜索 Wiki 知识库中的结构化页面。支持 wiki_read_page、wiki_list_pages、wiki_search_pages 三个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0); +VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、搜索 Wiki 知识库中的结构化页面，并追溯原始来源文件。支持 wiki_read_page、wiki_list_pages、wiki_search_pages、wiki_trace_source 四个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0); diff --git a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue index 2aa566e3..c56f8d30 100644 --- a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue +++ b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue @@ -264,6 +264,7 @@ async function handleScanDir() { .status-badge.pending { background: var(--mc-bg-sunken); color: var(--mc-text-tertiary); } .status-badge.processing { background: var(--mc-primary-bg); color: var(--mc-primary); } .status-badge.completed { background: rgba(90, 138, 90, 0.15); color: var(--mc-success); } +.status-badge.partial { background: rgba(217, 119, 87, 0.15); color: var(--mc-primary); } .status-badge.failed { background: var(--mc-danger-bg); color: var(--mc-danger); } /* Process button */