opensource/mateclaw
2
0
Fork 0
mirror of https://gitee.com/mateos/mateclaw.git synced 2026-09-13 03:13:41 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix(wiki): auto-resolve kbId, improve search and processing robustness

Browse Source
This commit is contained in:
matevip 2026-04-08 16:09:22 +08:00
parent 642360a773
commit f6102c4733
7 changed files with 187 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java
View File

@ -43,8 +43,12 @@ public class WikiContextService {
StringBuilder sb = new StringBuilder();
sb.append("\n\n## Wiki Knowledge Base\n\n");
sb.append("You have access to structured wiki knowledge bases. ");
sb.append("Use `wiki_read_page` tool to read full page content when needed.\n\n");
sb.append("You have access to structured wiki knowledge bases. The knowledge base is automatically resolved from your agentId.\n\n");
sb.append("Wiki tools (only need agentId + slug or query, NO kbId needed):\n");
sb.append("- `wiki_search_pages(agentId, query)` — full-text search across titles, summaries, and content\n");
sb.append("- `wiki_read_page(agentId, slug)` — read full page content with source file info\n");
sb.append("- `wiki_list_pages(agentId)` — list all pages with summaries\n");
sb.append("- `wiki_trace_source(agentId, slug)` — find which original documents a page was generated from\n\n");
int totalChars = 0;
int maxChars = properties.getMaxContextChars();
@ -57,21 +61,34 @@ public class WikiContextService {
if (kb.getDescription() != null && !kb.getDescription().isBlank()) {
sb.append(" — ").append(kb.getDescription());
}
sb.append("\n\n");
sb.append("Knowledge Base ID: `").append(kb.getId()).append("`\n\n");
sb.append("Available pages:\n");
sb.append(" (").append(pages.size()).append(" pages)\n\n");
for (WikiPageEntity page : pages) {
String line = "- **[[" + page.getTitle() + "]]** (`" + page.getSlug() + "`): "
+ (page.getSummary() != null ? page.getSummary() : "No summary") + "\n";
// 大量页面时只列标题索引（紧凑模式），节省 prompt 空间
boolean compact = pages.size() > 20;
if (totalChars + line.length() > maxChars) {
sb.append("- ... and more pages (use `wiki_list_pages` to see all)\n");
break;
if (compact) {
sb.append("Page index (use `wiki_search_pages` to find relevant pages, `wiki_read_page` to read full content):\n");
for (WikiPageEntity page : pages) {
String line = "- `" + page.getSlug() + "` — " + page.getTitle() + "\n";
if (totalChars + line.length() > maxChars) {
sb.append("- ... and ").append(pages.size()).append(" total pages (use `wiki_list_pages` to see all)\n");
break;
}
sb.append(line);
totalChars += line.length();
}
} else {
sb.append("Available pages:\n");
for (WikiPageEntity page : pages) {
String line = "- **[[" + page.getTitle() + "]]** (`" + page.getSlug() + "`): "
+ (page.getSummary() != null ? page.getSummary() : "No summary") + "\n";
if (totalChars + line.length() > maxChars) {
sb.append("- ... and more pages (use `wiki_list_pages` to see all)\n");
break;
}
sb.append(line);
totalChars += line.length();
}
sb.append(line);
totalChars += line.length();
}
sb.append("\n");

10
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java
View File

@ -43,6 +43,16 @@ public class WikiPageService {
return pages;
}
/**
* 列出知识库所有页面（含 content，用于全文搜索）
*/
public List<WikiPageEntity> listByKbIdWithContent(Long kbId) {
return pageMapper.selectList(
new LambdaQueryWrapper<WikiPageEntity>()
.eq(WikiPageEntity::getKbId, kbId)
.orderByAsc(WikiPageEntity::getTitle));
}
/**
* 列出页面摘要（用于上下文注入和 LLM 消化）
*/

52
mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java
View File

@ -75,16 +75,26 @@ public class WikiProcessingService {
}
// Phase 2: LLM 消化
int totalPages;
// result[0] = totalPages, result[1] = failedChunks, result[2] = totalChunks
int[] result;
if (textContent.length() > properties.getMaxChunkSize()) {
totalPages = processInChunks(kb, raw, textContent);
result = processInChunks(kb, raw, textContent);
} else {
totalPages = processChunk(kb, raw, textContent);
int pages = processChunk(kb, raw, textContent);
result = new int[]{pages, pages == 0 ? 1 : 0, 1};
}
int totalPages = result[0];
int failedChunks = result[1];
int totalChunks = result[2];
// Phase 3: 更新状态和计数
if (totalPages == 0) {
rawService.updateProcessingStatus(rawId, "failed", "No pages generated from LLM response");
} else if (failedChunks > 0) {
// 部分成功：有些 chunk 失败但有些产出了页面
rawService.updateProcessingStatus(rawId, "partial",
failedChunks + " of " + totalChunks + " chunks failed, " + totalPages + " pages generated");
} else {
rawService.updateProcessingStatus(rawId, "completed", null);
}
@ -120,13 +130,14 @@ public class WikiProcessingService {
/**
* 分块处理大文档
*
* @return 创建+更新的页面总数
* @return int[3]: [totalPages, failedChunks, totalChunks]
*/
private int processInChunks(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw, String text) {
private int[] processInChunks(WikiKnowledgeBaseEntity kb, WikiRawMaterialEntity raw, String text) {
int chunkSize = properties.getMaxChunkSize();
int overlap = 500; // 块间重叠
int start = 0;
int totalPages = 0;
int failedChunks = 0;
int chunkIndex = 0;
while (start < text.length()) {
@ -145,7 +156,17 @@ public class WikiProcessingService {
String chunk = text.substring(start, end);
log.info("[Wiki] Processing chunk {}: chars {}-{} of {}", chunkIndex, start, end, text.length());
totalPages += processChunk(kb, raw, chunk);
try {
totalPages += processChunk(kb, raw, chunk);
} catch (Exception e) {
failedChunks++;
// content_filter 错误：标注后继续处理其他 chunk
if (e.getMessage() != null && e.getMessage().contains("content_filter")) {
log.warn("[Wiki] Chunk {} blocked by content filter, skipping", chunkIndex);
} else {
log.warn("[Wiki] Chunk {} failed: {}", chunkIndex, e.getMessage());
}
}
start = end - overlap;
if (start < 0) start = 0;
@ -153,7 +174,7 @@ public class WikiProcessingService {
if (start <= previousStart) start = end;
chunkIndex++;
}
return totalPages;
return new int[]{totalPages, failedChunks, chunkIndex};
}
/**
@ -287,6 +308,8 @@ public class WikiProcessingService {
if (response == null || response.isBlank()) return null;
String cleaned = response.trim();
// 1. 剥离 markdown 代码块标记
if (cleaned.startsWith("```json")) {
cleaned = cleaned.substring(7);
} else if (cleaned.startsWith("```")) {
@ -297,9 +320,24 @@ public class WikiProcessingService {
}
cleaned = cleaned.trim();
// 2. 清洗控制字符（保留 \n \r \t），防止 LLM 输出含不可见字符导致 JSON 解析失败
cleaned = cleaned.replaceAll("[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F]", "");
// 3. 第一次尝试直接解析
try {
return objectMapper.readTree(cleaned);
} catch (Exception e) {
// 4. 如果整体不是 JSON，尝试提取第一个 JSON 对象块（LLM 可能在 JSON 前后加了说明文字）
int jsonStart = cleaned.indexOf("{");
int jsonEnd = cleaned.lastIndexOf("}");
if (jsonStart >= 0 && jsonEnd > jsonStart) {
String extracted = cleaned.substring(jsonStart, jsonEnd + 1);
try {
return objectMapper.readTree(extracted);
} catch (Exception e2) {
log.warn("[Wiki] Failed to parse extracted JSON block: {}", e2.getMessage());
}
}
log.warn("[Wiki] Failed to parse JSON response: {}", e.getMessage());
return null;
}

135
mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java
View File

@ -3,6 +3,8 @@ package vip.mate.wiki.tool;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONObject;
import cn.hutool.json.JSONUtil;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.ai.tool.annotation.Tool;
@ -10,8 +12,10 @@ import org.springframework.ai.tool.annotation.ToolParam;
import org.springframework.stereotype.Component;
import vip.mate.wiki.model.WikiKnowledgeBaseEntity;
import vip.mate.wiki.model.WikiPageEntity;
import vip.mate.wiki.model.WikiRawMaterialEntity;
import vip.mate.wiki.service.WikiKnowledgeBaseService;
import vip.mate.wiki.service.WikiPageService;
import vip.mate.wiki.service.WikiRawMaterialService;
import java.util.List;
@ -19,6 +23,7 @@ import java.util.List;
* Wiki 知识库工具
* <p>
* 供 Agent 在对话中按需读取 Wiki 页面内容。
* kbId 通过 agentId 自动解析，LLM 无需传递。
*
* @author MateClaw Team
*/
@ -29,23 +34,25 @@ public class WikiTool {
private final WikiPageService pageService;
private final WikiKnowledgeBaseService kbService;
private final WikiRawMaterialService rawService;
@Tool(description = """
读取 Wiki 知识库中指定页面的完整内容。
当系统提示词中的 Wiki 页面摘要不够详细时，使用此工具获取完整内容。
返回 Markdown 格式的页面内容，包含 [[双向链接]]。
返回 Markdown 格式的页面内容，包含 [[双向链接]] 和来源原始文件信息。
""")
public String wiki_read_page(
@ToolParam(description = "当前 Agent 的 ID") Long agentId,
@ToolParam(description = "知识库 ID") Long kbId,
@ToolParam(description = "页面标识符 (slug)") String slug) {
if (kbId == null || slug == null || slug.isBlank()) {
return error("kbId and slug are required");
if (slug == null || slug.isBlank()) {
return error("slug is required");
}
String accessError = checkAccess(agentId, kbId);
if (accessError != null) return accessError;
Long kbId = resolveKbId(agentId);
if (kbId == null) {
return error("No wiki knowledge base found for this agent");
}
WikiPageEntity page = pageService.getBySlug(kbId, slug);
if (page == null) {
@ -57,7 +64,8 @@ public class WikiTool {
.set("slug", page.getSlug())
.set("version", page.getVersion())
.set("lastUpdatedBy", page.getLastUpdatedBy())
.set("content", page.getContent());
.set("content", page.getContent())
.set("sourceFiles", resolveSourceFiles(page.getSourceRawIds()));
return result.toString();
}
@ -66,16 +74,13 @@ public class WikiTool {
返回页面列表，包含标题、slug 和摘要。
""")
public String wiki_list_pages(
@ToolParam(description = "当前 Agent 的 ID") Long agentId,
@ToolParam(description = "知识库 ID") Long kbId) {
@ToolParam(description = "当前 Agent 的 ID") Long agentId) {
Long kbId = resolveKbId(agentId);
if (kbId == null) {
return error("kbId is required");
return error("No wiki knowledge base found for this agent");
}
String accessError = checkAccess(agentId, kbId);
if (accessError != null) return accessError;
List<WikiPageEntity> pages = pageService.listSummaries(kbId);
JSONArray arr = new JSONArray();
for (WikiPageEntity page : pages) {
@ -94,33 +99,41 @@ public class WikiTool {
@Tool(description = """
在 Wiki 知识库中搜索页面。
按关键词搜索页面标题和摘要，返回匹配的页面列表。
按关键词搜索页面标题、摘要和正文内容，返回匹配的页面列表及其来源文件。
""")
public String wiki_search_pages(
@ToolParam(description = "当前 Agent 的 ID") Long agentId,
@ToolParam(description = "知识库 ID") Long kbId,
@ToolParam(description = "搜索关键词") String query) {
if (kbId == null || query == null || query.isBlank()) {
return error("kbId and query are required");
if (query == null || query.isBlank()) {
return error("query is required");
}
String accessError = checkAccess(agentId, kbId);
if (accessError != null) return accessError;
Long kbId = resolveKbId(agentId);
if (kbId == null) {
return error("No wiki knowledge base found for this agent");
}
String queryLower = query.toLowerCase();
List<WikiPageEntity> pages = pageService.listSummaries(kbId);
List<WikiPageEntity> matched = pages.stream()
List<WikiPageEntity> allPages = pageService.listByKbIdWithContent(kbId);
List<WikiPageEntity> matched = allPages.stream()
.filter(p -> (p.getTitle() != null && p.getTitle().toLowerCase().contains(queryLower))
|| (p.getSummary() != null && p.getSummary().toLowerCase().contains(queryLower)))
|| (p.getSummary() != null && p.getSummary().toLowerCase().contains(queryLower))
|| (p.getContent() != null && p.getContent().toLowerCase().contains(queryLower)))
.limit(20)
.toList();
JSONArray arr = new JSONArray();
for (WikiPageEntity page : matched) {
arr.add(JSONUtil.createObj()
JSONObject obj = JSONUtil.createObj()
.set("title", page.getTitle())
.set("slug", page.getSlug())
.set("summary", page.getSummary()));
.set("summary", page.getSummary())
.set("sourceFiles", resolveSourceFiles(page.getSourceRawIds()));
boolean titleMatch = page.getTitle() != null && page.getTitle().toLowerCase().contains(queryLower);
boolean contentMatch = page.getContent() != null && page.getContent().toLowerCase().contains(queryLower);
obj.set("matchIn", titleMatch ? "title" : contentMatch ? "content" : "summary");
arr.add(obj);
}
return JSONUtil.createObj()
@ -131,24 +144,72 @@ public class WikiTool {
.toString();
}
@Tool(description = """
追溯 Wiki 页面的来源原始文件。
查询指定页面是由哪些原始文档生成的，返回文件名、类型、路径等信息。
用于回答"这个内容出自哪篇文档"类的问题。
""")
public String wiki_trace_source(
@ToolParam(description = "当前 Agent 的 ID") Long agentId,
@ToolParam(description = "页面标识符 (slug)") String slug) {
if (slug == null || slug.isBlank()) {
return error("slug is required");
}
Long kbId = resolveKbId(agentId);
if (kbId == null) {
return error("No wiki knowledge base found for this agent");
}
WikiPageEntity page = pageService.getBySlug(kbId, slug);
if (page == null) {
return error("Page not found: " + slug);
}
return JSONUtil.createObj()
.set("pageTitle", page.getTitle())
.set("pageSlug", page.getSlug())
.set("sourceFiles", resolveSourceFiles(page.getSourceRawIds()))
.toString();
}
/**
* 校验 Agent 是否有权访问指定知识库
* 通过 agentId 自动解析关联的知识库 ID
* <p>
* 查找逻辑：Agent 专属 KB + 公共 KB（agent_id IS NULL），取第一个。
*/
private String checkAccess(Long agentId, Long kbId) {
WikiKnowledgeBaseEntity kb = kbService.getById(kbId);
if (kb == null) {
return error("Knowledge base not found: " + kbId);
private Long resolveKbId(Long agentId) {
List<WikiKnowledgeBaseEntity> kbs = kbService.listByAgentId(agentId);
if (kbs.isEmpty()) {
// agentId 为 null 时也尝试查公共 KB
kbs = kbService.listAll();
}
// KB 绑定了 agent 时，必须提供匹配的 agentId
if (kb.getAgentId() != null) {
if (agentId == null) {
return error("Access denied: agentId is required for this knowledge base");
}
if (!kb.getAgentId().equals(agentId)) {
return error("Access denied: knowledge base is not associated with this agent");
return kbs.isEmpty() ? null : kbs.get(0).getId();
}
/**
* 将 sourceRawIds JSON 数组解析为原始文件信息列表
*/
private JSONArray resolveSourceFiles(String sourceRawIdsJson) {
JSONArray result = new JSONArray();
if (sourceRawIdsJson == null || sourceRawIdsJson.isBlank()) return result;
try {
List<Long> rawIds = new ObjectMapper().readValue(sourceRawIdsJson, new TypeReference<List<Long>>() {});
for (Long rawId : rawIds) {
WikiRawMaterialEntity raw = rawService.getById(rawId);
if (raw != null) {
result.add(JSONUtil.createObj()
.set("rawId", raw.getId())
.set("title", raw.getTitle())
.set("sourceType", raw.getSourceType())
.set("sourcePath", raw.getSourcePath()));
}
}
} catch (Exception e) {
log.warn("[WikiTool] Failed to resolve source files: {}", e.getMessage());
}
return null;
return result;
}
private String error(String message) {

2
mateclaw-server/src/main/resources/db/tools-sync-mysql.sql
View File

@ -59,5 +59,5 @@ VALUES (1000000016, 'SqlQueryTool', 'SQL 查询', '在外部数据源上执行
ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), builtin=VALUES(builtin), update_time=NOW();
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、列出和搜索 Wiki 知识库中的结构化页面。支持 wiki_read_page、wiki_list_pages、wiki_search_pages 三个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0)
VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、搜索 Wiki 知识库中的结构化页面，并追溯原始来源文件。支持 wiki_read_page、wiki_list_pages、wiki_search_pages、wiki_trace_source 四个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0)
ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), builtin=VALUES(builtin), update_time=NOW();

2
mateclaw-server/src/main/resources/db/tools-sync.sql
View File

@ -63,4 +63,4 @@ VALUES (1000000016, 'SqlQueryTool', 'SQL 查询', '在外部数据源上执行
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
KEY (id)
VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、列出和搜索 Wiki 知识库中的结构化页面。支持 wiki_read_page、wiki_list_pages、wiki_search_pages 三个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0);
VALUES (1000000017, 'WikiTool', 'Wiki 知识库', '读取、搜索 Wiki 知识库中的结构化页面，并追溯原始来源文件。支持 wiki_read_page、wiki_list_pages、wiki_search_pages、wiki_trace_source 四个工具。', 'builtin', 'wikiTool', '📚', TRUE, TRUE, NOW(), NOW(), 0);

1
mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue
View File

@ -264,6 +264,7 @@ async function handleScanDir() {
.status-badge.pending { background: var(--mc-bg-sunken); color: var(--mc-text-tertiary); }
.status-badge.processing { background: var(--mc-primary-bg); color: var(--mc-primary); }
.status-badge.completed { background: rgba(90, 138, 90, 0.15); color: var(--mc-success); }
.status-badge.partial { background: rgba(217, 119, 87, 0.15); color: var(--mc-primary); }
.status-badge.failed { background: var(--mc-danger-bg); color: var(--mc-danger); }
/* Process button */