diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 5476857b..db166183 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -132,6 +132,7 @@ public class AgentGraphBuilder { private final vip.mate.i18n.I18nService i18nService; private final vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker; private final vip.mate.llm.chatmodel.ProviderChatModelFactory chatModelFactory; + private final vip.mate.llm.failover.AvailableProviderPool providerPool; /** * 根据 AgentEntity 构建完整的 Agent 实例 @@ -189,12 +190,12 @@ public class AgentGraphBuilder { BaseAgent agent; boolean toolCallingEnabled; if ("plan_execute".equals(entity.getAgentType())) { - agent = buildPlanExecuteAgent(toolSet, runtimeModel, maxIter); + agent = buildPlanExecuteAgent(toolSet, runtimeModel, maxIter, entity.getId()); toolCallingEnabled = true; log.info("Built StateGraph Plan-Execute agent: {} (maxIterations={}, tools={}, protocol={})", entity.getName(), maxIter, toolSet.size(), protocol.getId()); } else { - agent = buildReActAgent(toolSet, runtimeModel, maxIter); + agent = buildReActAgent(toolSet, runtimeModel, maxIter, entity.getId()); // StateGraph 路径下工具调用由 ActionNode 控制,始终启用 toolCallingEnabled = true; log.info("Built StateGraph ReAct agent: {} (maxIterations={}, tools={}, protocol={})", @@ -236,34 +237,51 @@ public class AgentGraphBuilder { // ==================== Agent 构建方法 ==================== StateGraphReActAgent buildReActAgent(AgentToolSet toolSet, ModelConfigEntity runtimeModel, int maxIter) { + return buildReActAgent(toolSet, runtimeModel, maxIter, null); + } + + StateGraphReActAgent buildReActAgent(AgentToolSet toolSet, ModelConfigEntity runtimeModel, + int maxIter, Long agentId) { ChatModel chatModel = buildRuntimeChatModel(runtimeModel); ChatClient chatClient = ChatClient.create(chatModel); String reasoningEffort = resolveReasoningEffortForModel(runtimeModel); - CompiledGraph compiledGraph = buildReActGraph(toolSet, chatModel, maxIter, reasoningEffort, runtimeModel); + CompiledGraph compiledGraph = buildReActGraph(toolSet, chatModel, maxIter, reasoningEffort, runtimeModel, agentId); return new StateGraphReActAgent(chatClient, conversationService, compiledGraph, chatModel, conversationWindowManager); } StateGraphPlanExecuteAgent buildPlanExecuteAgent(AgentToolSet toolSet, ModelConfigEntity runtimeModel, int maxIter) { + return buildPlanExecuteAgent(toolSet, runtimeModel, maxIter, null); + } + + StateGraphPlanExecuteAgent buildPlanExecuteAgent(AgentToolSet toolSet, ModelConfigEntity runtimeModel, + int maxIter, Long agentId) { ChatModel chatModel = buildRuntimeChatModel(runtimeModel); ChatClient chatClient = ChatClient.create(chatModel); String reasoningEffort = resolveReasoningEffortForModel(runtimeModel); - CompiledGraph graph = buildPlanExecuteGraph(toolSet, chatModel, maxIter, reasoningEffort, runtimeModel); + CompiledGraph graph = buildPlanExecuteGraph(toolSet, chatModel, maxIter, reasoningEffort, runtimeModel, agentId); return new StateGraphPlanExecuteAgent(chatClient, conversationService, graph, planningService, chatModel, conversationWindowManager); } CompiledGraph buildPlanExecuteGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort) { - return buildPlanExecuteGraph(toolSet, chatModel, maxIterations, reasoningEffort, null); + return buildPlanExecuteGraph(toolSet, chatModel, maxIterations, reasoningEffort, null, null); } CompiledGraph buildPlanExecuteGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort, ModelConfigEntity primaryModelConfig) { + return buildPlanExecuteGraph(toolSet, chatModel, maxIterations, reasoningEffort, primaryModelConfig, null); + } + + CompiledGraph buildPlanExecuteGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, + String reasoningEffort, ModelConfigEntity primaryModelConfig, + Long agentId) { try { - List fallbackChain = buildFallbackChain(primaryModelConfig); + List fallbackChain = buildFallbackChain(primaryModelConfig, agentId); NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper( streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker, - primaryModelConfig != null ? primaryModelConfig.getProvider() : null); + primaryModelConfig != null ? primaryModelConfig.getProvider() : null, + providerPool); ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry); PlanGenerationNode planGenerationNode = new PlanGenerationNode(chatModel, planningService, streamingHelper, conversationWindowManager, toolSet); StepExecutionNode stepExecutionNode = new StepExecutionNode(chatModel, toolSet, executor, planningService, streamTracker, reasoningEffort, streamingHelper, conversationWindowManager); @@ -355,16 +373,23 @@ public class AgentGraphBuilder { } CompiledGraph buildReActGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort) { - return buildReActGraph(toolSet, chatModel, maxIterations, reasoningEffort, null); + return buildReActGraph(toolSet, chatModel, maxIterations, reasoningEffort, null, null); } CompiledGraph buildReActGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, String reasoningEffort, ModelConfigEntity primaryModelConfig) { + return buildReActGraph(toolSet, chatModel, maxIterations, reasoningEffort, primaryModelConfig, null); + } + + CompiledGraph buildReActGraph(AgentToolSet toolSet, ChatModel chatModel, int maxIterations, + String reasoningEffort, ModelConfigEntity primaryModelConfig, + Long agentId) { try { - List fallbackChain = buildFallbackChain(primaryModelConfig); + List fallbackChain = buildFallbackChain(primaryModelConfig, agentId); NodeStreamingChatHelper streamingHelper = new NodeStreamingChatHelper( streamTracker, fallbackChain, llmCacheMetricsAggregator, providerHealthTracker, - primaryModelConfig != null ? primaryModelConfig.getProvider() : null); + primaryModelConfig != null ? primaryModelConfig.getProvider() : null, + providerPool); ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, toolGuardService, approvalService, streamTracker, toolTimeoutProperties, toolResultStorage, toolConcurrencyRegistry); ReasoningNode reasoningNode = new ReasoningNode(chatModel, toolSet, reasoningEffort, streamingHelper, conversationWindowManager, streamTracker, 0, wikiContextService); ActionNode actionNode = new ActionNode(executor, streamTracker); @@ -512,7 +537,7 @@ public class AgentGraphBuilder { } /** - * build the full multi-provider failover chain for a primary + * RFC-009: build the full multi-provider failover chain for a primary * model. Providers are read from {@code mate_model_provider} ordered by * {@code fallback_priority ASC} (positive values only), each resolved to * its default {@link ModelConfigEntity} and turned into a {@link ChatModel} @@ -526,13 +551,27 @@ public class AgentGraphBuilder { *

The primary model is excluded from the chain when its provider + * model name matches a chain entry. Previously only reference equality * was checked, which meant a DashScope-primary deployment ended up with - * {@code null} fallback — the case this design targets.

+ * {@code null} fallback — exactly the case RFC-009 targets.

* * @param primaryModelConfig the {@code ModelConfigEntity} used to build * the primary model; used to identity-filter the chain * @return ordered, possibly-empty list of fallback {@link ChatModel}s */ List buildFallbackChain(ModelConfigEntity primaryModelConfig) { + return buildFallbackChain(primaryModelConfig, null); + } + + /** + * RFC-009 PR-3 overload: when {@code agentId} is non-null, the agent's + * {@code mate_agent_provider_preference} rows bias the chain order — listed + * providers come first in their declared {@code sort_order}, then the + * remaining providers fall in by global {@code fallback_priority} ascending, + * tie-broken by provider id alphabetically. {@code null} agentId keeps the + * pre-PR-3 ordering (pure global priority) — that's the path for legacy + * callers and tests. + */ + List buildFallbackChain(ModelConfigEntity primaryModelConfig, + Long agentId) { List providers; try { providers = modelProviderService.listFallbackChain(); @@ -547,8 +586,29 @@ public class AgentGraphBuilder { String primaryProviderId = primaryModelConfig != null ? primaryModelConfig.getProvider() : null; String primaryModelName = primaryModelConfig != null ? primaryModelConfig.getModelName() : null; + // RFC-009 PR-3: bias by agent preferences (if any). Listed providers win + // their declared order; everything else keeps the global priority order. + List preferred = agentId == null + ? java.util.Collections.emptyList() + : agentBindingService.getPreferredProviderIds(agentId); + if (!preferred.isEmpty()) { + providers = reorderByPreferences(providers, preferred); + log.debug("[LlmFailover] agent={} preferences={} -> chain head reordered", agentId, preferred); + } + List chain = new ArrayList<>(); for (ModelProviderEntity p : providers) { + // RFC-009 Phase 4: skip providers known-bad at build time. This is + // a perf optimization (one fewer ChatModel to construct + one + // fewer round-trip on the chain walk); the runtime walker in + // NodeStreamingChatHelper re-checks pool membership per request, + // so a provider that re-enters the pool later still gets used + // (the graph is rebuilt on ModelConfigChangedEvent). + if (providerPool != null && !providerPool.contains(p.getProviderId())) { + log.debug("[LlmFailover] skipping provider {} — not in available pool", + p.getProviderId()); + continue; + } ModelConfigEntity fallbackConfig; try { fallbackConfig = modelConfigService.getDefaultModelByProvider(p.getProviderId()); @@ -584,6 +644,36 @@ public class AgentGraphBuilder { return chain; } + /** + * Reorder a provider list by an agent's preference list. Listed provider + * ids come first in their preference order; any provider not in the + * preference list keeps its original position relative to other unlisted + * providers (stable partition). Preference entries that don't match any + * actual provider are silently dropped. + */ + /** Package-private for unit testing — see {@code AgentGraphBuilderPreferenceTest}. */ + static List reorderByPreferences(List providers, + List preferredOrder) { + Map byId = new java.util.LinkedHashMap<>(); + for (ModelProviderEntity p : providers) { + byId.put(p.getProviderId(), p); + } + List reordered = new ArrayList<>(providers.size()); + Set placed = new java.util.HashSet<>(); + for (String prefId : preferredOrder) { + ModelProviderEntity p = byId.get(prefId); + if (p != null && placed.add(prefId)) { + reordered.add(p); + } + } + for (ModelProviderEntity p : providers) { + if (placed.add(p.getProviderId())) { + reordered.add(p); + } + } + return reordered; + } + /** * @deprecated use {@link #buildFallbackChain(ModelConfigEntity)} — the * single-fallback variant cannot represent an ordered chain and only diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/controller/AgentBindingController.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/controller/AgentBindingController.java index 39b28594..4e0a96f5 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/binding/controller/AgentBindingController.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/controller/AgentBindingController.java @@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.*; import vip.mate.agent.AgentService; +import vip.mate.agent.binding.model.AgentProviderPreference; import vip.mate.agent.binding.model.AgentSkillBinding; import vip.mate.agent.binding.model.AgentToolBinding; import vip.mate.agent.binding.service.AgentBindingService; @@ -102,6 +103,33 @@ public class AgentBindingController { return R.ok(); } + // ==================== Provider Preferences (RFC-009 PR-3) ==================== + + @Operation(summary = "获取 Agent 的偏好 Provider 顺序") + @GetMapping("/provider-preferences") + @RequireWorkspaceRole("viewer") + public R> listProviderPreferences( + @PathVariable Long agentId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyAgentWorkspace(agentId, workspaceId); + return R.ok(bindingService.listProviderPreferences(agentId)); + } + + @Operation(summary = "批量设置 Agent 的偏好 Provider 顺序(替换模式)") + @PutMapping("/provider-preferences") + @RequireWorkspaceRole("member") + public R setProviderPreferences( + @PathVariable Long agentId, + @RequestBody List providerIds, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyAgentWorkspace(agentId, workspaceId); + bindingService.setProviderPreferences(agentId, providerIds); + agentService.invalidateAgentCache(agentId); + auditEventService.record("UPDATE", "AGENT_PROVIDER_PREF", String.valueOf(agentId), + "providers=" + providerIds.size(), null); + return R.ok(); + } + // ==================== Workspace Verification ==================== private void verifyAgentWorkspace(Long agentId, Long headerWorkspaceId) { diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/model/AgentProviderPreference.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/model/AgentProviderPreference.java new file mode 100644 index 00000000..81eeebe7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/model/AgentProviderPreference.java @@ -0,0 +1,44 @@ +package vip.mate.agent.binding.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * RFC-009 Phase 4 PR-3 — agent → preferred provider routing hint. + * + *

An agent with zero rows here uses the global fallback chain order + * (no behavior change from pre-PR-3 deployments). When rows exist, + * {@code AgentGraphBuilder.buildFallbackChain} sorts those provider ids + * to the front by ascending {@code sortOrder}; non-listed providers + * follow in their global priority order.

+ * + *

Pool/cooldown gating still applies: a preferred provider that is + * HARD-removed or cooling down is still skipped by the runtime walker.

+ */ +@Data +@TableName("mate_agent_provider_preference") +public class AgentProviderPreference { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long agentId; + + /** Provider id (matches {@code mate_model_provider.provider_id}). */ + private String providerId; + + /** Lower wins. Two rows with the same value tie-break on provider_id alphabetically. */ + private Integer sortOrder; + + private Boolean enabled; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/repository/AgentProviderPreferenceMapper.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/repository/AgentProviderPreferenceMapper.java new file mode 100644 index 00000000..8e6cac4c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/repository/AgentProviderPreferenceMapper.java @@ -0,0 +1,9 @@ +package vip.mate.agent.binding.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.agent.binding.model.AgentProviderPreference; + +@Mapper +public interface AgentProviderPreferenceMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java index d5fab91c..f0c3532c 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java @@ -4,8 +4,10 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import vip.mate.agent.binding.model.AgentProviderPreference; import vip.mate.agent.binding.model.AgentSkillBinding; import vip.mate.agent.binding.model.AgentToolBinding; +import vip.mate.agent.binding.repository.AgentProviderPreferenceMapper; import vip.mate.agent.binding.repository.AgentSkillBindingMapper; import vip.mate.agent.binding.repository.AgentToolBindingMapper; @@ -30,6 +32,7 @@ public class AgentBindingService { private final AgentSkillBindingMapper skillBindingMapper; private final AgentToolBindingMapper toolBindingMapper; + private final AgentProviderPreferenceMapper providerPreferenceMapper; // ==================== Skill Bindings ==================== @@ -167,4 +170,52 @@ public class AgentBindingService { } } } + + // ==================== Provider Preferences (RFC-009 PR-3) ==================== + + /** Raw rows for the agent edit form. Sorted by sort_order ascending. */ + public List listProviderPreferences(Long agentId) { + return providerPreferenceMapper.selectList( + new LambdaQueryWrapper() + .eq(AgentProviderPreference::getAgentId, agentId) + .orderByAsc(AgentProviderPreference::getSortOrder)); + } + + /** + * Ordered list of provider ids the agent prefers, lowest sort_order + * first. Disabled rows are filtered out. Empty list means "no + * preference — fall back to the global chain order". + * + *

Used by {@code AgentGraphBuilder.buildFallbackChain} to bias the + * fallback chain order per agent.

+ */ + public List getPreferredProviderIds(Long agentId) { + if (agentId == null) return Collections.emptyList(); + return listProviderPreferences(agentId).stream() + .filter(p -> Boolean.TRUE.equals(p.getEnabled())) + .map(AgentProviderPreference::getProviderId) + .collect(Collectors.toList()); + } + + /** + * Replace the full preference list for an agent. {@code providerIds} + * is the new ordered preference (index 0 = highest preference). + * Empty / null list clears all preferences for the agent. + */ + public void setProviderPreferences(Long agentId, List providerIds) { + providerPreferenceMapper.delete( + new LambdaQueryWrapper() + .eq(AgentProviderPreference::getAgentId, agentId)); + if (providerIds == null) return; + int order = 0; + for (String providerId : providerIds) { + if (providerId == null || providerId.isBlank()) continue; + AgentProviderPreference row = new AgentProviderPreference(); + row.setAgentId(agentId); + row.setProviderId(providerId.trim()); + row.setSortOrder(order++); + row.setEnabled(true); + providerPreferenceMapper.insert(row); + } + } } diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java index a9cfe105..97716f09 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java @@ -47,7 +47,7 @@ public class NodeStreamingChatHelper { /** * Ordered fallback chain tried after the primary model exhausts retries. * Each entry is attempted once (no retry); the first successful response - * wins. Empty list disables fallover entirely. + * wins. Empty list disables fallover entirely. See RFC-009. * *

Stored as {@link vip.mate.llm.failover.FallbackEntry} (providerId + * ChatModel) so the chain walker can consult {@link vip.mate.llm.failover.ProviderHealthTracker} @@ -71,17 +71,26 @@ public class NodeStreamingChatHelper { */ private final String primaryProviderId; + /** + * RFC-009 Phase 4: membership gate for usable providers. A provider is + * removed from the pool on HARD errors (AUTH_ERROR / BILLING / + * MODEL_NOT_FOUND) so subsequent requests skip it entirely without + * burning a round-trip. {@code null} disables the gate (legacy callers, + * tests) — every provider then counts as in-pool (fail-open). + */ + private final vip.mate.llm.failover.AvailableProviderPool providerPool; + public NodeStreamingChatHelper(ChatStreamTracker streamTracker) { - this(streamTracker, List.of(), null, null, null); + this(streamTracker, List.of(), null, null, null, null); } /** * @deprecated use the full constructor — a single fallback cannot - * express the ordered multi-provider chain + * express the ordered multi-provider chain from RFC-009. */ @Deprecated public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel) { - this(streamTracker, wrap(fallbackModel), null, null, null); + this(streamTracker, wrap(fallbackModel), null, null, null, null); } /** @@ -90,7 +99,7 @@ public class NodeStreamingChatHelper { @Deprecated public NodeStreamingChatHelper(ChatStreamTracker streamTracker, ChatModel fallbackModel, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) { - this(streamTracker, wrap(fallbackModel), cacheMetrics, null, null); + this(streamTracker, wrap(fallbackModel), cacheMetrics, null, null, null); } /** @@ -100,7 +109,7 @@ public class NodeStreamingChatHelper { public NodeStreamingChatHelper(ChatStreamTracker streamTracker, List fallbackChain, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics) { - this(streamTracker, fallbackChain, cacheMetrics, null, null); + this(streamTracker, fallbackChain, cacheMetrics, null, null, null); } /** @@ -112,19 +121,41 @@ public class NodeStreamingChatHelper { List fallbackChain, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics, vip.mate.llm.failover.ProviderHealthTracker healthTracker) { - this(streamTracker, fallbackChain, cacheMetrics, healthTracker, null); + this(streamTracker, fallbackChain, cacheMetrics, healthTracker, null, null); } + /** + * Constructor that wires health tracker + primary provider id but leaves + * the {@link vip.mate.llm.failover.AvailableProviderPool} disabled. Kept + * so existing tests (e.g. {@code NodeStreamingChatHelperFailoverTest}) + * compile unchanged — they don't exercise the pool gate. + */ public NodeStreamingChatHelper(ChatStreamTracker streamTracker, List fallbackChain, vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics, vip.mate.llm.failover.ProviderHealthTracker healthTracker, String primaryProviderId) { + this(streamTracker, fallbackChain, cacheMetrics, healthTracker, primaryProviderId, null); + } + + /** + * Full constructor — preferred for production wiring. The + * {@link vip.mate.llm.failover.AvailableProviderPool} hookup gates both + * the primary short-circuit and the fallback walker; passing {@code null} + * runs in fail-open mode (every provider counted as in-pool). + */ + public NodeStreamingChatHelper(ChatStreamTracker streamTracker, + List fallbackChain, + vip.mate.llm.cache.LlmCacheMetricsAggregator cacheMetrics, + vip.mate.llm.failover.ProviderHealthTracker healthTracker, + String primaryProviderId, + vip.mate.llm.failover.AvailableProviderPool providerPool) { this.streamTracker = streamTracker; this.fallbackChain = fallbackChain == null ? List.of() : List.copyOf(fallbackChain); this.cacheMetrics = cacheMetrics; this.healthTracker = healthTracker; this.primaryProviderId = primaryProviderId; + this.providerPool = providerPool; } private static List wrap(ChatModel m) { @@ -144,6 +175,47 @@ public class NodeStreamingChatHelper { else healthTracker.recordFailure(primaryProviderId); } + /** + * RFC-009 Phase 4 — map an {@link ErrorType} to the matching pool + * {@link vip.mate.llm.failover.AvailableProviderPool.RemovalSource} for + * HARD failures (AUTH / BILLING / MODEL_NOT_FOUND). Returns {@code null} + * for SOFT errors and benign types — those keep the provider in-pool and + * are handled by {@link vip.mate.llm.failover.ProviderHealthTracker}'s + * cooldown instead. + */ + private static vip.mate.llm.failover.AvailableProviderPool.RemovalSource hardRemovalSource(ErrorType type) { + if (type == null) return null; + return switch (type) { + case AUTH_ERROR -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.AUTH_ERROR; + case BILLING -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.BILLING; + case MODEL_NOT_FOUND -> vip.mate.llm.failover.AvailableProviderPool.RemovalSource.MODEL_NOT_FOUND; + default -> null; + }; + } + + /** Convenience: pool-aware membership check. Null pool means fail-open (everyone in). */ + private boolean inPool(String providerId) { + return providerPool == null || providerId == null || providerPool.contains(providerId); + } + + /** + * Remove the given provider from the pool if {@code errorType} is HARD + * (AUTH_ERROR / BILLING / MODEL_NOT_FOUND). No-op when the pool is + * disabled, the provider id is unknown, or the error is SOFT. + */ + private void removeFromPool(String providerId, ErrorType errorType, String message) { + if (providerPool == null || providerId == null) return; + var source = hardRemovalSource(errorType); + if (source == null) return; + providerPool.remove(providerId, source, message != null ? message : errorType.name()); + } + + /** Defensively re-affirm pool membership after a successful call. Idempotent + cheap. */ + private void addToPool(String providerId) { + if (providerPool == null || providerId == null) return; + providerPool.add(providerId); + } + /** * 流式调用 LLM 并实时广播增量内容 * @@ -235,7 +307,7 @@ public class NodeStreamingChatHelper { || msg.contains("thinking block")) { return ErrorType.THINKING_BLOCK_ERROR; } - // BILLING — payment / quota exhausted. Distinct from AUTH because + // RFC-009 P3.2: BILLING — payment / quota exhausted. Distinct from AUTH because // a different provider may have credits, so we should fall back instead of // terminating the call. Both OpenAI ("insufficient_quota") and Anthropic // ("credit balance is too low") use these phrases in 402-class responses. @@ -246,7 +318,7 @@ public class NodeStreamingChatHelper { || msg.contains("quota exceeded") || msg.contains("Quota exceeded")) { return ErrorType.BILLING; } - // MODEL_NOT_FOUND — provider rejects the requested model id. + // RFC-009 P3.2: MODEL_NOT_FOUND — provider rejects the requested model id. // Includes DashScope's "[InvalidParameter] url error, please check url" // (https://help.aliyun.com/zh/model-studio/error-code#error-url) which despite // the wording is the provider rejecting an unknown/unsupported model id on @@ -300,20 +372,23 @@ public class NodeStreamingChatHelper { throw new CancellationException("Stream stopped by user"); } - // if the primary's provider is in cooldown (3+ recent - // consecutive failures within the cooldown window), skip the 5-retry - // primary loop entirely and head straight to the fallback chain. - // Without this short-circuit a degraded primary forces every LLM - // call in the conversation to wait through the full backoff. - boolean primarySkipped = primaryProviderId != null + // RFC-009 P3.1 + Phase 4: short-circuit the primary retry loop in two cases. + // (a) primary is in cooldown (P3.3) — soft, transient + // (b) primary was HARD-removed from the pool (Phase 4) — auth/billing/missing model + // Either way, retrying the same model wastes seconds; head straight to fallback. + boolean primaryInCooldown = primaryProviderId != null && healthTracker != null && healthTracker.isInCooldown(primaryProviderId); + boolean primaryOutOfPool = primaryProviderId != null && !inPool(primaryProviderId); + boolean primarySkipped = primaryInCooldown || primaryOutOfPool; if (primarySkipped) { - log.warn("[{}] Primary provider={} is in cooldown — skipping straight to fallback chain", - phase, primaryProviderId); + String reason = primaryOutOfPool ? "removed from pool" : "in cooldown"; + log.warn("[{}] Primary provider={} {} — skipping straight to fallback chain", + phase, primaryProviderId, reason); if (broadcast) { broadcastDelta(conversationId, "warning", - buildDeltaJson("主模型暂时不可用(冷却中),直接尝试备选模型...")); + buildDeltaJson("主模型暂时不可用(" + (primaryOutOfPool ? "已下线" : "冷却中") + + "),直接尝试备选模型...")); } } @@ -334,9 +409,10 @@ public class NodeStreamingChatHelper { if (lastResult.errorType() == ErrorType.AUTH_ERROR) { log.warn("[{}] Primary auth failed — skipping same-model retries, handing off to fallback chain", phase); recordPrimary(false); + removeFromPool(primaryProviderId, ErrorType.AUTH_ERROR, lastResult.errorMessage()); break; } - // BILLING / MODEL_NOT_FOUND — provider-side hard failures + // RFC-009 P3.2: BILLING / MODEL_NOT_FOUND — provider-side hard failures // that won't change on retry. Skip to fallback chain (a different // provider may have credits, or the model name may be valid there). if (lastResult.errorType() == ErrorType.BILLING @@ -344,6 +420,7 @@ public class NodeStreamingChatHelper { log.warn("[{}] Primary error={} — skipping same-model retries, handing off to fallback chain", phase, lastResult.errorType()); recordPrimary(false); + removeFromPool(primaryProviderId, lastResult.errorType(), lastResult.errorMessage()); break; } // CLIENT_ERROR (400 Bad Request): 不重试(参数/格式错误重试也不会变) @@ -359,7 +436,7 @@ public class NodeStreamingChatHelper { if (lastResult.errorType() == ErrorType.THINKING_BLOCK_ERROR) { return lastResult; // 已经重试过了 } - // EMPTY_RESPONSE — break the primary-retry loop and fall through to + // RFC-009: EMPTY_RESPONSE — break the primary-retry loop and fall through to // the fallback chain. Retrying the same model that returned nothing is rarely // productive; a different provider has a better chance of succeeding. if (lastResult.errorType() == ErrorType.EMPTY_RESPONSE) { @@ -370,6 +447,7 @@ public class NodeStreamingChatHelper { // 成功 if (lastResult.errorMessage() == null || lastResult.errorType() == ErrorType.NONE) { recordPrimary(true); + addToPool(primaryProviderId); return lastResult; } // Any other non-null errored result with a classified type that doStreamCall @@ -390,12 +468,21 @@ public class NodeStreamingChatHelper { // Each fallback gets a single shot (no retry); first successful result wins. // Same-instance entries (e.g., primary accidentally included in the chain) // are skipped so we don't re-try the exact model that just failed. - // Providers in cooldown are also skipped so a known-bad + // Providers in cooldown (RFC-009 P3.3) are also skipped so a known-bad // provider doesn't add latency to every conversation turn. for (int i = 0; i < fallbackChain.size(); i++) { vip.mate.llm.failover.FallbackEntry entry = fallbackChain.get(i); ChatModel fallback = entry.chatModel(); if (fallback == chatModel) continue; + // RFC-009 Phase 4 — pool gate (the real runtime fence). A provider + // HARD-removed earlier (or by another conversation) must not even + // be attempted here. Build-time filtering is best-effort; this is + // the one that matters when pool state changes mid-conversation. + if (!inPool(entry.providerId())) { + log.info("[{}] Skipping fallback {}/{} provider={} — not in pool", + phase, i + 1, fallbackChain.size(), entry.providerId()); + continue; + } if (healthTracker != null && healthTracker.isInCooldown(entry.providerId())) { log.info("[{}] Skipping fallback {}/{} provider={} — in cooldown", phase, i + 1, fallbackChain.size(), entry.providerId()); @@ -417,10 +504,15 @@ public class NodeStreamingChatHelper { && fallbackResult.errorType() == ErrorType.NONE && fallbackResult.errorMessage() == null) { if (healthTracker != null) healthTracker.recordSuccess(entry.providerId()); + addToPool(entry.providerId()); return fallbackResult; } if (healthTracker != null) healthTracker.recordFailure(entry.providerId()); if (fallbackResult != null) { + // RFC-009 Phase 4: HARD errors evict from the pool so later + // walks skip this provider outright. SOFT errors keep it + // in-pool and let the tracker's cooldown absorb the blip. + removeFromPool(entry.providerId(), fallbackResult.errorType(), fallbackResult.errorMessage()); lastResult = fallbackResult; // remember most recent to report if the whole chain fails } } @@ -438,7 +530,7 @@ public class NodeStreamingChatHelper { boolean broadcast, int attempt) { if (attempt > 0) { long delay = Math.min(BACKOFF_BASE_MS * (1L << (attempt - 1)), BACKOFF_CAP_MS); - // 加入 jitter 防止雷群效应(a comparable reference runtime 风格) + // 加入 jitter 防止雷群效应(Hermes 风格) delay += ThreadLocalRandom.current().nextLong(0, Math.max(1, delay / 2)); delay = Math.min(delay, BACKOFF_CAP_MS); log.warn("[{}] Retry attempt {}/{} after {}ms for conversation {}", @@ -661,7 +753,7 @@ public class NodeStreamingChatHelper { // warning 已在 dispose 时广播,无需重复 } - // guard against silent empty responses. Some providers return + // RFC-009: guard against silent empty responses. Some providers return // HTTP 200 with an empty body under soft-failure conditions (rate-limit // capacity, context filter, upstream overload). Treat this as a failure // signal so streamCallInternal can hand off to the fallback chain. @@ -931,14 +1023,14 @@ public class NodeStreamingChatHelper { /** Thinking 块错误(旧消息中的 thinking block 不可修改)— 可剥离后单次重试 */ THINKING_BLOCK_ERROR, /** - * LLM returned no content, no thinking, and no tool calls. + * RFC-009: LLM returned no content, no thinking, and no tool calls. * Treated as a soft failure — skip same-model retries and hand off to * the fallback chain directly. Typical cause: upstream rate-limit * rejection that comes back as HTTP 200 with empty body. */ EMPTY_RESPONSE, /** - * payment / billing failure (HTTP 402, "insufficient_quota", + * RFC-009 P3.2: payment / billing failure (HTTP 402, "insufficient_quota", * "credit balance is too low", etc.). Distinct from {@link #AUTH_ERROR} * because the right response is to switch provider (a different * provider may have credits) rather than just terminate. Skips same-model @@ -946,7 +1038,7 @@ public class NodeStreamingChatHelper { */ BILLING, /** - * requested model id not recognized by the provider + * RFC-009 P3.2: requested model id not recognized by the provider * (HTTP 404, "Model not exist", "model_not_found", DashScope's * "url error"). Same handling as {@link #BILLING} — heads straight * to the fallback chain instead of looping retries against a model diff --git a/mateclaw-server/src/main/java/vip/mate/llm/controller/ProviderPoolController.java b/mateclaw-server/src/main/java/vip/mate/llm/controller/ProviderPoolController.java new file mode 100644 index 00000000..11be60d4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/controller/ProviderPoolController.java @@ -0,0 +1,120 @@ +package vip.mate.llm.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import vip.mate.common.result.R; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.AvailableProviderPool.RemovalReason; +import vip.mate.llm.failover.ProbeResult; +import vip.mate.llm.failover.ProviderHealthTracker; +import vip.mate.llm.failover.ProviderHealthTracker.ProviderHealthSnapshot; +import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.model.ProviderInfoDTO; +import vip.mate.llm.service.ModelProviderService; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * RFC-009 Phase 4 — read-only diagnostic endpoint for the provider pool. + * + *

Surfaces the union of three pieces of state:

+ *
    + *
  • {@link AvailableProviderPool} membership + last removal reason + * (HARD failures, init probe verdicts, manual eviction).
  • + *
  • {@link ProviderHealthTracker} cooldown status (SOFT failures).
  • + *
  • {@link ModelProviderService} configuration metadata so the UI can + * skip rendering rows for providers the user has never set up.
  • + *
+ * + *

Manual reprobe + auto-reprobe-on-config-change land in PR-1e.

+ */ +@Tag(name = "Provider 可用池") +@RestController +@RequestMapping("/api/v1/llm/provider-pool") +@RequiredArgsConstructor +public class ProviderPoolController { + + private final AvailableProviderPool providerPool; + private final ProviderHealthTracker healthTracker; + private final ModelProviderService providerService; + private final ProviderInitProbe initProbe; + + @Operation(summary = "查询所有 provider 的池状态 + 冷却信息") + @GetMapping + public R> snapshot() { + Map poolView = providerPool.snapshot(); + Map healthView = healthTracker.snapshot(); + List providers = providerService.listProviders(); + + List rows = new ArrayList<>(providers.size()); + for (ProviderInfoDTO p : providers) { + String id = p.getId(); + // Pool membership: snapshot() returns id->null for in-pool, id->reason for removed. + // An id missing from the snapshot has never been touched (treat as not-in-pool until first probe). + boolean tracked = poolView.containsKey(id); + RemovalReason reason = poolView.get(id); + boolean inPool = tracked && reason == null; + ProviderHealthSnapshot health = healthView.get(id); + + rows.add(new ProviderPoolEntryDTO( + id, + p.getName(), + inPool, + reason == null ? null : reason.source().name(), + reason == null ? null : reason.message(), + reason == null ? null : reason.removedAtMs(), + health != null && health.cooldownRemainingMs() > 0, + health == null ? 0L : health.cooldownRemainingMs(), + health == null ? 0L : health.consecutiveFailures() + )); + } + return R.ok(rows); + } + + @Operation(summary = "手动重新探测某个 provider,立即更新池状态") + @PostMapping("/{providerId}/reprobe") + public R reprobe(@PathVariable String providerId) { + ProbeResult result = initProbe.probeOne(providerId); + return R.ok(new ReprobeResultDTO( + providerId, + result.success(), + result.latencyMs(), + result.errorMessage(), + providerPool.contains(providerId) + )); + } + + /** Result of a manual reprobe. {@code inPool} reflects pool state after the probe ran. */ + public record ReprobeResultDTO( + String providerId, + boolean success, + long latencyMs, + String errorMessage, + boolean inPool + ) {} + + /** + * Per-row payload for the {@code /provider-pool} response. Flat record + * because the UI renders each provider as one card and we want a stable + * shape that's trivial to bind in TypeScript. + */ + public record ProviderPoolEntryDTO( + String providerId, + String providerName, + boolean inPool, + String removalSource, + String removalMessage, + Long removedAtMs, + boolean inCooldown, + long cooldownRemainingMs, + long consecutiveFailures + ) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java index 61d2104c..68a89fa0 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/FallbackEntry.java @@ -3,7 +3,7 @@ package vip.mate.llm.failover; import org.springframework.ai.chat.model.ChatModel; /** - * a single rung of the multi-model failover chain. Pairs a + * RFC-009 P3.3: a single rung of the multi-model failover chain. Pairs a * {@link ChatModel} with the {@code providerId} that built it so the * chain walker can consult {@link ProviderHealthTracker} (which keys cooldown * state by provider id, not by ChatModel instance). diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java index 4233b6d2..3b84f2b6 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthProperties.java @@ -3,7 +3,7 @@ package vip.mate.llm.failover; import org.springframework.boot.context.properties.ConfigurationProperties; /** - * thresholds for the per-provider health tracker. + * RFC-009 P3.3: thresholds for the per-provider health tracker. * *
  * mateclaw:
diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java
index 02d04dec..b0dd24ee 100644
--- a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java
+++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderHealthTracker.java
@@ -11,7 +11,7 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
 /**
- * per-provider health tracker for the multi-model failover chain.
+ * RFC-009 P3.3: per-provider health tracker for the multi-model failover chain.
  *
  * 

Tracks consecutive failure counts per provider id. When a provider hits * {@link ProviderHealthProperties#getFailureThreshold} consecutive failures, diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java index 8be0804f..1a679dd4 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderInitProbe.java @@ -3,8 +3,10 @@ package vip.mate.llm.failover; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import org.springframework.util.StringUtils; +import vip.mate.llm.event.ModelConfigChangedEvent; import vip.mate.llm.model.ModelProtocol; import vip.mate.llm.model.ModelProviderEntity; import vip.mate.llm.repository.ModelProviderMapper; @@ -73,6 +75,24 @@ public class ProviderInitProbe { probeAllConfigured(); } + /** + * RFC-009 Phase 4: when the user edits a provider's API key / Base URL + * (or any other field that {@link ModelProviderService} considers + * material), re-probe the whole configured set. The event has no + * providerId payload, so we re-probe everything — cheap because most + * probes use free {@code /v1/models} endpoints. + * + *

Async so the user's "save provider config" call returns + * immediately; the probe runs in the background and badges update on + * the next poll.

+ */ + @Async + @EventListener(ModelConfigChangedEvent.class) + public void onModelConfigChanged(ModelConfigChangedEvent event) { + log.info("[ProviderInitProbe] re-probing after ModelConfigChangedEvent (reason={})", event.reason()); + probeAllConfigured(); + } + /** * Probe every configured provider in parallel. Public so PR-1e's manual * reprobe / config-changed listener can trigger a full refresh, not just diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ModelProviderEntity.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ModelProviderEntity.java index ebdab08f..2ceaf7d8 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ModelProviderEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ModelProviderEntity.java @@ -50,7 +50,7 @@ public class ModelProviderEntity { private String oauthAccountId; /** - * position of this provider in the fallback chain. + * RFC-009: position of this provider in the fallback chain. * {@code 0} (default) means "not in the chain"; positive values are tried * in ascending order after the primary model exhausts its retries. */ diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java index e53d5772..97982771 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java @@ -12,7 +12,7 @@ public class ProviderConfigRequest { private String chatModel; private Map generateKwargs; /** - * provider's position in the multi-model failover chain. + * RFC-009 P3.5: provider's position in the multi-model failover chain. * {@code 0} = excluded; positive ints define ascending try-order. When * {@code null} the field is left untouched on update. */ diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java index 75fac2c7..eadc6fcc 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java @@ -29,6 +29,6 @@ public class ProviderInfoDTO { private String authType; private Boolean oauthConnected; private Long oauthExpiresAt; - /** position in the failover chain (0 = excluded, 1..N = priority). */ + /** RFC-009 P3.5: position in the failover chain (0 = excluded, 1..N = priority). */ private Integer fallbackPriority; } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java index 575331ae..96c21718 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java @@ -72,7 +72,7 @@ public class ModelProviderService { provider.setBaseUrl(request.getBaseUrl()); provider.setChatModel(ModelProtocol.resolveChatModel(request.getProtocol(), request.getChatModel())); provider.setGenerateKwargs(writeJson(request.getGenerateKwargs())); - // only update fallback priority when the caller explicitly + // RFC-009 P3.5: only update fallback priority when the caller explicitly // sends a value. null leaves it untouched (existing chain unchanged). if (request.getFallbackPriority() != null) { int p = Math.max(0, request.getFallbackPriority()); @@ -150,7 +150,7 @@ public class ModelProviderService { } /** - * ordered list of providers that participate in the multi-model + * RFC-009: ordered list of providers that participate in the multi-model * failover chain. Filters by {@code fallback_priority > 0} and sorts * ascending, so priority 1 is tried first after the primary model * exhausts retries. An empty list disables fallover entirely. diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiAutoConfiguration.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiAutoConfiguration.java index 1954237e..95ea616e 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiAutoConfiguration.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiAutoConfiguration.java @@ -1,14 +1,31 @@ package vip.mate.wiki; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Configuration; +import org.springframework.context.event.EventListener; +import vip.mate.wiki.job.WikiProcessingJobService; /** - * Wiki 知识库模块自动配置 + * Wiki module auto-configuration * * @author MateClaw Team */ +@Slf4j @Configuration @EnableConfigurationProperties(WikiProperties.class) +@RequiredArgsConstructor public class WikiAutoConfiguration { + + private final WikiProcessingJobService wikiProcessingJobService; + + /** + * RFC-030: Recover stuck wiki processing jobs on startup. + */ + @EventListener(ApplicationReadyEvent.class) + public void recoverWikiJobs(ApplicationReadyEvent event) { + wikiProcessingJobService.recoverOnStartup(); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java index 47f194ce..9d64bd2c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java @@ -110,4 +110,21 @@ public class WikiProperties { /** 混合搜索默认模式:keyword / semantic / hybrid */ private String searchDefaultMode = "hybrid"; + + // ==================== RFC-031: Light processing tiers ==================== + + /** Whether to auto-dispatch a LIGHT_ENRICH job after heavy ingest completes */ + private boolean lightEnrichEnabled = true; + + /** Delay before light enrichment starts (ms) */ + private long lightEnrichDelayMs = 2000; + + /** + * Minimum ratio of enriched content length to original content length. + * If the LLM returns text shorter than this ratio, the enrichment is rejected. + */ + private double wikilinkMinContentRatio = 0.5; + + /** Maximum characters for local repair single-page regeneration */ + private int localRepairMaxChars = 8000; } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java new file mode 100644 index 00000000..854d9d3a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java @@ -0,0 +1,164 @@ +package vip.mate.wiki.controller; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.web.bind.annotation.*; +import vip.mate.wiki.dto.*; +import vip.mate.wiki.job.WikiProcessingJobService; +import vip.mate.wiki.job.event.WikiJobCreatedEvent; +import vip.mate.wiki.repository.WikiProcessingJobMapper; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageCitationMapper; +import vip.mate.wiki.service.*; + +import java.util.List; +import java.util.Map; + +/** + * RFC-029/030/031/032/033: REST endpoints for wiki relations, jobs, enrichment, and search. + */ +@Slf4j +@RestController +@RequestMapping("/api/v1/wiki") +@RequiredArgsConstructor +public class WikiRelationController { + + private final WikiRelationService relationService; + private final WikiProcessingJobService jobService; + private final WikiProcessingJobMapper jobMapper; + private final WikiPageService pageService; + private final WikiPageCitationMapper citationMapper; + private final HybridRetriever hybridRetriever; + private final ApplicationEventPublisher eventPublisher; + private final ObjectMapper objectMapper; + + // ==================== RFC-029: Relations ==================== + + @GetMapping("/kb/{kbId}/pages/{slug}/related") + public List relatedPages( + @PathVariable Long kbId, + @PathVariable String slug, + @RequestParam(defaultValue = "5") int topK) { + return relationService.relatedPages(kbId, slug, Math.min(topK, 20)); + } + + @GetMapping("/kb/{kbId}/pages/{slugA}/relation/{slugB}") + public RelationExplanation explainRelation( + @PathVariable Long kbId, + @PathVariable String slugA, + @PathVariable String slugB) { + return relationService.explain(kbId, slugA, slugB); + } + + @GetMapping("/raw/{rawId}/pages") + public List pagesByRawId(@PathVariable Long rawId) { + return relationService.pagesByRawId(rawId); + } + + @GetMapping("/chunks/{chunkId}/pages") + public List pagesByChunkId(@PathVariable Long chunkId) { + return relationService.pagesByChunkId(chunkId); + } + + // ==================== RFC-029: Citations ==================== + + @GetMapping("/kb/{kbId}/pages/{pageId}/citations") + public List pageCitations( + @PathVariable Long kbId, + @PathVariable Long pageId) { + return citationMapper.listWithRawByPageId(pageId); + } + + // ==================== RFC-030: Jobs ==================== + + @GetMapping("/kb/{kbId}/jobs") + public List getJobs( + @PathVariable Long kbId, + @RequestParam(required = false) Long rawId) { + if (rawId != null) { + return jobMapper.findLatestByRawId(rawId) + .map(List::of).orElse(List.of()); + } + return jobMapper.listQueued(kbId, 20); + } + + // ==================== RFC-030/033: KB Stats ==================== + + @GetMapping("/kb/{kbId}/stats") + public Map kbStats(@PathVariable Long kbId) { + int pageCount = pageService.countByKbId(kbId); + // Count enriched pages (those containing [[wikilinks]]) + long enrichedCount = pageService.listByKbIdWithContent(kbId).stream() + .filter(p -> p.getContent() != null && p.getContent().contains("[[")) + .count(); + // Use listByKbId (all statuses) instead of listQueued (queued-only) + var allJobs = jobMapper.listByKbId(kbId, 200); + int failedJobCount = (int) allJobs.stream() + .filter(j -> "failed".equals(j.getStatus())) + .count(); + int runningJobCount = (int) allJobs.stream() + .filter(j -> "running".equals(j.getStatus())) + .count(); + + return Map.of( + "pageCount", pageCount, + "enrichedPageCount", enrichedCount, + "failedJobCount", failedJobCount, + "runningJobCount", runningJobCount + ); + } + + // ==================== RFC-031: Enrichment & Repair ==================== + + @PostMapping("/kb/{kbId}/pages/{slug}/enrich") + public Map enrichPage(@PathVariable Long kbId, @PathVariable String slug) { + WikiPageEntity page = pageService.getBySlug(kbId, slug); + if (page == null) return Map.of("error", "Page not found: " + slug); + + Long rawId = 0L; + try { + List rawIds = objectMapper.readValue( + page.getSourceRawIds() != null ? page.getSourceRawIds() : "[]", + new TypeReference>() {}); + if (!rawIds.isEmpty()) rawId = rawIds.get(0); + } catch (Exception ignored) {} + + WikiProcessingJobEntity job = jobService.createLightEnrich(kbId, rawId); + eventPublisher.publishEvent(new WikiJobCreatedEvent(job.getId())); + return Map.of("jobId", job.getId()); + } + + @PostMapping("/kb/{kbId}/pages/{slug}/repair") + public Map repairPage(@PathVariable Long kbId, @PathVariable String slug) { + WikiPageEntity page = pageService.getBySlug(kbId, slug); + if (page == null) return Map.of("error", "Page not found: " + slug); + + Long rawId = 0L; + try { + List rawIds = objectMapper.readValue( + page.getSourceRawIds() != null ? page.getSourceRawIds() : "[]", + new TypeReference>() {}); + if (!rawIds.isEmpty()) rawId = rawIds.get(0); + } catch (Exception ignored) {} + + WikiProcessingJobEntity job = jobService.createLocalRepair(kbId, rawId, page.getId()); + eventPublisher.publishEvent(new WikiJobCreatedEvent(job.getId())); + return Map.of("jobId", job.getId()); + } + + // ==================== RFC-032: Search preview ==================== + + @PostMapping("/kb/{kbId}/search-preview") + public List searchPreview( + @PathVariable Long kbId, + @RequestBody Map body) { + String query = (String) body.getOrDefault("query", ""); + String mode = (String) body.getOrDefault("mode", "hybrid"); + int topK = body.containsKey("topK") ? ((Number) body.get("topK")).intValue() : 5; + return hybridRetriever.search(kbId, query, mode, Math.min(topK, 20)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/ChunkPageRef.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/ChunkPageRef.java new file mode 100644 index 00000000..4247a3a4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/ChunkPageRef.java @@ -0,0 +1,6 @@ +package vip.mate.wiki.dto; + +/** + * RFC-029: Lightweight chunk-to-page reference for shared-chunk signal computation. + */ +public record ChunkPageRef(Long chunkId, Long pageId) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageCitationWithRaw.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageCitationWithRaw.java new file mode 100644 index 00000000..5158b8d5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageCitationWithRaw.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.dto; + +import java.math.BigDecimal; + +/** + * RFC-029: Citation record joined with the chunk's raw material ID. + */ +public record PageCitationWithRaw(Long id, Long pageId, Long chunkId, Long rawId, + Integer paragraphIdx, String anchorText, BigDecimal confidence) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageSearchResult.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageSearchResult.java new file mode 100644 index 00000000..d0e458ae --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/PageSearchResult.java @@ -0,0 +1,16 @@ +package vip.mate.wiki.dto; + +import java.util.List; + +/** + * RFC-032: Enhanced search result with snippet, match metadata, and relevance reason. + */ +public record PageSearchResult( + String slug, + String title, + String summary, + String snippet, + List matchedBy, + String reason, + double score +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/RawTitleRef.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RawTitleRef.java new file mode 100644 index 00000000..85a88b89 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RawTitleRef.java @@ -0,0 +1,6 @@ +package vip.mate.wiki.dto; + +/** + * RFC-032: Lightweight raw material ID-to-title projection for batch lookups. + */ +public record RawTitleRef(Long id, String title) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelatedPageResult.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelatedPageResult.java new file mode 100644 index 00000000..ba1fa593 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelatedPageResult.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.dto; + +import java.util.List; + +/** + * RFC-029: A related page with aggregated relation score and contributing signals. + */ +public record RelatedPageResult(String slug, String title, String summary, + double score, List signals) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelationExplanation.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelationExplanation.java new file mode 100644 index 00000000..83a0aa50 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/RelationExplanation.java @@ -0,0 +1,13 @@ +package vip.mate.wiki.dto; + +import java.util.List; + +/** + * RFC-029: Detailed breakdown of the relation between two pages. + */ +public record RelationExplanation(String slugA, String slugB, double totalScore, + List breakdown) { + public static RelationExplanation notFound() { + return new RelationExplanation(null, null, 0, List.of()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/SignalScore.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/SignalScore.java new file mode 100644 index 00000000..ecd72583 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/SignalScore.java @@ -0,0 +1,6 @@ +package vip.mate.wiki.dto; + +/** + * RFC-029: Individual signal contribution to a relation score. + */ +public record SignalScore(String signal, double weight, double score) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiPageLite.java b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiPageLite.java new file mode 100644 index 00000000..2246c6f8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/dto/WikiPageLite.java @@ -0,0 +1,6 @@ +package vip.mate.wiki.dto; + +/** + * RFC-029: Lightweight page projection without content. + */ +public record WikiPageLite(Long id, String slug, String title, String summary) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiHardModelException.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiHardModelException.java new file mode 100644 index 00000000..5db916b8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiHardModelException.java @@ -0,0 +1,17 @@ +package vip.mate.wiki.job; + +/** + * RFC-031: Thrown when a wiki processing step encounters a hard/fatal + * model error that warrants fallback to a different model. + */ +public class WikiHardModelException extends RuntimeException { + + private final String errorCode; + + public WikiHardModelException(String errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public String getErrorCode() { return errorCode; } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobDispatcher.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobDispatcher.java new file mode 100644 index 00000000..785ee4aa --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobDispatcher.java @@ -0,0 +1,72 @@ +package vip.mate.wiki.job; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import vip.mate.wiki.job.event.WikiJobCreatedEvent; +import vip.mate.wiki.repository.WikiProcessingJobMapper; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.job.template.WikiProcessingTemplate; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.stream.Collectors; + +/** + * RFC-031: Listens for WikiJobCreatedEvent and dispatches jobs + * to the appropriate processing template. + */ +@Slf4j +@Component +public class WikiJobDispatcher { + + private final Map templateMap; + private final WikiProcessingJobMapper jobMapper; + + private static final ExecutorService DISPATCH_EXECUTOR = + Executors.newVirtualThreadPerTaskExecutor(); + + public WikiJobDispatcher(List templates, + WikiProcessingJobMapper jobMapper) { + this.jobMapper = jobMapper; + this.templateMap = templates.stream().collect( + Collectors.toMap(t -> t.getClass().getSimpleName(), t -> t)); + log.info("[WikiDispatcher] Registered templates: {}", templateMap.keySet()); + } + + @EventListener(WikiJobCreatedEvent.class) + public void onJobCreated(WikiJobCreatedEvent event) { + DISPATCH_EXECUTOR.submit(() -> dispatch(event.jobId())); + } + + public void dispatch(Long jobId) { + WikiProcessingJobEntity job = jobMapper.selectById(jobId); + if (job == null) return; + + try { + WikiJobStage stage = WikiJobStage.valueOf(job.getStage().toUpperCase()); + if (stage.isTerminal()) return; + } catch (IllegalArgumentException e) { + log.warn("[WikiDispatch] Unknown stage '{}' for job {}", job.getStage(), jobId); + return; + } + + WikiProcessingTemplate template = resolveTemplate(job.getJobType()); + if (template == null) { + log.error("[WikiDispatch] No template for job type: {}", job.getJobType()); + return; + } + template.execute(job); + } + + private WikiProcessingTemplate resolveTemplate(String jobType) { + return switch (jobType) { + case "heavy_ingest" -> templateMap.get("HeavyIngestTemplate"); + case "light_enrich" -> templateMap.get("LightEnrichTemplate"); + case "local_repair" -> templateMap.get("LocalRepairTemplate"); + default -> null; + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStage.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStage.java new file mode 100644 index 00000000..19d57f1b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStage.java @@ -0,0 +1,16 @@ +package vip.mate.wiki.job; + +/** + * RFC-030: Stage state machine for wiki processing jobs. + */ +public enum WikiJobStage { + QUEUED, ROUTING, + PHASE_A_RUNNING, PHASE_A_DONE, + PHASE_B_RUNNING, + ENRICHING, EMBEDDING, + COMPLETED, FAILED, PARTIAL, CANCELLED; + + public boolean isTerminal() { + return this == COMPLETED || this == FAILED || this == PARTIAL || this == CANCELLED; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStatus.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStatus.java new file mode 100644 index 00000000..aef9659e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStatus.java @@ -0,0 +1,8 @@ +package vip.mate.wiki.job; + +/** + * RFC-030: Top-level status of a wiki processing job. + */ +public enum WikiJobStatus { + QUEUED, RUNNING, COMPLETED, FAILED, PARTIAL, CANCELLED +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java new file mode 100644 index 00000000..707a8a8d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobStep.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.job; + +/** + * RFC-030: Logical steps within a wiki processing job, + * used for per-step model routing. + */ +public enum WikiJobStep { + ROUTE, CREATE_PAGE, MERGE_PAGE, ENRICH, SUMMARY +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobType.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobType.java new file mode 100644 index 00000000..55cbf343 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiJobType.java @@ -0,0 +1,10 @@ +package vip.mate.wiki.job; + +/** + * RFC-030: Types of wiki processing jobs. + */ +public enum WikiJobType { + HEAVY_INGEST, + LIGHT_ENRICH, + LOCAL_REPAIR +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java new file mode 100644 index 00000000..b391a7e1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiKbConfig.java @@ -0,0 +1,20 @@ +package vip.mate.wiki.job; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * RFC-030: Per-KB processing configuration POJO, deserialized from + * {@link vip.mate.wiki.model.WikiKnowledgeBaseEntity#getConfigContent()}. + */ +@Data +public class WikiKbConfig { + + /** Per-step model overrides: "heavy_ingest.create_page" → modelId */ + private Map stepModels; + + /** Global fallback model chain for all steps in this KB */ + private List fallbackModelIds; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelRoutingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelRoutingService.java new file mode 100644 index 00000000..a61a0753 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelRoutingService.java @@ -0,0 +1,138 @@ +package vip.mate.wiki.job; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; +import vip.mate.llm.chatmodel.ProviderChatModelFactory; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.job.fallback.ModelFallbackHandler; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.job.strategy.WikiStepModelStrategy; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.service.WikiKnowledgeBaseService; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * RFC-030: Model routing service — selects the right model for each + * wiki processing step, with fallback chain support. + */ +@Slf4j +@Service +public class WikiModelRoutingService { + + private final List strategies; + private final ModelFallbackHandler fallbackChainHead; + private final ProviderChatModelFactory chatModelFactory; + private final ModelConfigService modelConfigService; + private final WikiKnowledgeBaseService kbService; + private final ObjectMapper objectMapper; + + @Autowired(required = false) + private AvailableProviderPool providerPool; + + private static final RetryTemplate WIKI_NO_RETRY = RetryTemplate.builder() + .maxAttempts(1).build(); + + public WikiModelRoutingService(List strategies, + List fallbackHandlers, + ProviderChatModelFactory chatModelFactory, + ModelConfigService modelConfigService, + WikiKnowledgeBaseService kbService, + ObjectMapper objectMapper) { + this.strategies = strategies; + this.chatModelFactory = chatModelFactory; + this.modelConfigService = modelConfigService; + this.kbService = kbService; + this.objectMapper = objectMapper; + this.fallbackChainHead = buildChain(fallbackHandlers); + } + + /** + * Select the best model for a given job and step, respecting strategy + * priority and provider pool availability. + */ + public Long selectModelId(WikiProcessingJobEntity job, WikiJobStep step) { + WikiKnowledgeBaseEntity kb = (job != null) ? kbService.getById(job.getKbId()) : null; + for (WikiStepModelStrategy strategy : strategies) { + if (strategy.supports(step)) { + Long modelId = strategy.selectModelId(job, kb, step); + if (modelId != null && isAvailable(modelId)) return modelId; + } + } + throw new WikiModelUnavailableException("No available model for step: " + step); + } + + /** + * Select a fallback model after a failure. + */ + public Long selectFallbackModel(WikiProcessingJobEntity job, WikiJobStep step, String errorCode) { + return fallbackChainHead.handle(job, step, errorCode) + .orElseThrow(() -> new WikiModelUnavailableException( + "Exhausted all fallback models for step: " + step)); + } + + /** + * Build a ChatModel instance for a given model ID. + */ + public ChatModel buildChatModel(Long modelId) { + ModelConfigEntity model = modelConfigService.getModel(modelId); + return chatModelFactory.buildFor(model, WIKI_NO_RETRY); + } + + /** + * Build the fallback chain JSON for a job (called once during routing stage). + */ + public String buildFallbackChainJson(Long kbId) { + WikiKbConfig config = kbConfigOf(kbId); + List chain; + if (config != null && config.getFallbackModelIds() != null && !config.getFallbackModelIds().isEmpty()) { + chain = config.getFallbackModelIds(); + } else { + chain = List.of(modelConfigService.getDefaultModel().getId()); + } + if (providerPool != null) { + chain = chain.stream().filter(this::isAvailable).collect(Collectors.toList()); + } + try { + return objectMapper.writeValueAsString(chain); + } catch (Exception e) { + return "[]"; + } + } + + private boolean isAvailable(Long modelId) { + if (providerPool == null) return true; + try { + ModelConfigEntity m = modelConfigService.getModel(modelId); + return providerPool.contains(m.getProvider()); + } catch (Exception e) { + return false; + } + } + + private WikiKbConfig kbConfigOf(Long kbId) { + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + if (kb == null || kb.getConfigContent() == null) return null; + try { + return objectMapper.readValue(kb.getConfigContent(), WikiKbConfig.class); + } catch (Exception e) { + return null; + } + } + + private static ModelFallbackHandler buildChain(List handlers) { + if (handlers.isEmpty()) throw new IllegalStateException("No ModelFallbackHandler registered"); + for (int i = 0; i < handlers.size() - 1; i++) { + handlers.get(i).setNext(handlers.get(i + 1)); + } + return handlers.get(0); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelUnavailableException.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelUnavailableException.java new file mode 100644 index 00000000..8a058561 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiModelUnavailableException.java @@ -0,0 +1,10 @@ +package vip.mate.wiki.job; + +/** + * RFC-030: Thrown when no model is available for a wiki processing step. + */ +public class WikiModelUnavailableException extends RuntimeException { + public WikiModelUnavailableException(String message) { + super(message); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java new file mode 100644 index 00000000..7e5f1be3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiProcessingJobService.java @@ -0,0 +1,185 @@ +package vip.mate.wiki.job; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.repository.WikiProcessingJobMapper; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +import java.time.LocalDateTime; +import java.util.Map; + +/** + * RFC-030: Wiki processing job lifecycle service — creates, transitions, + * and records errors for processing jobs. + */ +@Slf4j +@Service +public class WikiProcessingJobService { + + private final WikiProcessingJobMapper jobMapper; + private final WikiModelRoutingService routingService; + private final ObjectMapper objectMapper; + + @Autowired(required = false) + private AvailableProviderPool providerPool; + + @Autowired + private ModelConfigService modelConfigService; + + public WikiProcessingJobService(WikiProcessingJobMapper jobMapper, + WikiModelRoutingService routingService, + ObjectMapper objectMapper) { + this.jobMapper = jobMapper; + this.routingService = routingService; + this.objectMapper = objectMapper; + } + + public WikiProcessingJobEntity createHeavyIngest(Long kbId, Long rawId) { + WikiProcessingJobEntity job = new WikiProcessingJobEntity(); + job.setKbId(kbId); + job.setRawId(rawId); + job.setJobType(WikiJobType.HEAVY_INGEST.name().toLowerCase()); + job.setStage(WikiJobStage.QUEUED.name().toLowerCase()); + job.setStatus(WikiJobStatus.QUEUED.name().toLowerCase()); + job.setMaxRetries(3); + job.setRetryCount(0); + jobMapper.insert(job); + return job; + } + + public WikiProcessingJobEntity createLightEnrich(Long kbId, Long rawId) { + WikiProcessingJobEntity job = new WikiProcessingJobEntity(); + job.setKbId(kbId); + job.setRawId(rawId); + job.setJobType(WikiJobType.LIGHT_ENRICH.name().toLowerCase()); + job.setStage(WikiJobStage.QUEUED.name().toLowerCase()); + job.setStatus(WikiJobStatus.QUEUED.name().toLowerCase()); + job.setMaxRetries(2); + job.setRetryCount(0); + jobMapper.insert(job); + return job; + } + + public WikiProcessingJobEntity createLocalRepair(Long kbId, Long rawId, Long targetPageId) { + WikiProcessingJobEntity job = new WikiProcessingJobEntity(); + job.setKbId(kbId); + job.setRawId(rawId); + job.setJobType(WikiJobType.LOCAL_REPAIR.name().toLowerCase()); + job.setStage(WikiJobStage.QUEUED.name().toLowerCase()); + job.setStatus(WikiJobStatus.QUEUED.name().toLowerCase()); + job.setMaxRetries(2); + job.setRetryCount(0); + try { + job.setMetaJson(objectMapper.writeValueAsString(Map.of("targetPageId", targetPageId))); + } catch (Exception e) { + job.setMetaJson("{}"); + } + jobMapper.insert(job); + return job; + } + + public WikiProcessingJobEntity transition(Long jobId, WikiJobStage newStage) { + WikiProcessingJobEntity job = jobMapper.selectById(jobId); + if (job == null) return null; + + job.setStage(newStage.name().toLowerCase()); + if (newStage == WikiJobStage.ROUTING) { + job.setStartedAt(LocalDateTime.now()); + job.setStatus(WikiJobStatus.RUNNING.name().toLowerCase()); + job.setFallbackChainJson(routingService.buildFallbackChainJson(job.getKbId())); + } else if (newStage == WikiJobStage.COMPLETED) { + job.setFinishedAt(LocalDateTime.now()); + job.setStatus(WikiJobStatus.COMPLETED.name().toLowerCase()); + } + jobMapper.updateById(job); + return job; + } + + public void recordHardError(Long jobId, String errorCode, String errorMessage) { + WikiProcessingJobEntity job = jobMapper.selectById(jobId); + if (job == null) return; + + job.setErrorCode(errorCode); + job.setErrorMessage(truncate(errorMessage, 2000)); + job.setFinishedAt(LocalDateTime.now()); + + if (job.getRetryCount() < job.getMaxRetries()) { + job.setRetryCount(job.getRetryCount() + 1); + job.setResumeFromStage(job.getStage()); + job.setStage(WikiJobStage.QUEUED.name().toLowerCase()); + job.setStatus(WikiJobStatus.QUEUED.name().toLowerCase()); + } else { + job.setStatus(WikiJobStatus.FAILED.name().toLowerCase()); + job.setStage(WikiJobStage.FAILED.name().toLowerCase()); + } + jobMapper.updateById(job); + + if (providerPool != null && isHardError(errorCode) && job.getCurrentModelId() != null) { + notifyPoolHardError(job.getCurrentModelId(), errorCode); + } + } + + public void recordSoftError(Long jobId, String errorCode, String errorMessage) { + WikiProcessingJobEntity job = jobMapper.selectById(jobId); + if (job == null) return; + + job.setErrorCode(errorCode); + job.setErrorMessage(truncate(errorMessage, 2000)); + job.setFinishedAt(LocalDateTime.now()); + + if (job.getRetryCount() < job.getMaxRetries()) { + job.setRetryCount(job.getRetryCount() + 1); + job.setResumeFromStage(job.getStage()); + job.setStage(WikiJobStage.QUEUED.name().toLowerCase()); + job.setStatus(WikiJobStatus.QUEUED.name().toLowerCase()); + } else { + job.setStatus(WikiJobStatus.FAILED.name().toLowerCase()); + job.setStage(WikiJobStage.FAILED.name().toLowerCase()); + } + jobMapper.updateById(job); + } + + /** + * Recover stuck jobs on startup (routing or *_running → queued). + */ + public void recoverOnStartup() { + int recovered = jobMapper.recoverStuckJobs(); + if (recovered > 0) { + log.info("[WikiJob] Recovered {} stuck jobs on startup", recovered); + } + } + + private static boolean isHardError(String errorCode) { + return errorCode != null && ( + errorCode.equals("AUTH_ERROR") || + errorCode.equals("BILLING") || + errorCode.equals("MODEL_NOT_FOUND")); + } + + private void notifyPoolHardError(Long modelId, String errorCode) { + try { + ModelConfigEntity model = modelConfigService.getModel(modelId); + if (model != null) { + AvailableProviderPool.RemovalSource source; + try { + source = AvailableProviderPool.RemovalSource.valueOf(errorCode); + } catch (IllegalArgumentException e) { + source = AvailableProviderPool.RemovalSource.AUTH_ERROR; + } + providerPool.remove(model.getProvider(), source, "Wiki job hard error: " + errorCode); + } + } catch (Exception e) { + log.warn("[WikiJob] Failed to notify provider pool: {}", e.getMessage()); + } + } + + private static String truncate(String s, int maxLen) { + if (s == null) return null; + return s.length() <= maxLen ? s : s.substring(0, maxLen); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiSoftModelException.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiSoftModelException.java new file mode 100644 index 00000000..bf8e391d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiSoftModelException.java @@ -0,0 +1,17 @@ +package vip.mate.wiki.job; + +/** + * RFC-031: Thrown when a wiki processing step encounters a transient + * model error that can be retried. + */ +public class WikiSoftModelException extends RuntimeException { + + private final String errorCode; + + public WikiSoftModelException(String errorCode, String message) { + super(message); + this.errorCode = errorCode; + } + + public String getErrorCode() { return errorCode; } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/event/WikiJobCreatedEvent.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/event/WikiJobCreatedEvent.java new file mode 100644 index 00000000..8a3d4ea5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/event/WikiJobCreatedEvent.java @@ -0,0 +1,7 @@ +package vip.mate.wiki.job.event; + +/** + * RFC-031: Published when a new wiki processing job is created, + * triggering the dispatcher to execute it. + */ +public record WikiJobCreatedEvent(Long jobId) {} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/GlobalFallbackModelHandler.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/GlobalFallbackModelHandler.java new file mode 100644 index 00000000..b79f3e08 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/GlobalFallbackModelHandler.java @@ -0,0 +1,35 @@ +package vip.mate.wiki.job.fallback; + +import lombok.RequiredArgsConstructor; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +import java.util.Optional; + +/** + * RFC-030: Final fallback — returns the system default model. + */ +@Component +@Order(2) +@RequiredArgsConstructor +public class GlobalFallbackModelHandler implements ModelFallbackHandler { + + private final ModelConfigService modelConfigService; + + private ModelFallbackHandler next; + + @Override + public Optional handle(WikiProcessingJobEntity job, WikiJobStep step, String errorCode) { + try { + return Optional.of(modelConfigService.getDefaultModel().getId()); + } catch (Exception e) { + return Optional.empty(); + } + } + + @Override + public void setNext(ModelFallbackHandler next) { this.next = next; } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/ModelFallbackHandler.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/ModelFallbackHandler.java new file mode 100644 index 00000000..0570c86b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/ModelFallbackHandler.java @@ -0,0 +1,16 @@ +package vip.mate.wiki.job.fallback; + +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +import java.util.Optional; + +/** + * RFC-030: Chain of responsibility for model fallback selection. + */ +public interface ModelFallbackHandler { + + Optional handle(WikiProcessingJobEntity job, WikiJobStep step, String errorCode); + + void setNext(ModelFallbackHandler next); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/PoolAwareModelFallbackHandler.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/PoolAwareModelFallbackHandler.java new file mode 100644 index 00000000..9316f8a5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/fallback/PoolAwareModelFallbackHandler.java @@ -0,0 +1,70 @@ +package vip.mate.wiki.job.fallback; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +import java.util.List; +import java.util.Optional; + +/** + * RFC-030: Pool-aware fallback — walks the job's fallback chain, + * skipping models whose provider is no longer in the AvailableProviderPool. + */ +@Slf4j +@Component +@Order(1) +public class PoolAwareModelFallbackHandler implements ModelFallbackHandler { + + @Autowired(required = false) + private AvailableProviderPool providerPool; + + @Autowired + private ModelConfigService modelConfigService; + + @Autowired + private ObjectMapper objectMapper; + + private ModelFallbackHandler next; + + @Override + public Optional handle(WikiProcessingJobEntity job, WikiJobStep step, String errorCode) { + List fallbackChain = parseFallbackChain(job.getFallbackChainJson()); + for (Long modelId : fallbackChain) { + if (!modelId.equals(job.getCurrentModelId()) && isAvailable(modelId)) { + return Optional.of(modelId); + } + } + return next != null ? next.handle(job, step, errorCode) : Optional.empty(); + } + + private boolean isAvailable(Long modelId) { + if (providerPool == null) return true; + try { + ModelConfigEntity model = modelConfigService.getModel(modelId); + return providerPool.contains(model.getProvider()); + } catch (Exception e) { + return false; + } + } + + private List parseFallbackChain(String json) { + if (json == null || json.isBlank()) return List.of(); + try { + return objectMapper.readValue(json, new TypeReference<>() {}); + } catch (Exception e) { + return List.of(); + } + } + + @Override + public void setNext(ModelFallbackHandler next) { this.next = next; } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java new file mode 100644 index 00000000..7be65cca --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/model/WikiProcessingJobEntity.java @@ -0,0 +1,60 @@ +package vip.mate.wiki.job.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * RFC-030: Wiki processing job entity — tracks the lifecycle of a single + * raw material processing run with per-stage state and model routing info. + */ +@Data +@TableName("mate_wiki_processing_job") +public class WikiProcessingJobEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long kbId; + + private Long rawId; + + private String jobType; + + private String stage; + + private String status; + + private Long primaryModelId; + + private Long currentModelId; + + private String fallbackChainJson; + + private Integer retryCount; + + private Integer maxRetries; + + private String errorCode; + + private String errorMessage; + + private String resumeFromStage; + + /** Generic JSON metadata (e.g. targetPageId for LOCAL_REPAIR) */ + private String metaJson; + + private LocalDateTime startedAt; + + private LocalDateTime finishedAt; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/GlobalDefaultStepModelStrategy.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/GlobalDefaultStepModelStrategy.java new file mode 100644 index 00000000..3a7267ed --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/GlobalDefaultStepModelStrategy.java @@ -0,0 +1,33 @@ +package vip.mate.wiki.job.strategy; + +import lombok.RequiredArgsConstructor; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; + +/** + * RFC-030: Final fallback strategy — uses the system default model. + * Cheap steps (ROUTE, ENRICH, SUMMARY) prefer a lighter/cheaper model; + * strong steps (CREATE_PAGE, MERGE_PAGE) use the default model. + */ +@Component +@Order(2) +@RequiredArgsConstructor +public class GlobalDefaultStepModelStrategy implements WikiStepModelStrategy { + + private final ModelConfigService modelConfigService; + + @Override + public boolean supports(WikiJobStep step) { return true; } + + @Override + public Long selectModelId(WikiProcessingJobEntity job, WikiKnowledgeBaseEntity kb, WikiJobStep step) { + // RFC-030: cheap steps (ROUTE, ENRICH, SUMMARY) should ideally use a lighter model, + // but ModelConfigService has no "cheapest chat model" concept yet. + // When per-step pricing metadata is added, this switch can differentiate. + return modelConfigService.getDefaultModel().getId(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/KbConfigStepModelStrategy.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/KbConfigStepModelStrategy.java new file mode 100644 index 00000000..ea44de14 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/KbConfigStepModelStrategy.java @@ -0,0 +1,44 @@ +package vip.mate.wiki.job.strategy; + +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.annotation.Order; +import org.springframework.stereotype.Component; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiKbConfig; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; + +import java.util.Map; + +/** + * RFC-030: Highest-priority strategy — uses per-KB step model overrides + * from the KB's configContent JSON. + */ +@Slf4j +@Component +@Order(1) +@RequiredArgsConstructor +public class KbConfigStepModelStrategy implements WikiStepModelStrategy { + + private final ObjectMapper objectMapper; + + @Override + public boolean supports(WikiJobStep step) { return true; } + + @Override + public Long selectModelId(WikiProcessingJobEntity job, WikiKnowledgeBaseEntity kb, WikiJobStep step) { + if (kb == null || kb.getConfigContent() == null) return null; + try { + WikiKbConfig config = objectMapper.readValue(kb.getConfigContent(), WikiKbConfig.class); + Map stepModels = config.getStepModels(); + if (stepModels == null) return null; + String key = job.getJobType() + "." + step.name().toLowerCase(); + return stepModels.get(key); + } catch (Exception e) { + log.debug("[KbConfigStrategy] Failed to parse KB config: {}", e.getMessage()); + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/WikiStepModelStrategy.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/WikiStepModelStrategy.java new file mode 100644 index 00000000..df1b98a0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/strategy/WikiStepModelStrategy.java @@ -0,0 +1,21 @@ +package vip.mate.wiki.job.strategy; + +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; + +/** + * RFC-030: Strategy for selecting a model for a specific processing step. + * Implementations are evaluated in @Order priority; first non-null result wins. + */ +public interface WikiStepModelStrategy { + + boolean supports(WikiJobStep step); + + /** + * Select a model ID for the given job and step. + * + * @return model ID, or null to defer to the next strategy + */ + Long selectModelId(WikiProcessingJobEntity job, WikiKnowledgeBaseEntity kb, WikiJobStep step); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/template/HeavyIngestTemplate.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/HeavyIngestTemplate.java new file mode 100644 index 00000000..f98fd5b2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/HeavyIngestTemplate.java @@ -0,0 +1,53 @@ +package vip.mate.wiki.job.template; + +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.job.*; +import vip.mate.wiki.job.event.WikiJobCreatedEvent; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.service.WikiProcessingService; + +/** + * RFC-031: Heavy ingest template — delegates to the legacy processing + * service for the two-phase digest pipeline, then optionally dispatches + * a light enrichment job. + */ +@Component +public class HeavyIngestTemplate extends WikiProcessingTemplate { + + private final WikiProcessingService legacyProcessingService; + + public HeavyIngestTemplate(WikiModelRoutingService routingService, + WikiProcessingJobService jobService, + ApplicationEventPublisher eventPublisher, + WikiProperties wikiProperties, + WikiProcessingService legacyProcessingService) { + super(routingService, jobService, eventPublisher, wikiProperties); + this.legacyProcessingService = legacyProcessingService; + } + + @Override + protected WikiJobStep routingStep() { return WikiJobStep.CREATE_PAGE; } + + @Override + protected WikiJobStage mainStage() { return WikiJobStage.PHASE_A_RUNNING; } + + @Override + protected void doProcess(WikiProcessingJobEntity job, Long modelId) { + // Transition period: delegate to existing processing pipeline as a whole. + // TODO: RFC-031 future — split into processInChunksForJob(job, modelId), + // processChunkTwoPhaseForJob(job, modelId), scheduleEmbeddingAsync(rawId) + // for per-stage job transitions and per-step model routing. + legacyProcessingService.processRawMaterial(job.getRawId()); + } + + @Override + protected void onSuccess(WikiProcessingJobEntity job) { + if (wikiProperties.isLightEnrichEnabled()) { + WikiProcessingJobEntity enrichJob = + jobService.createLightEnrich(job.getKbId(), job.getRawId()); + eventPublisher.publishEvent(new WikiJobCreatedEvent(enrichJob.getId())); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LightEnrichTemplate.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LightEnrichTemplate.java new file mode 100644 index 00000000..155a3796 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LightEnrichTemplate.java @@ -0,0 +1,59 @@ +package vip.mate.wiki.job.template; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.WikiPageLite; +import vip.mate.wiki.job.*; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.service.WikiCitationService; +import vip.mate.wiki.service.WikiLinkEnrichmentService; +import vip.mate.wiki.service.WikiRelationService; + +import java.util.List; + +/** + * RFC-031: Light enrichment template — adds [[wikilinks]] and rebuilds + * citations without regenerating page content. + */ +@Slf4j +@Component +public class LightEnrichTemplate extends WikiProcessingTemplate { + + private final WikiLinkEnrichmentService enrichmentService; + private final WikiCitationService citationService; + private final WikiRelationService relationService; + + public LightEnrichTemplate(WikiModelRoutingService routingService, + WikiProcessingJobService jobService, + ApplicationEventPublisher eventPublisher, + WikiProperties wikiProperties, + WikiLinkEnrichmentService enrichmentService, + WikiCitationService citationService, + WikiRelationService relationService) { + super(routingService, jobService, eventPublisher, wikiProperties); + this.enrichmentService = enrichmentService; + this.citationService = citationService; + this.relationService = relationService; + } + + @Override + protected WikiJobStep routingStep() { return WikiJobStep.ENRICH; } + + @Override + protected WikiJobStage mainStage() { return WikiJobStage.ENRICHING; } + + @Override + protected void doProcess(WikiProcessingJobEntity job, Long modelId) { + List pages = relationService.pagesByRawId(job.getRawId()); + for (WikiPageLite page : pages) { + try { + enrichmentService.enrichPage(page.id(), modelId); + citationService.buildCitations(page.id(), job.getKbId()); + } catch (Exception e) { + log.warn("[LightEnrich] Failed to enrich page {}: {}", page.slug(), e.getMessage()); + } + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LocalRepairTemplate.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LocalRepairTemplate.java new file mode 100644 index 00000000..3cf0dca3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/LocalRepairTemplate.java @@ -0,0 +1,69 @@ +package vip.mate.wiki.job.template; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Component; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.job.*; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; +import vip.mate.wiki.service.WikiCitationService; +import vip.mate.wiki.service.WikiProcessingService; + +import java.util.Map; + +/** + * RFC-031: Local repair template — regenerates a single target page + * without affecting other pages derived from the same raw material. + */ +@Slf4j +@Component +public class LocalRepairTemplate extends WikiProcessingTemplate { + + private final WikiProcessingService legacyProcessingService; + private final WikiCitationService citationService; + private final ObjectMapper objectMapper; + + public LocalRepairTemplate(WikiModelRoutingService routingService, + WikiProcessingJobService jobService, + ApplicationEventPublisher eventPublisher, + WikiProperties wikiProperties, + WikiProcessingService legacyProcessingService, + WikiCitationService citationService, + ObjectMapper objectMapper) { + super(routingService, jobService, eventPublisher, wikiProperties); + this.legacyProcessingService = legacyProcessingService; + this.citationService = citationService; + this.objectMapper = objectMapper; + } + + @Override + protected WikiJobStep routingStep() { return WikiJobStep.CREATE_PAGE; } + + @Override + protected WikiJobStage mainStage() { return WikiJobStage.PHASE_B_RUNNING; } + + @Override + protected void doProcess(WikiProcessingJobEntity job, Long modelId) { + Map meta = parseMetaJson(job.getMetaJson()); + Object pageIdObj = meta.get("targetPageId"); + if (pageIdObj == null) { + throw new IllegalArgumentException("targetPageId missing from job metaJson, jobId=" + job.getId()); + } + Long targetPageId = Long.valueOf(pageIdObj.toString()); + + legacyProcessingService.repairSinglePage(targetPageId, modelId); + citationService.buildCitations(targetPageId, job.getKbId()); + } + + private Map parseMetaJson(String json) { + if (json == null || json.isBlank()) return Map.of(); + try { + return objectMapper.readValue(json, new TypeReference<>() {}); + } catch (Exception e) { + log.warn("[LocalRepair] Failed to parse metaJson: {}", e.getMessage()); + return Map.of(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/template/WikiProcessingTemplate.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/WikiProcessingTemplate.java new file mode 100644 index 00000000..276d0fbb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/template/WikiProcessingTemplate.java @@ -0,0 +1,87 @@ +package vip.mate.wiki.job.template; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.job.*; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +/** + * RFC-031: Template method base class for all wiki processing job types. + * The {@link #execute} skeleton handles routing, error classification, + * and fallback; subclasses implement the actual processing in {@link #doProcess}. + */ +@Slf4j +public abstract class WikiProcessingTemplate { + + protected final WikiModelRoutingService routingService; + protected final WikiProcessingJobService jobService; + protected final ApplicationEventPublisher eventPublisher; + protected final WikiProperties wikiProperties; + + protected WikiProcessingTemplate(WikiModelRoutingService routingService, + WikiProcessingJobService jobService, + ApplicationEventPublisher eventPublisher, + WikiProperties wikiProperties) { + this.routingService = routingService; + this.jobService = jobService; + this.eventPublisher = eventPublisher; + this.wikiProperties = wikiProperties; + } + + /** + * Non-overridable skeleton: route → process → handle errors. + */ + public final void execute(WikiProcessingJobEntity job) { + try { + job = jobService.transition(job.getId(), WikiJobStage.ROUTING); + Long modelId = routingService.selectModelId(job, routingStep()); + job.setCurrentModelId(modelId); + + job = jobService.transition(job.getId(), mainStage()); + try { + doProcess(job, modelId); + jobService.transition(job.getId(), WikiJobStage.COMPLETED); + onSuccess(job); + } catch (WikiHardModelException e) { + handleHardError(job, e); + } catch (WikiSoftModelException e) { + handleSoftError(job, e); + } catch (Exception e) { + jobService.recordSoftError(job.getId(), "UNKNOWN", e.getMessage()); + } + } catch (WikiModelUnavailableException e) { + log.error("[WikiTemplate] No model available for job {}: {}", job.getId(), e.getMessage()); + jobService.recordHardError(job.getId(), "MODEL_NOT_FOUND", e.getMessage()); + } + } + + /** The logical step used for model selection during routing. */ + protected abstract WikiJobStep routingStep(); + + /** The primary stage to transition to before processing starts. */ + protected abstract WikiJobStage mainStage(); + + /** Subclass implements the actual processing logic. */ + protected abstract void doProcess(WikiProcessingJobEntity job, Long modelId); + + /** Hook called after successful completion (optional override). */ + protected void onSuccess(WikiProcessingJobEntity job) {} + + private void handleHardError(WikiProcessingJobEntity job, WikiHardModelException e) { + try { + Long fallbackModelId = routingService.selectFallbackModel( + job, routingStep(), e.getErrorCode()); + job.setCurrentModelId(fallbackModelId); + doProcess(job, fallbackModelId); + jobService.transition(job.getId(), WikiJobStage.COMPLETED); + onSuccess(job); + } catch (Exception fallbackEx) { + jobService.recordHardError(job.getId(), e.getErrorCode(), e.getMessage()); + } + } + + private void handleSoftError(WikiProcessingJobEntity job, WikiSoftModelException e) { + jobService.recordSoftError(job.getId(), e.getErrorCode(), e.getMessage()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java new file mode 100644 index 00000000..8de827ad --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageCitationEntity.java @@ -0,0 +1,36 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.*; +import lombok.Data; + +import java.math.BigDecimal; +import java.time.LocalDateTime; + +/** + * RFC-029: Wiki page citation entity — links a page to the chunks it was derived from. + */ +@Data +@TableName("mate_wiki_page_citation") +public class WikiPageCitationEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long pageId; + + private Long chunkId; + + private Integer paragraphIdx; + + private String anchorText; + + private BigDecimal confidence; + + private String createdBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java index ec9c34e3..554885ba 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java @@ -42,7 +42,13 @@ public class WikiPageEntity { @TableField(updateStrategy = FieldStrategy.ALWAYS) private String sourceRawIds; - /** 版本号(每次 AI 更新递增) */ + /** Page type: entity / concept / source / synthesis */ + private String pageType; + + /** Purpose hint for LLM ingest routing */ + private String purposeHint; + + /** Version number (incremented on each AI update) */ private Integer version; /** 最后更新者:ai / manual */ diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/relation/DirectLinkSignal.java b/mateclaw-server/src/main/java/vip/mate/wiki/relation/DirectLinkSignal.java new file mode 100644 index 00000000..059ee813 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/relation/DirectLinkSignal.java @@ -0,0 +1,70 @@ +package vip.mate.wiki.relation; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageMapper; +import vip.mate.wiki.service.WikiPageService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * RFC-029: Direct-link signal — pages connected via [[wikilinks]] + * are related (weight = 2.0). Checks both outgoing and incoming links. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class DirectLinkSignal implements RelationSignalStrategy { + + private final WikiPageMapper pageMapper; + private final WikiPageService pageService; + private final ObjectMapper objectMapper; + + @Override + public String signalName() { return "direct_link"; } + + @Override + public double weight() { return 2.0; } + + @Override + public Map score(Long seedPageId, Long kbId) { + WikiPageEntity seed = pageMapper.selectById(seedPageId); + if (seed == null) return Map.of(); + + Map scores = new HashMap<>(); + + // Outgoing links: slugs this page links to + List outgoing = parseStringList(seed.getOutgoingLinks()); + for (String slug : outgoing) { + WikiPageEntity target = pageService.getBySlug(kbId, slug); + if (target != null) { + scores.put(target.getId(), weight()); + } + } + + // Incoming links: pages whose outgoingLinks contain seed's slug + List inbound = pageService.getBacklinks(kbId, seed.getSlug()); + for (WikiPageEntity p : inbound) { + scores.merge(p.getId(), weight(), Double::sum); + } + + scores.remove(seedPageId); + return scores; + } + + private List parseStringList(String json) { + if (json == null || json.isBlank()) return List.of(); + try { + return objectMapper.readValue(json, new TypeReference<>() {}); + } catch (Exception e) { + log.warn("[DirectLinkSignal] Failed to parse outgoingLinks: {}", e.getMessage()); + return List.of(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/relation/RelationSignalStrategy.java b/mateclaw-server/src/main/java/vip/mate/wiki/relation/RelationSignalStrategy.java new file mode 100644 index 00000000..9b27fc11 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/relation/RelationSignalStrategy.java @@ -0,0 +1,27 @@ +package vip.mate.wiki.relation; + +import java.util.Map; + +/** + * RFC-029: Strategy interface for computing a single relation signal + * between a seed page and other pages in the knowledge base. + *

+ * Each implementation returns a map of pageId → raw score for pages + * that have a positive signal. Pages not in the map score 0. + * The seed page itself is filtered out by {@link vip.mate.wiki.service.WikiRelationService}. + */ +public interface RelationSignalStrategy { + + String signalName(); + + double weight(); + + /** + * Compute scores for pages related to the given seed page. + * + * @param seedPageId the seed page ID + * @param kbId the knowledge base ID + * @return pageId → weighted score (only positive entries) + */ + Map score(Long seedPageId, Long kbId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/relation/SemanticNearSignal.java b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SemanticNearSignal.java new file mode 100644 index 00000000..c32b36c1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SemanticNearSignal.java @@ -0,0 +1,109 @@ +package vip.mate.wiki.relation; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageCitationMapper; +import vip.mate.wiki.service.WikiChunkService; +import vip.mate.wiki.service.WikiEmbeddingService; +import vip.mate.wiki.service.WikiPageService; + +import java.util.*; + +/** + * RFC-029: Semantic-near signal (optional, weight = 1.0). + * Uses chunk embeddings to find semantically similar pages. + * Silently returns empty when embedding service is unavailable. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class SemanticNearSignal implements RelationSignalStrategy { + + private static final double THRESHOLD = 0.85; + + private final WikiEmbeddingService embeddingService; + private final WikiChunkService chunkService; + private final WikiPageCitationMapper citationMapper; + private final WikiPageService pageService; + + @Override + public String signalName() { return "semantic_near"; } + + @Override + public double weight() { return 1.0; } + + @Override + public Map score(Long seedPageId, Long kbId) { + if (!embeddingService.isAvailable()) return Map.of(); + + // Get seed page's chunks and their embeddings + WikiPageEntity seed = pageService.getById(seedPageId); + if (seed == null) return Map.of(); + + // Collect seed chunk embeddings via citation mapper + List seedChunkIds = citationMapper.listWithRawByPageId(seedPageId) + .stream().map(c -> c.chunkId()).distinct().toList(); + if (seedChunkIds.isEmpty()) return Map.of(); + + // Get all chunks in the KB + List allChunks = chunkService.listByKbId(kbId); + Set seedChunkSet = new HashSet<>(seedChunkIds); + + // Get seed chunk vectors + List seedVectors = new ArrayList<>(); + for (WikiChunkEntity chunk : allChunks) { + if (seedChunkSet.contains(chunk.getId()) && chunk.getEmbedding() != null) { + seedVectors.add(WikiEmbeddingService.bytesToFloats(chunk.getEmbedding())); + } + } + if (seedVectors.isEmpty()) return Map.of(); + + // Average seed vectors into a single representative vector + float[] seedVec = averageVectors(seedVectors); + + // Score all non-seed chunks by cosine similarity, aggregate to page level + Map chunkScores = new HashMap<>(); + for (WikiChunkEntity chunk : allChunks) { + if (seedChunkSet.contains(chunk.getId()) || chunk.getEmbedding() == null) continue; + float[] vec = WikiEmbeddingService.bytesToFloats(chunk.getEmbedding()); + double sim = WikiEmbeddingService.cosine(seedVec, vec); + if (sim >= THRESHOLD) { + chunkScores.merge(chunk.getId(), sim, Math::max); + } + } + + // Map chunk scores to page scores via citation + Map pageScores = new HashMap<>(); + for (var entry : chunkScores.entrySet()) { + List pageIds = citationMapper.listPageIdsByChunkId(entry.getKey()); + for (Long pid : pageIds) { + if (!pid.equals(seedPageId)) { + pageScores.merge(pid, entry.getValue() * weight(), Math::max); + } + } + } + return pageScores; + } + + private float[] averageVectors(List vectors) { + if (vectors.size() == 1) return vectors.get(0); + int dim = vectors.get(0).length; + float[] avg = new float[dim]; + for (float[] v : vectors) { + for (int i = 0; i < dim; i++) avg[i] += v[i]; + } + float norm = 0; + for (int i = 0; i < dim; i++) { + avg[i] /= vectors.size(); + norm += avg[i] * avg[i]; + } + norm = (float) Math.sqrt(norm); + if (norm > 0) { + for (int i = 0; i < dim; i++) avg[i] /= norm; + } + return avg; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedChunkSignal.java b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedChunkSignal.java new file mode 100644 index 00000000..50b3ed49 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedChunkSignal.java @@ -0,0 +1,44 @@ +package vip.mate.wiki.relation; + +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Component; +import vip.mate.wiki.dto.PageCitationWithRaw; +import vip.mate.wiki.repository.WikiPageCitationMapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * RFC-029: Shared-chunk signal — pages that share the same source chunk + * are strongly related (weight = 5.0). + */ +@Component +@RequiredArgsConstructor +public class SharedChunkSignal implements RelationSignalStrategy { + + private final WikiPageCitationMapper citationMapper; + + @Override + public String signalName() { return "shared_chunk"; } + + @Override + public double weight() { return 5.0; } + + @Override + public Map score(Long seedPageId, Long kbId) { + List seedChunkIds = citationMapper.listWithRawByPageId(seedPageId) + .stream().map(PageCitationWithRaw::chunkId).distinct().toList(); + if (seedChunkIds.isEmpty()) return Map.of(); + + Map pageCount = new HashMap<>(); + citationMapper.listByChunkIds(seedChunkIds).stream() + .filter(ref -> !ref.pageId().equals(seedPageId)) + .forEach(ref -> pageCount.merge(ref.pageId(), 1L, Long::sum)); + + return pageCount.entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, + e -> e.getValue() * weight())); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedRawSignal.java b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedRawSignal.java new file mode 100644 index 00000000..9f50a772 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/relation/SharedRawSignal.java @@ -0,0 +1,61 @@ +package vip.mate.wiki.relation; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageCitationMapper; +import vip.mate.wiki.repository.WikiPageMapper; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * RFC-029: Shared-raw signal — pages derived from the same raw material + * are related (weight = 3.0). + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class SharedRawSignal implements RelationSignalStrategy { + + private final WikiPageCitationMapper citationMapper; + private final WikiPageMapper pageMapper; + private final ObjectMapper objectMapper; + + @Override + public String signalName() { return "shared_raw"; } + + @Override + public double weight() { return 3.0; } + + @Override + public Map score(Long seedPageId, Long kbId) { + WikiPageEntity seed = pageMapper.selectById(seedPageId); + if (seed == null) return Map.of(); + + List rawIds = parseRawIds(seed.getSourceRawIds()); + if (rawIds.isEmpty()) return Map.of(); + + Map scores = new HashMap<>(); + for (Long rawId : rawIds) { + citationMapper.listPageIdsByRawId(rawId).stream() + .filter(pid -> !pid.equals(seedPageId)) + .forEach(pid -> scores.merge(pid, weight(), Double::sum)); + } + return scores; + } + + private List parseRawIds(String json) { + if (json == null || json.isBlank()) return List.of(); + try { + return objectMapper.readValue(json, new TypeReference<>() {}); + } catch (Exception e) { + log.warn("[SharedRawSignal] Failed to parse sourceRawIds: {}", e.getMessage()); + return List.of(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageCitationMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageCitationMapper.java new file mode 100644 index 00000000..20cf293c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageCitationMapper.java @@ -0,0 +1,41 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.*; +import vip.mate.wiki.dto.ChunkPageRef; +import vip.mate.wiki.dto.PageCitationWithRaw; +import vip.mate.wiki.model.WikiPageCitationEntity; + +import java.util.Collection; +import java.util.List; + +/** + * RFC-029: Wiki page citation mapper — bidirectional page↔chunk queries. + */ +@Mapper +public interface WikiPageCitationMapper extends BaseMapper { + + @Select("SELECT c.id, c.page_id, c.chunk_id, wc.raw_id, c.paragraph_idx, c.anchor_text, c.confidence " + + "FROM mate_wiki_page_citation c " + + "JOIN mate_wiki_chunk wc ON c.chunk_id = wc.id " + + "WHERE c.page_id = #{pageId} AND c.deleted = 0") + List listWithRawByPageId(@Param("pageId") Long pageId); + + @Select("SELECT page_id FROM mate_wiki_page_citation " + + "WHERE chunk_id = #{chunkId} AND deleted = 0") + List listPageIdsByChunkId(@Param("chunkId") Long chunkId); + + @Select("SELECT DISTINCT c.page_id FROM mate_wiki_page_citation c " + + "JOIN mate_wiki_chunk wc ON c.chunk_id = wc.id " + + "WHERE wc.raw_id = #{rawId} AND c.deleted = 0 AND wc.deleted = 0") + List listPageIdsByRawId(@Param("rawId") Long rawId); + + @Update("UPDATE mate_wiki_page_citation SET deleted = 1 WHERE page_id = #{pageId}") + void softDeleteByPageId(@Param("pageId") Long pageId); + + @Select("") + List listByChunkIds(@Param("chunkIds") Collection chunkIds); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageMapper.java index 87bae508..4d2ef695 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageMapper.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiPageMapper.java @@ -4,12 +4,14 @@ import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; import org.apache.ibatis.annotations.Select; +import vip.mate.wiki.dto.WikiPageLite; import vip.mate.wiki.model.WikiPageEntity; +import java.util.Collection; import java.util.List; /** - * Wiki 页面 Mapper + * Wiki page mapper * * @author MateClaw Team */ @@ -17,8 +19,8 @@ import java.util.List; public interface WikiPageMapper extends BaseMapper { /** - * DB 级别关键词搜索(H2 + MySQL 通用 LIKE)。 - * 不 SELECT content CLOB,避免全量加载到 Java 内存。 + * DB keyword search (H2 + MySQL compatible LIKE). + * Does not SELECT content CLOB to avoid loading large blobs into Java heap. */ @Select("SELECT id, kb_id, slug, title, summary, source_raw_ids, last_updated_by " + "FROM mate_wiki_page " + @@ -27,4 +29,55 @@ public interface WikiPageMapper extends BaseMapper { " OR LOWER(content) LIKE #{pattern}) " + "ORDER BY title LIMIT 20") List searchByKeyword(@Param("kbId") Long kbId, @Param("pattern") String pattern); + + // ==================== RFC-029: Relation model ==================== + + /** + * Batch-fetch lightweight page projections by IDs (no content). + */ + @Select("") + List selectBatchLite(@Param("ids") Collection ids); + + /** + * List all pages as lightweight projections (no content). + */ + @Select("SELECT id, slug, title, summary FROM mate_wiki_page " + + "WHERE kb_id = #{kbId} AND deleted = 0 ORDER BY update_time DESC") + List selectAllLite(@Param("kbId") Long kbId); + + /** + * Fetch only the content column for a single page (lazy-load for snippet extraction). + */ + @Select("SELECT content FROM mate_wiki_page WHERE id = #{id} AND deleted = 0") + String selectContentById(@Param("id") Long id); + + // ==================== RFC-032: Two-phase keyword search ==================== + + /** + * Phase 1 (fast): search only title + summary columns. + */ + @Select("SELECT id FROM mate_wiki_page " + + "WHERE kb_id = #{kbId} AND deleted = 0 " + + "AND (LOWER(title) LIKE #{kw} OR LOWER(summary) LIKE #{kw}) " + + "LIMIT #{limit}") + List searchFastIds(@Param("kbId") Long kbId, + @Param("kw") String kw, + @Param("limit") int limit); + + /** + * Phase 2 (slow): search full content, excluding already-found IDs. + */ + @Select("") + List searchContentIds(@Param("kbId") Long kbId, + @Param("kw") String kw, + @Param("excludeIds") List excludeIds, + @Param("limit") int limit); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiProcessingJobMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiProcessingJobMapper.java new file mode 100644 index 00000000..8fbbacae --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiProcessingJobMapper.java @@ -0,0 +1,42 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.*; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; + +import java.util.List; +import java.util.Optional; + +/** + * RFC-030: Wiki processing job mapper. + */ +@Mapper +public interface WikiProcessingJobMapper extends BaseMapper { + + @Select("SELECT * FROM mate_wiki_processing_job " + + "WHERE kb_id = #{kbId} AND status = 'queued' AND deleted = 0 " + + "ORDER BY create_time ASC LIMIT #{limit}") + List listQueued(@Param("kbId") Long kbId, @Param("limit") int limit); + + @Select("SELECT * FROM mate_wiki_processing_job " + + "WHERE raw_id = #{rawId} AND deleted = 0 ORDER BY create_time DESC LIMIT 1") + Optional findLatestByRawId(@Param("rawId") Long rawId); + + /** + * List all non-deleted jobs for a KB (for stats/dashboard queries). + */ + @Select("SELECT * FROM mate_wiki_processing_job " + + "WHERE kb_id = #{kbId} AND deleted = 0 " + + "ORDER BY create_time DESC LIMIT #{limit}") + List listByKbId(@Param("kbId") Long kbId, @Param("limit") int limit); + + /** + * Recover stuck jobs on startup: reset routing/*_running stages back to queued. + */ + @Update("UPDATE mate_wiki_processing_job " + + "SET status = 'queued', " + + " stage = COALESCE(resume_from_stage, 'queued'), " + + " update_time = CURRENT_TIMESTAMP(3) " + + "WHERE (stage = 'routing' OR stage LIKE '%_running') AND deleted = 0") + int recoverStuckJobs(); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiRawMaterialMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiRawMaterialMapper.java index 2f4ae307..777df692 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiRawMaterialMapper.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiRawMaterialMapper.java @@ -2,13 +2,27 @@ package vip.mate.wiki.repository; import com.baomidou.mybatisplus.core.mapper.BaseMapper; import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import vip.mate.wiki.dto.RawTitleRef; import vip.mate.wiki.model.WikiRawMaterialEntity; +import java.util.Collection; +import java.util.List; + /** - * Wiki 原始材料 Mapper + * Wiki raw material mapper * * @author MateClaw Team */ @Mapper public interface WikiRawMaterialMapper extends BaseMapper { + + /** + * RFC-032: Batch-fetch raw material titles by IDs (fixes N+1 in wiki_semantic_search). + */ + @Select("") + List selectBatchTitles(@Param("ids") Collection ids); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/retrieval/SnippetExtractor.java b/mateclaw-server/src/main/java/vip/mate/wiki/retrieval/SnippetExtractor.java new file mode 100644 index 00000000..beabedb5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/retrieval/SnippetExtractor.java @@ -0,0 +1,30 @@ +package vip.mate.wiki.retrieval; + +/** + * RFC-032: Extracts a context-aware snippet from page content + * around a query match point. + */ +public class SnippetExtractor { + + private static final int CONTEXT_CHARS = 150; + + /** + * Extract a snippet from content centered on the first occurrence of the query. + * If no exact match is found, returns the first ~300 characters. + */ + public static String extract(String content, String query) { + if (content == null || query == null) return null; + int idx = content.toLowerCase().indexOf(query.toLowerCase()); + if (idx < 0) { + return content.length() <= CONTEXT_CHARS * 2 + ? content + : content.substring(0, CONTEXT_CHARS * 2) + "..."; + } + int start = Math.max(0, idx - CONTEXT_CHARS); + int end = Math.min(content.length(), idx + query.length() + CONTEXT_CHARS); + String snippet = content.substring(start, end); + if (start > 0) snippet = "..." + snippet; + if (end < content.length()) snippet = snippet + "..."; + return snippet; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java index afc1dc8f..c41b0b49 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java @@ -2,52 +2,70 @@ package vip.mate.wiki.service; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.PageSearchResult; +import vip.mate.wiki.dto.RelatedPageResult; +import vip.mate.wiki.dto.WikiPageLite; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageMapper; +import vip.mate.wiki.retrieval.SnippetExtractor; import java.util.*; import java.util.stream.Collectors; /** - * RFC-011: 混合检索服务 + * RFC-011 + RFC-032: Hybrid retrieval service. *

- * 支持三种模式: - *

    - *
  • {@code keyword} — DB LIKE 搜索(现有 WikiPageService.searchPages)
  • - *
  • {@code semantic} — chunk 向量 cosine 相似度 → 回溯到 page
  • - *
  • {@code hybrid} — 两者融合,RRF (Reciprocal Rank Fusion) 排名
  • - *
- * - * @author MateClaw Team + * Three search modes: keyword (DB LIKE), semantic (chunk vectors), + * hybrid (RRF fusion). RFC-032 adds: N+1 fix, two-phase keyword search, + * relation boost, snippet extraction, and PageSearchResult DTO. */ @Slf4j @Service -@RequiredArgsConstructor public class HybridRetriever { private final WikiPageService pageService; private final WikiChunkService chunkService; private final WikiEmbeddingService embeddingService; private final WikiProperties properties; + private final WikiPageMapper pageMapper; + + @Autowired(required = false) + private WikiRelationService relationService; + + private static final double RELATION_BOOST = 0.15; + + public HybridRetriever(WikiPageService pageService, + WikiChunkService chunkService, + WikiEmbeddingService embeddingService, + WikiProperties properties, + WikiPageMapper pageMapper) { + this.pageService = pageService; + this.chunkService = chunkService; + this.embeddingService = embeddingService; + this.properties = properties; + this.pageMapper = pageMapper; + } public enum Mode { KEYWORD, SEMANTIC, HYBRID } /** - * 搜索结果(页面级) + * Legacy page hit record (kept for backward compatibility). */ public record PageHit(Long pageId, String slug, String title, String summary, double score) {} /** - * 搜索结果(chunk 级,语义搜索专用) + * Chunk-level search result (semantic search). */ public record ChunkHit(Long chunkId, Long rawId, String snippet, float score) {} /** - * 执行混合搜索,返回页面级结果 + * RFC-032: Enhanced search returning PageSearchResult with snippet and matchedBy metadata. */ - public List searchPages(Long kbId, String query, String modeStr, int topK) { + public List search(Long kbId, String query, String modeStr, int topK) { Mode mode = parseMode(modeStr); List semantic = List.of(); @@ -60,9 +78,7 @@ public class HybridRetriever { keyword = keywordSearch(kbId, query, topK * 3); } - // 如果 semantic 不可用(no embedding model),回退到 keyword if (mode == Mode.SEMANTIC && semantic.isEmpty()) { - log.debug("[HybridRetriever] Semantic unavailable, falling back to keyword"); keyword = keywordSearch(kbId, query, topK * 3); } @@ -75,21 +91,50 @@ public class HybridRetriever { fused = rrfFuse(semantic, keyword, 60); } - // 取 topK,装配 PageHit - return fused.stream() - .limit(topK) - .map(ri -> { - WikiPageEntity page = pageService.getById(ri.pageId); - if (page == null) return null; - return new PageHit(ri.pageId, page.getSlug(), page.getTitle(), - page.getSummary(), ri.score); - }) - .filter(Objects::nonNull) - .toList(); + // RFC-032: Relation boost (1-hop expansion on top-3 seeds) + fused = applyRelationBoost(fused, kbId, topK); + + // Batch-fetch page info (N+1 fix) + List topIds = fused.stream().limit(topK).map(ri -> ri.pageId).toList(); + if (topIds.isEmpty()) return List.of(); + + Map liteMap = pageMapper.selectBatchLite(topIds) + .stream().collect(Collectors.toMap(WikiPageLite::id, p -> p)); + + // Build result with snippets + List results = new ArrayList<>(); + for (RankedItem ri : fused.stream().limit(topK).toList()) { + WikiPageLite lite = liteMap.get(ri.pageId); + if (lite == null) continue; + + String snippet = null; + if (!ri.matchedBy.contains("relation_boost")) { + String content = pageMapper.selectContentById(ri.pageId); + if (content != null) { + snippet = SnippetExtractor.extract(content, query); + } + } + + String reason = buildReason(lite, ri.matchedBy, query); + results.add(new PageSearchResult( + lite.slug(), lite.title(), lite.summary(), + snippet != null ? snippet : lite.summary(), + ri.matchedBy, reason, ri.score)); + } + return results; } /** - * chunk 级语义搜索(Agent 直接拿 chunk 片段作为证据) + * Legacy searchPages — returns PageHit for backward compatibility. + */ + public List searchPages(Long kbId, String query, String modeStr, int topK) { + return search(kbId, query, modeStr, topK).stream() + .map(r -> new PageHit(null, r.slug(), r.title(), r.summary(), r.score())) + .toList(); + } + + /** + * Chunk-level semantic search. */ public List searchChunks(Long kbId, String query, int topK) { if (!embeddingService.isAvailable()) return List.of(); @@ -114,9 +159,9 @@ public class HybridRetriever { .toList(); } - // ==================== 内部方法 ==================== + // ==================== Internal methods ==================== - /** 语义搜索:chunk cosine → 聚合到 page(同页多 chunk 取最高分) */ + /** Semantic search: chunk cosine → aggregate to page level */ private List semanticSearch(Long kbId, String query, int limit) { float[] queryVec = embeddingService.embedQuery(kbId, query); if (queryVec == null) return List.of(); @@ -124,25 +169,19 @@ public class HybridRetriever { List allChunks = chunkService.listByKbId(kbId); if (allChunks.isEmpty()) return List.of(); - // chunk → score, 然后 需要映射到 page。 - // 当前没有 chunk → page 的直接关联(chunk 只有 rawId)。 - // 走 rawId → 找该 rawId 对应的所有 page(source_raw_ids 含该 rawId) - // 这是个近似:一个 rawId 可能产出多个 page,都算命中。 Map chunkScores = new HashMap<>(); for (WikiChunkEntity chunk : allChunks) { - if (chunk.getEmbedding() == null) continue; + if (chunk.getEmbedding() == null || chunk.getRawId() == null) continue; float[] vec = WikiEmbeddingService.bytesToFloats(chunk.getEmbedding()); float score = WikiEmbeddingService.cosine(queryVec, vec); - chunkScores.merge(chunk.getRawId(), score, Math::max); // rawId 级聚合 + chunkScores.merge(chunk.getRawId(), score, Math::max); } - // rawId → page IDs List allPages = pageService.listByKbId(kbId); Map pageScores = new HashMap<>(); for (WikiPageEntity page : allPages) { String rawIds = page.getSourceRawIds(); if (rawIds == null) continue; - // 解析 "[1,2,3]" 格式 for (String rawIdStr : rawIds.replaceAll("[\\[\\]\\s]", "").split(",")) { try { long rawId = Long.parseLong(rawIdStr.trim()); @@ -157,32 +196,108 @@ public class HybridRetriever { return pageScores.entrySet().stream() .sorted(Map.Entry.comparingByValue().reversed()) .limit(limit) - .map(e -> new RankedItem(e.getKey(), e.getValue())) + .map(e -> new RankedItem(e.getKey(), e.getValue(), List.of("semantic"))) .toList(); } - /** 关键词搜索:走现有 DB LIKE */ + /** + * RFC-032: Two-phase keyword search — fast path (title+summary) first, + * full content search only if needed to fill topK. + */ private List keywordSearch(Long kbId, String query, int limit) { - List results = pageService.searchPages(kbId, query); + String kw = "%" + query.toLowerCase() + .replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + "%"; + + // Phase 1: fast path (title + summary only) + List fastIds = pageMapper.searchFastIds(kbId, kw, limit); + List ranked = new ArrayList<>(); - for (int i = 0; i < Math.min(results.size(), limit); i++) { - // LIKE 无分数,用倒序排名作为伪分数 - ranked.add(new RankedItem(results.get(i).getId(), 1.0 / (i + 1))); + for (int i = 0; i < fastIds.size(); i++) { + ranked.add(new RankedItem(fastIds.get(i), 1.0 / (i + 1), List.of("title"))); } + + if (fastIds.size() >= limit) return ranked; + + // Phase 2: full content search (supplement) + List contentIds = pageMapper.searchContentIds(kbId, kw, fastIds, limit - fastIds.size()); + for (int i = 0; i < contentIds.size(); i++) { + ranked.add(new RankedItem(contentIds.get(i), + 1.0 / (fastIds.size() + i + 1), List.of("content"))); + } + return ranked; } - /** RRF 融合:score = Σ 1/(k + rank_i) */ + /** RRF fusion: score = Σ 1/(k + rank_i) */ private List rrfFuse(List a, List b, int k) { Map fused = new HashMap<>(); - for (int i = 0; i < a.size(); i++) fused.merge(a.get(i).pageId, 1.0 / (k + i + 1), Double::sum); - for (int i = 0; i < b.size(); i++) fused.merge(b.get(i).pageId, 1.0 / (k + i + 1), Double::sum); + Map> matchedByMap = new HashMap<>(); + + for (int i = 0; i < a.size(); i++) { + fused.merge(a.get(i).pageId, 1.0 / (k + i + 1), Double::sum); + matchedByMap.computeIfAbsent(a.get(i).pageId, x -> new ArrayList<>()).addAll(a.get(i).matchedBy); + } + for (int i = 0; i < b.size(); i++) { + fused.merge(b.get(i).pageId, 1.0 / (k + i + 1), Double::sum); + matchedByMap.computeIfAbsent(b.get(i).pageId, x -> new ArrayList<>()).addAll(b.get(i).matchedBy); + } + return fused.entrySet().stream() .sorted(Map.Entry.comparingByValue().reversed()) - .map(e -> new RankedItem(e.getKey(), e.getValue())) + .map(e -> new RankedItem(e.getKey(), e.getValue(), + matchedByMap.getOrDefault(e.getKey(), List.of()).stream().distinct().toList())) .toList(); } + /** + * RFC-032: 1-hop relation boost on top-3 seed pages. + */ + private List applyRelationBoost(List hits, Long kbId, int topK) { + if (relationService == null || hits.isEmpty()) return hits; + + List seedIds = hits.stream().limit(3).map(h -> h.pageId).toList(); + Map boostMap = new HashMap<>(); + + for (Long seedId : seedIds) { + List seedLites = pageMapper.selectBatchLite(List.of(seedId)); + if (seedLites.isEmpty()) continue; + WikiPageLite seed = seedLites.get(0); + try { + relationService.relatedPages(kbId, seed.slug(), 3) + .forEach(r -> { + // Find the page ID from slug + WikiPageEntity relPage = pageService.getBySlug(kbId, r.slug()); + if (relPage != null) { + boostMap.merge(relPage.getId(), RELATION_BOOST, Double::sum); + } + }); + } catch (Exception e) { + log.debug("[HybridRetriever] Relation boost failed for seed {}: {}", seed.slug(), e.getMessage()); + } + } + + Set existingIds = hits.stream().map(h -> h.pageId).collect(Collectors.toSet()); + boostMap.keySet().removeAll(existingIds); + + if (boostMap.isEmpty()) return hits; + + List expanded = new ArrayList<>(hits); + boostMap.forEach((pid, score) -> expanded.add( + new RankedItem(pid, score, List.of("relation_boost")))); + return expanded; + } + + private String buildReason(WikiPageLite lite, List matchedBy, String query) { + if (matchedBy.contains("relation_boost")) return "Structurally related to top search results"; + if (matchedBy.contains("title") && matchedBy.contains("semantic")) return "Title and semantic match"; + if (matchedBy.contains("title")) return "Title match"; + if (matchedBy.contains("semantic")) return "Semantic similarity"; + if (matchedBy.contains("content")) return "Content match"; + return "Keyword match"; + } + private Mode parseMode(String mode) { if (mode == null || mode.isBlank()) { String defaultMode = properties.getSearchDefaultMode(); @@ -199,5 +314,5 @@ public class HybridRetriever { }; } - private record RankedItem(Long pageId, double score) {} + private record RankedItem(Long pageId, double score, List matchedBy) {} } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCitationService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCitationService.java new file mode 100644 index 00000000..d2560f89 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiCitationService.java @@ -0,0 +1,74 @@ +package vip.mate.wiki.service; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; +import vip.mate.wiki.model.WikiChunkEntity; +import vip.mate.wiki.model.WikiPageCitationEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiPageCitationMapper; +import vip.mate.wiki.repository.WikiPageMapper; + +import java.math.BigDecimal; +import java.util.List; + +/** + * RFC-029: Builds citation records linking pages to their source chunks. + * Called asynchronously after page creation or update to avoid blocking + * the main processing pipeline. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiCitationService { + + private final WikiPageMapper pageMapper; + private final WikiChunkService chunkService; + private final WikiPageCitationMapper citationMapper; + private final ObjectMapper objectMapper; + + @Async + public void buildCitationsAsync(Long pageId, Long kbId) { + buildCitations(pageId, kbId); + } + + /** + * Rebuild all citation records for a page based on its sourceRawIds. + * Soft-deletes existing citations first, then creates new ones for + * all chunks belonging to the page's source raw materials. + */ + public void buildCitations(Long pageId, Long kbId) { + WikiPageEntity page = pageMapper.selectById(pageId); + if (page == null) return; + + List rawIds = parseRawIds(page.getSourceRawIds()); + citationMapper.softDeleteByPageId(pageId); + + for (Long rawId : rawIds) { + List chunks = chunkService.listByRawId(rawId); + for (WikiChunkEntity chunk : chunks) { + WikiPageCitationEntity citation = new WikiPageCitationEntity(); + citation.setPageId(pageId); + citation.setChunkId(chunk.getId()); + citation.setConfidence(BigDecimal.ONE); + citation.setCreatedBy("system"); + citationMapper.insert(citation); + } + } + + log.debug("[WikiCitation] Built citations for pageId={}, kbId={}", pageId, kbId); + } + + private List parseRawIds(String json) { + if (json == null || json.isBlank()) return List.of(); + try { + return objectMapper.readValue(json, new TypeReference<>() {}); + } catch (Exception e) { + log.warn("[WikiCitation] Failed to parse sourceRawIds: {}", e.getMessage()); + return List.of(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java index 819b5825..ca2826e4 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiContextService.java @@ -4,18 +4,17 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.PageSearchResult; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; -import java.util.ArrayList; import java.util.List; /** - * Wiki 上下文服务 + * Wiki context service — builds context for agent conversation injection. *

- * 为 Agent 对话构建 Wiki 知识库上下文,注入到系统提示词中。 - * - * @author MateClaw Team + * RFC-032: buildRelevantContext now delegates to HybridRetriever instead + * of using a custom keyword matching algorithm. */ @Slf4j @Service @@ -24,17 +23,14 @@ public class WikiContextService { private final WikiKnowledgeBaseService kbService; private final WikiPageService pageService; + private final HybridRetriever hybridRetriever; private final WikiProperties properties; /** - * 构建与用户消息相关的 Wiki 上下文(任务前知识注入) + * Build relevant wiki context for the current user message. *

- * 从用户消息中提取关键词,匹配 Wiki 页面的标题和摘要, - * 注入 top-3 相关页面的完整内容到 system prompt 中。 - * - * @param agentId Agent ID - * @param userMessage 用户当前消息 - * @return 相关 Wiki 页面内容,如果没有匹配则返回空字符串 + * RFC-032: Uses HybridRetriever for consistent search quality, + * returns snippet + reason instead of just summary. */ public String buildRelevantContext(Long agentId, String userMessage) { if (!properties.isEnabled() || userMessage == null || userMessage.isBlank()) { @@ -46,72 +42,46 @@ public class WikiContextService { return ""; } - // 从用户消息中提取关键词(简单分词:按非字母数字中文分割,过滤短词) - String[] keywords = userMessage.toLowerCase() - .replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", " ") - .trim() - .split("\\s+"); - if (keywords.length == 0 || (keywords.length == 1 && keywords[0].isBlank())) { + Long kbId = kbs.get(0).getId(); + List hits = hybridRetriever.search(kbId, userMessage, "hybrid", 5); + if (hits.isEmpty()) { return ""; } - // 使用缓存的 listSummaries(不加载 content),按关键词评分 - record ScoredPage(WikiPageEntity page, int score) {} - List scored = new ArrayList<>(); - - for (WikiKnowledgeBaseEntity kb : kbs) { - List pages = pageService.listSummaries(kb.getId()); // 走缓存 - for (WikiPageEntity page : pages) { - String titleLower = page.getTitle() != null ? page.getTitle().toLowerCase() : ""; - String summaryLower = page.getSummary() != null ? page.getSummary().toLowerCase() : ""; - int score = 0; - for (String kw : keywords) { - if (kw.length() < 2) continue; - if (titleLower.contains(kw)) score += 3; - if (summaryLower.contains(kw)) score += 1; - } - if (score > 0) { - scored.add(new ScoredPage(page, score)); - } - } - } - - if (scored.isEmpty()) { - return ""; - } - - // 取 top-5 最相关页面,只注入摘要(不注入全文),受 token 预算限制 - scored.sort((a, b) -> Integer.compare(b.score, a.score)); - int topN = Math.min(5, scored.size()); - int maxChars = properties.getMaxContextChars(); - int totalChars = 0; - - StringBuilder sb = new StringBuilder(); - sb.append("\n"); + StringBuilder sb = new StringBuilder("\n"); sb.append("[Relevant wiki pages for this query. Use wiki_read_page(slug) for full content.]\n\n"); - for (int i = 0; i < topN; i++) { - WikiPageEntity page = scored.get(i).page; - String line = "- **" + page.getTitle() + "** (`" + page.getSlug() + "`)"; - if (page.getSummary() != null && !page.getSummary().isBlank()) { - line += " — " + page.getSummary(); - } - line += "\n"; - if (totalChars + line.length() > maxChars) { + int totalChars = 0; + int maxChars = properties.getMaxContextChars(); + + for (PageSearchResult hit : hits) { + String entry = buildContextEntry(hit); + if (totalChars + entry.length() > maxChars) { sb.append("- ... (use wiki_search_pages for more)\n"); break; } - sb.append(line); - totalChars += line.length(); + sb.append(entry); + totalChars += entry.length(); } sb.append(""); return sb.toString(); } + private String buildContextEntry(PageSearchResult hit) { + StringBuilder entry = new StringBuilder(); + entry.append("- **[[").append(hit.slug()).append("]]** ").append(hit.title()).append("\n"); + String excerpt = hit.snippet() != null ? hit.snippet() : hit.summary(); + if (excerpt != null) { + entry.append(" ").append(excerpt).append("\n"); + } + if (hit.reason() != null && !hit.reason().isBlank()) { + entry.append(" Relevance: ").append(hit.reason()).append("\n"); + } + entry.append("\n"); + return entry.toString(); + } + /** - * 构建指定 Agent 关联的 Wiki 上下文 - * - * @param agentId Agent ID - * @return Wiki 上下文字符串,如果没有关联知识库或页面则返回空字符串 + * Build full wiki context for agent system prompt. */ public String buildWikiContext(Long agentId) { if (!properties.isEnabled()) { @@ -140,8 +110,6 @@ public class WikiContextService { } sb.append(" (").append(pages.size()).append(" pages)\n\n"); - // 小 KB(≤20 页)保留 summary(成本低且是唯一的语义线索) - // 大 KB(>20 页)紧凑模式(slug + title) boolean compact = pages.size() > 20; for (WikiPageEntity page : pages) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java new file mode 100644 index 00000000..1cf12c6b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiLinkEnrichmentService.java @@ -0,0 +1,120 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.stereotype.Service; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.model.WikiPageEntity; + +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Semaphore; +import java.util.stream.Collectors; + +/** + * RFC-031: Lightweight wikilink enrichment service. + * Adds [[slug]] cross-references to page content without modifying + * the actual text. Corresponds to llm_wiki's enrich-wikilinks.ts. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiLinkEnrichmentService { + + private final WikiPageService pageService; + private final WikiModelRoutingService routingService; + private final WikiProperties wikiProperties; + + private static final ExecutorService WIKI_EXECUTOR = + Executors.newVirtualThreadPerTaskExecutor(); + + /** + * Enrich a single page with [[wikilinks]]. + */ + public void enrichPage(Long pageId, Long modelId) { + WikiPageEntity page = pageService.getById(pageId); + if (page == null || page.getContent() == null) return; + + String index = buildIndexPrompt(page.getKbId()); + ChatModel chatModel = routingService.buildChatModel(modelId); + String enriched = callEnrichLlm(chatModel, page.getContent(), index); + + if (enriched != null + && enriched.length() >= page.getContent().length() * wikiProperties.getWikilinkMinContentRatio()) { + page.setContent(enriched); + page.setOutgoingLinks(pageService.extractLinksAsJson(enriched)); + pageService.updateById(page); + } + } + + /** + * Batch-enrich all pages in a KB (e.g. after initial ingest). + */ + public void enrichAllPages(Long kbId, Long modelId) { + List pages = pageService.listByKbIdWithContent(kbId); + String index = buildIndexPrompt(kbId); + Semaphore sem = new Semaphore(wikiProperties.getMaxParallelPhaseBPages()); + + for (WikiPageEntity page : pages) { + sem.acquireUninterruptibly(); + WIKI_EXECUTOR.submit(() -> { + try { + enrichPageWithIndex(page, modelId, index); + } finally { + sem.release(); + } + }); + } + } + + private void enrichPageWithIndex(WikiPageEntity page, Long modelId, String index) { + ChatModel chatModel = routingService.buildChatModel(modelId); + String enriched = callEnrichLlm(chatModel, page.getContent(), index); + if (enriched != null + && enriched.length() >= page.getContent().length() * wikiProperties.getWikilinkMinContentRatio()) { + page.setContent(enriched); + page.setOutgoingLinks(pageService.extractLinksAsJson(enriched)); + pageService.updateById(page); + } + } + + private String callEnrichLlm(ChatModel model, String content, String index) { + String systemPrompt = """ + You are a wiki cross-referencing assistant. + Your ONLY job: add [[slug]] markers around entity and concept names that appear in the wiki index. + Rules: + - Do NOT change any content, rewrite sentences, or add new text. + - Do NOT modify YAML frontmatter. + - Only wrap existing words/phrases with [[ and ]]. + - Use the exact slug from the wiki index (not the title). + - Return the COMPLETE page text with [[wikilinks]] added. + """; + String userPrompt = "Wiki Index (slug → title):\n" + index + "\n\nPage content:\n" + content; + + try { + ChatResponse response = model.call( + new Prompt(List.of( + new SystemMessage(systemPrompt), + new UserMessage(userPrompt) + )) + ); + return response.getResult().getOutput().getText(); + } catch (Exception e) { + log.warn("[WikiEnrich] Failed to enrich page: {}", e.getMessage()); + return null; + } + } + + private String buildIndexPrompt(Long kbId) { + return pageService.listSummaries(kbId).stream() + .map(p -> p.getSlug() + " → " + p.getTitle()) + .collect(Collectors.joining("\n")); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java index 30f68252..4d842068 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiPageService.java @@ -168,6 +168,17 @@ public class WikiPageService { return pageMapper.selectById(id); } + /** + * Direct update by entity (used by enrichment service). + */ + @Transactional + public void updateById(WikiPageEntity entity) { + pageMapper.updateById(entity); + if (entity.getKbId() != null) { + evictSummaryCache(entity.getKbId()); + } + } + /** * 创建新 Wiki 页面 */ diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index a5c5af2e..bfcec07a 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -53,8 +53,13 @@ public class WikiProcessingService { private final AgentGraphBuilder agentGraphBuilder; private final ObjectMapper objectMapper; private final WikiProgressBus progressBus; + private final WikiCitationService citationService; - /** 并行 chunk / 材料处理执行器(JDK 21 虚拟线程);Listener 跨包需要引用,故 public */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + @org.springframework.context.annotation.Lazy + private vip.mate.wiki.job.WikiProcessingJobService wikiJobService; + + /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); /** @@ -132,6 +137,15 @@ public class WikiProcessingService { kbService.updateStatus(kb.getId(), "processing"); + // RFC-030 §9.1: create a processing job record before starting + if (wikiJobService != null) { + try { + wikiJobService.createHeavyIngest(kb.getId(), rawId); + } catch (Exception e) { + log.warn("[Wiki] Failed to create heavy ingest job record for raw={}: {}", rawId, e.getMessage()); + } + } + // RFC-012 M2 v2 UI v2:为本次 raw 处理创建共享进度计数器(多 chunk 共享,避免 race) progressCounters.put(rawId, new ProgressCounter()); rawService.updateProgress(rawId, "route", 0, 0); // UI 立即看到 indeterminate 滑条 @@ -777,8 +791,10 @@ public class WikiProcessingService { } String sourceRawIds = "[" + rawId + "]"; try { - pageService.createPage(kbId, slug, title, content, pageSummary, sourceRawIds); + WikiPageEntity created = pageService.createPage(kbId, slug, title, content, pageSummary, sourceRawIds); log.info("[Wiki] Phase B create page slug='{}' done (created)", slug); + // RFC-029: async citation build + citationService.buildCitationsAsync(created.getId(), kbId); return true; } catch (org.springframework.dao.DuplicateKeyException e) { // 兜底 2:select-then-create 在并发下不是原子操作。当 N 个 chunk 同时 @@ -846,8 +862,12 @@ public class WikiProcessingService { log.warn("[Wiki] Phase B merge page slug='{}' returned blank content, skipping", slug); return false; } - pageService.updatePageByAi(kbId, slug, content, summary, rawId); + WikiPageEntity updated = pageService.updatePageByAi(kbId, slug, content, summary, rawId); log.info("[Wiki] Phase B merge page slug='{}' done", slug); + // RFC-029: async citation rebuild + if (updated != null) { + citationService.buildCitationsAsync(updated.getId(), kbId); + } return true; } @@ -1129,11 +1149,129 @@ public class WikiProcessingService { return cls + ": " + trimmed; } - /** 瞬时错误的内部标记异常,确保空响应也能走重试路径 */ + /** Transient error marker to route empty responses through the retry path */ private static class TransientLlmException extends RuntimeException { TransientLlmException(String msg) { super(msg); } } + // ==================== RFC-030: Error classification ==================== + + /** + * Classify an exception into an error code aligned with RFC-009 ErrorType. + * + * @return error code string: AUTH_ERROR, BILLING, MODEL_NOT_FOUND, + * RATE_LIMIT, SERVER_ERROR, TIMEOUT, CONTENT_FILTER, UNKNOWN + */ + public String classifyErrorCode(Throwable t) { + Throwable cur = t; + int depth = 0; + while (cur != null && depth < 8) { + String className = cur.getClass().getSimpleName(); + if ("UnknownHostException".equals(className) + || "SSLHandshakeException".equals(className)) { + return "AUTH_ERROR"; + } + String msg = cur.getMessage(); + if (msg != null) { + String m = msg.toLowerCase(); + if (m.contains("401") || m.contains("unauthorized") || m.contains("403") + || m.contains("forbidden") || m.contains("invalid api key") + || m.contains("invalid_api_key") || m.contains("authentication")) { + return "AUTH_ERROR"; + } + if (m.contains("quota") || m.contains("insufficient_quota") || m.contains("billing")) { + return "BILLING"; + } + if (m.contains("model not found") || m.contains("model_not_found")) { + return "MODEL_NOT_FOUND"; + } + if (m.contains("429") || m.contains("rate_limit") || m.contains("too many requests")) { + return "RATE_LIMIT"; + } + if (m.contains("content_filter") || m.contains("content filter") + || m.contains("data_inspection_failed")) { + return "CONTENT_FILTER"; + } + if (m.contains("timeout") || m.contains("timed out")) { + return "TIMEOUT"; + } + if (m.contains("500") || m.contains("502") || m.contains("503") || m.contains("504")) { + return "SERVER_ERROR"; + } + } + cur = cur.getCause(); + depth++; + } + return "UNKNOWN"; + } + + // ==================== RFC-031: Methods for template delegation ==================== + + /** + * Repair a single page by regenerating its content. + * Used by LocalRepairTemplate. + */ + public void repairSinglePage(Long targetPageId, Long modelId) { + WikiPageEntity page = pageService.getById(targetPageId); + if (page == null) { + log.warn("[Wiki] repairSinglePage: page not found: {}", targetPageId); + return; + } + + WikiKnowledgeBaseEntity kb = kbService.getById(page.getKbId()); + if (kb == null) return; + + // Find source raw material + List rawIds = parseSourceRawIds(page.getSourceRawIds()); + if (rawIds.isEmpty()) { + log.warn("[Wiki] repairSinglePage: no source raw IDs for page {}", targetPageId); + return; + } + + WikiRawMaterialEntity raw = rawService.getById(rawIds.get(0)); + if (raw == null) return; + + String textContent = rawService.getTextContent(raw); + if (textContent == null || textContent.isBlank()) return; + + // Use existing two-phase single-page create logic + String existingPagesIndex = buildExistingPagesIndex(kb.getId()); + String configContent = kb.getConfigContent() != null ? kb.getConfigContent() : ""; + String createSystem = PromptLoader.loadPrompt("wiki/create-page-system"); + String createUserTemplate = PromptLoader.loadPrompt("wiki/create-page-user"); + String createUser = createUserTemplate + .replace("{config}", configContent) + .replace("{existing_pages}", existingPagesIndex) + .replace("{page_slug}", page.getSlug()) + .replace("{page_title}", page.getTitle()) + .replace("{page_summary}", page.getSummary() != null ? page.getSummary() : "") + .replace("{raw_title}", raw.getTitle()) + .replace("{raw_content}", textContent); + Prompt prompt = new Prompt(List.of( + new SystemMessage(createSystem), + new UserMessage(createUser) + )); + String response = callLlmWithResilientRetry(prompt, "repair page=" + page.getSlug()); + com.fasterxml.jackson.databind.JsonNode pageJson = parseJsonResponse(response); + if (pageJson == null) return; + + String content = pageJson.path("content").asText(""); + String summary = pageJson.path("summary").asText(""); + if (!content.isBlank()) { + pageService.updatePageByAi(kb.getId(), page.getSlug(), content, summary, rawIds.get(0)); + log.info("[Wiki] Repaired page: {} (kbId={})", page.getSlug(), kb.getId()); + } + } + + private List parseSourceRawIds(String json) { + if (json == null || json.isBlank()) return List.of(); + try { + return objectMapper.readValue(json, new com.fasterxml.jackson.core.type.TypeReference>() {}); + } catch (Exception e) { + return List.of(); + } + } + private JsonNode parseJsonResponse(String response) { if (response == null || response.isBlank()) return null; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRelationService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRelationService.java new file mode 100644 index 00000000..6b0e4a7d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRelationService.java @@ -0,0 +1,122 @@ +package vip.mate.wiki.service; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.dto.*; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.relation.RelationSignalStrategy; +import vip.mate.wiki.repository.WikiPageCitationMapper; +import vip.mate.wiki.repository.WikiPageMapper; + +import java.util.*; +import java.util.stream.Collectors; + +/** + * RFC-029: Wiki relation service — computes multi-signal structural + * relevance between pages using all registered {@link RelationSignalStrategy} beans. + */ +@Slf4j +@Service +public class WikiRelationService { + + private final List signals; + private final WikiPageMapper pageMapper; + private final WikiPageService pageService; + private final WikiPageCitationMapper citationMapper; + + public WikiRelationService(List signals, + WikiPageMapper pageMapper, + WikiPageService pageService, + WikiPageCitationMapper citationMapper) { + this.signals = signals; + this.pageMapper = pageMapper; + this.pageService = pageService; + this.citationMapper = citationMapper; + } + + /** + * Find pages related to a seed page, ranked by multi-signal score. + */ + public List relatedPages(Long kbId, String seedSlug, int topK) { + WikiPageEntity seed = pageService.getBySlug(kbId, seedSlug); + if (seed == null) return List.of(); + + Map totalScores = new HashMap<>(); + Map> signalHits = new HashMap<>(); + + for (RelationSignalStrategy signal : signals) { + try { + signal.score(seed.getId(), kbId).forEach((pid, s) -> { + totalScores.merge(pid, s, Double::sum); + signalHits.computeIfAbsent(pid, k -> new ArrayList<>()).add(signal.signalName()); + }); + } catch (Exception e) { + log.warn("[WikiRelation] Signal '{}' failed for seed={}: {}", + signal.signalName(), seedSlug, e.getMessage()); + } + } + + List topIds = totalScores.entrySet().stream() + .sorted(Map.Entry.comparingByValue().reversed()) + .limit(topK) + .map(Map.Entry::getKey) + .toList(); + + if (topIds.isEmpty()) return List.of(); + Map liteMap = pageMapper.selectBatchLite(topIds) + .stream().collect(Collectors.toMap(WikiPageLite::id, l -> l)); + + return topIds.stream() + .filter(liteMap::containsKey) + .map(pid -> new RelatedPageResult( + liteMap.get(pid).slug(), + liteMap.get(pid).title(), + liteMap.get(pid).summary(), + totalScores.get(pid), + signalHits.getOrDefault(pid, List.of()))) + .toList(); + } + + /** + * Explain the relation between two pages with a per-signal breakdown. + */ + public RelationExplanation explain(Long kbId, String slugA, String slugB) { + WikiPageEntity a = pageService.getBySlug(kbId, slugA); + WikiPageEntity b = pageService.getBySlug(kbId, slugB); + if (a == null || b == null) return RelationExplanation.notFound(); + + List breakdown = new ArrayList<>(); + double total = 0; + for (RelationSignalStrategy signal : signals) { + try { + Double score = signal.score(a.getId(), kbId).get(b.getId()); + if (score != null && score > 0) { + breakdown.add(new SignalScore(signal.signalName(), signal.weight(), score)); + total += score; + } + } catch (Exception e) { + log.warn("[WikiRelation] Signal '{}' failed for explain {}<->{}: {}", + signal.signalName(), slugA, slugB, e.getMessage()); + } + } + return new RelationExplanation(slugA, slugB, total, breakdown); + } + + /** + * Find all pages derived from a given raw material. + */ + public List pagesByRawId(Long rawId) { + List pageIds = citationMapper.listPageIdsByRawId(rawId); + if (pageIds.isEmpty()) return List.of(); + return pageMapper.selectBatchLite(pageIds); + } + + /** + * Find all pages that cite a given chunk. + */ + public List pagesByChunkId(Long chunkId) { + List pageIds = citationMapper.listPageIdsByChunkId(chunkId); + if (pageIds.isEmpty()) return List.of(); + return pageMapper.selectBatchLite(pageIds); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index bf680716..4341adfa 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -5,47 +5,79 @@ import cn.hutool.json.JSONObject; import cn.hutool.json.JSONUtil; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.ai.tool.annotation.Tool; import org.springframework.ai.tool.annotation.ToolParam; import org.springframework.stereotype.Component; +import vip.mate.wiki.dto.*; +import vip.mate.wiki.job.WikiProcessingJobService; +import vip.mate.wiki.job.event.WikiJobCreatedEvent; +import vip.mate.wiki.job.model.WikiProcessingJobEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; -import vip.mate.wiki.service.HybridRetriever; -import vip.mate.wiki.service.WikiKnowledgeBaseService; -import vip.mate.wiki.service.WikiPageService; -import vip.mate.wiki.service.WikiRawMaterialService; +import vip.mate.wiki.repository.WikiRawMaterialMapper; +import vip.mate.wiki.service.*; -import java.util.List; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; /** - * Wiki 知识库工具 + * Wiki knowledge base tools for agent conversations. *

- * 供 Agent 在对话中按需读取 Wiki 页面内容。 - * kbId 通过 agentId 自动解析,LLM 无需传递。 + * All tools auto-resolve kbId from agentId; LLM never needs to pass it directly. * * @author MateClaw Team */ @Slf4j @Component -@RequiredArgsConstructor public class WikiTool { private final WikiPageService pageService; private final WikiKnowledgeBaseService kbService; private final WikiRawMaterialService rawService; private final HybridRetriever hybridRetriever; + private final ObjectMapper objectMapper; + + @Autowired(required = false) + private WikiRelationService relationService; + + @Autowired(required = false) + private WikiProcessingJobService jobService; + + @Autowired(required = false) + private ApplicationEventPublisher eventPublisher; + + @Autowired(required = false) + private WikiRawMaterialMapper rawMaterialMapper; + + public WikiTool(WikiPageService pageService, + WikiKnowledgeBaseService kbService, + WikiRawMaterialService rawService, + HybridRetriever hybridRetriever, + ObjectMapper objectMapper) { + this.pageService = pageService; + this.kbService = kbService; + this.rawService = rawService; + this.hybridRetriever = hybridRetriever; + this.objectMapper = objectMapper; + } + + // ==================== RFC-032: Enhanced wiki_read_page ==================== @Tool(description = """ - 读取 Wiki 知识库中指定页面的完整内容。 - 当系统提示词中的 Wiki 页面摘要不够详细时,使用此工具获取完整内容。 - 返回 Markdown 格式的页面内容,包含 [[双向链接]] 和来源原始文件信息。 + Read a wiki page. Use maxChars to limit size (recommended: 3000-6000 for most tasks). + Use sectionHeading to read only one section by its heading text. """) public String wiki_read_page( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "页面标识符 (slug)") String slug) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page slug") String slug, + @ToolParam(description = "Max characters to return (null = full page)", required = false) Integer maxChars, + @ToolParam(description = "Section heading to extract (null = all sections)", required = false) String sectionHeading) { if (slug == null || slug.isBlank()) { return error("slug is required"); @@ -61,38 +93,66 @@ public class WikiTool { return error("Page not found: " + slug); } - // Agent 引用追踪 pageService.trackReference(kbId, slug); + String content = page.getContent(); + + if (sectionHeading != null && !sectionHeading.isBlank()) { + content = extractSection(content, sectionHeading); + } + if (maxChars != null && maxChars > 0) { + content = applyMaxChars(content, maxChars); + } + JSONObject result = JSONUtil.createObj() .set("title", page.getTitle()) .set("slug", page.getSlug()) .set("version", page.getVersion()) .set("lastUpdatedBy", page.getLastUpdatedBy()) - .set("content", page.getContent()) + .set("content", content) .set("sourceFiles", resolveSourceFiles(page.getSourceRawIds())); return result.toString(); } + // ==================== RFC-032: Enhanced wiki_list_pages ==================== + @Tool(description = """ - 列出 Wiki 知识库中的所有页面。 - 返回页面列表,包含标题、slug 和摘要。 + List wiki pages. Add query to filter by title keyword (max 30 results). + Without query returns all pages (use only for small KBs). """) public String wiki_list_pages( - @ToolParam(description = "当前 Agent 的 ID") Long agentId) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Title keyword filter (optional)", required = false) String query) { Long kbId = resolveKbId(agentId); if (kbId == null) { return error("No wiki knowledge base found for this agent"); } - List pages = pageService.listSummaries(kbId); + List pages; + if (query != null && !query.isBlank()) { + List ids = pageService.searchPages(kbId, query).stream() + .map(WikiPageEntity::getId).limit(30).toList(); + if (ids.isEmpty()) { + pages = List.of(); + } else { + pages = pageService.listSummaries(kbId).stream() + .filter(p -> ids.stream().anyMatch(id -> Objects.equals(id, p.getId()))) + .map(p -> new WikiPageLite(p.getId(), p.getSlug(), p.getTitle(), p.getSummary())) + .toList(); + } + } else { + pages = pageService.listSummaries(kbId).stream() + .map(p -> new WikiPageLite(p.getId(), p.getSlug(), p.getTitle(), p.getSummary())) + .toList(); + } + JSONArray arr = new JSONArray(); - for (WikiPageEntity page : pages) { + for (WikiPageLite page : pages) { arr.add(JSONUtil.createObj() - .set("title", page.getTitle()) - .set("slug", page.getSlug()) - .set("summary", page.getSummary())); + .set("slug", page.slug()) + .set("title", page.title()) + .set("summary", page.summary())); } return JSONUtil.createObj() @@ -102,15 +162,17 @@ public class WikiTool { .toString(); } + // ==================== RFC-032: Enhanced wiki_search_pages ==================== + @Tool(description = """ - 在 Wiki 知识库中搜索页面。 - 支持三种模式:keyword(关键词匹配)、semantic(语义向量相似度)、hybrid(两者融合,默认)。 - 返回匹配的页面列表及其来源文件。 + Search wiki pages. Returns snippet so you can judge relevance without reading the full page. + Default topK=5 is sufficient for most queries. """) public String wiki_search_pages( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "搜索关键词或自然语言问题") String query, - @ToolParam(description = "搜索模式:keyword | semantic | hybrid(默认 hybrid)", required = false) String mode) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Search query") String query, + @ToolParam(description = "Mode: keyword|semantic|hybrid (default: hybrid)", required = false) String mode, + @ToolParam(description = "Max results (default 5, max 20)", required = false) Integer topK) { if (query == null || query.isBlank()) { return error("query is required"); @@ -121,41 +183,44 @@ public class WikiTool { return error("No wiki knowledge base found for this agent"); } - // RFC-011:走混合检索 - List hits = hybridRetriever.searchPages(kbId, query, mode, 20); + int k = (topK != null && topK > 0) ? Math.min(topK, 20) : 5; + List results = hybridRetriever.search(kbId, query, mode, k); - // Agent 引用追踪 - for (HybridRetriever.PageHit h : hits) { - pageService.trackReference(kbId, h.slug()); + for (PageSearchResult r : results) { + pageService.trackReference(kbId, r.slug()); } JSONArray arr = new JSONArray(); - for (HybridRetriever.PageHit hit : hits) { + for (PageSearchResult r : results) { arr.add(JSONUtil.createObj() - .set("title", hit.title()) - .set("slug", hit.slug()) - .set("summary", hit.summary()) - .set("score", String.format("%.4f", hit.score()))); + .set("slug", r.slug()) + .set("title", r.title()) + .set("snippet", r.snippet() != null ? r.snippet() : r.summary()) + .set("matchedBy", r.matchedBy()) + .set("reason", r.reason() != null ? r.reason() : "") + .set("score", String.format("%.4f", r.score()))); } return JSONUtil.createObj() .set("kbId", kbId) .set("query", query) .set("mode", mode != null ? mode : "hybrid") - .set("matchCount", hits.size()) + .set("matchCount", results.size()) .set("pages", arr) .toString(); } + // ==================== RFC-032: N+1 fixed wiki_semantic_search ==================== + @Tool(description = """ - 在 Wiki 知识库中进行 chunk 级语义搜索。 - 返回与查询语义最接近的原始文本片段(chunk),包含相似度分数。 - 当 wiki_search_pages 返回的页面摘要不够具体时,使用此工具获取精确的源文本证据。 + Chunk-level semantic search in the wiki knowledge base. + Returns raw text fragments closest to the query with similarity scores. + Use when wiki_search_pages results are not specific enough. """) public String wiki_semantic_search( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "自然语言查询") String query, - @ToolParam(description = "返回条数(默认 5)", required = false) Integer topK) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Natural language query") String query, + @ToolParam(description = "Max results (default 5)", required = false) Integer topK) { if (query == null || query.isBlank()) { return error("query is required"); @@ -178,13 +243,21 @@ public class WikiTool { .toString(); } + // RFC-032: Batch-fetch raw titles (N+1 fix) + Set rawIds = hits.stream().map(HybridRetriever.ChunkHit::rawId).collect(Collectors.toSet()); + Map rawTitles; + if (rawMaterialMapper != null && !rawIds.isEmpty()) { + rawTitles = rawMaterialMapper.selectBatchTitles(rawIds) + .stream().collect(Collectors.toMap(RawTitleRef::id, RawTitleRef::title)); + } else { + rawTitles = Map.of(); + } + JSONArray arr = new JSONArray(); for (HybridRetriever.ChunkHit hit : hits) { - // 解析 raw material 标题 - WikiRawMaterialEntity raw = rawService.getById(hit.rawId()); arr.add(JSONUtil.createObj() .set("chunkId", hit.chunkId()) - .set("rawTitle", raw != null ? raw.getTitle() : "unknown") + .set("rawTitle", rawTitles.getOrDefault(hit.rawId(), "unknown")) .set("snippet", hit.snippet()) .set("score", String.format("%.4f", hit.score()))); } @@ -198,13 +271,12 @@ public class WikiTool { } @Tool(description = """ - 追溯 Wiki 页面的来源原始文件。 - 查询指定页面是由哪些原始文档生成的,返回文件名、类型、路径等信息。 - 用于回答"这个内容出自哪篇文档"类的问题。 + Trace the source raw materials for a wiki page. + Returns file names, types, and paths of the original documents. """) public String wiki_trace_source( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "页面标识符 (slug)") String slug) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page slug") String slug) { if (slug == null || slug.isBlank()) { return error("slug is required"); @@ -228,14 +300,13 @@ public class WikiTool { } @Tool(description = """ - 在 Wiki 知识库中创建新页面。 - 用于保存任务执行结果、分析报告、会议纪要等有价值的信息。 - 内容使用 Markdown 格式。页面标识符 (slug) 会从标题自动生成。 + Create a new wiki page. Used to save task results, analysis reports, etc. + Content should be Markdown. Slug is auto-generated from title. """) public String wiki_create_page( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "页面标题") String title, - @ToolParam(description = "页面内容 (Markdown 格式)") String content) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page title") String title, + @ToolParam(description = "Page content (Markdown)") String content) { if (title == null || title.isBlank()) { return error("title is required"); @@ -249,7 +320,6 @@ public class WikiTool { return error("No wiki knowledge base found for this agent. Create one first."); } - // 从标题生成 slug String slug = title.toLowerCase() .replaceAll("[^a-z0-9\\u4e00-\\u9fff]+", "-") .replaceAll("^-|-$", ""); @@ -257,17 +327,13 @@ public class WikiTool { slug = "page-" + System.currentTimeMillis(); } - // 检查 slug 是否已存在 WikiPageEntity existing = pageService.getBySlug(kbId, slug); if (existing != null) { slug = slug + "-" + System.currentTimeMillis() % 10000; } - // 生成摘要(取前 200 字符) String summary = content.length() > 200 ? content.substring(0, 200) + "..." : content; - WikiPageEntity page = pageService.createPage(kbId, slug, title, content, summary, null); - log.info("[WikiTool] Created page: {} (slug={}, kbId={})", title, slug, kbId); return JSONUtil.createObj() @@ -280,12 +346,11 @@ public class WikiTool { } @Tool(description = """ - 删除一个 AI 生成的 Wiki 页面。无法删除人工维护的页面(lastUpdatedBy = 'manual')。 - 用于清理过时、冗余或不准确的 Wiki 页面。 + Delete an AI-generated wiki page. Cannot delete manually curated pages. """) public String wiki_delete_page( - @ToolParam(description = "当前 Agent 的 ID") Long agentId, - @ToolParam(description = "要删除的页面 slug") String slug) { + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page slug to delete") String slug) { if (slug == null || slug.isBlank()) { return error("slug is required"); @@ -301,7 +366,6 @@ public class WikiTool { return error("Page not found: " + slug); } - // 安全保护:禁止删除人工维护的页面 if ("manual".equals(page.getLastUpdatedBy())) { return error("Cannot delete manually curated page: " + page.getTitle() + ". Please manage via admin UI."); } @@ -317,24 +381,104 @@ public class WikiTool { .toString(); } - /** - * 通过 agentId 自动解析关联的知识库 ID - *

- * 查找逻辑:Agent 专属 KB + 公共 KB(agent_id IS NULL),取第一个。 - */ + // ==================== RFC-029: Relation tools ==================== + + @Tool(description = """ + Find pages structurally related to a given page (shared sources, links, + semantic similarity). More reliable than keyword search for discovering connected knowledge. + """) + public String wiki_related_pages( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page slug") String slug, + @ToolParam(description = "Max results (default 5, max 10)", required = false) Integer topK) { + + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (relationService == null) return error("Relation service not available"); + + int k = (topK != null && topK > 0) ? Math.min(topK, 10) : 5; + List results = relationService.relatedPages(kbId, slug, k); + + JSONArray arr = new JSONArray(); + for (RelatedPageResult r : results) { + arr.add(JSONUtil.createObj() + .set("slug", r.slug()) + .set("title", r.title()) + .set("score", String.format("%.2f", r.score())) + .set("signals", r.signals())); + } + + return JSONUtil.createObj() + .set("slug", slug) + .set("relatedCount", results.size()) + .set("pages", arr) + .toString(); + } + + @Tool(description = """ + Explain why two wiki pages are related. Returns signal breakdown with scores. + """) + public String wiki_explain_relation( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "First page slug") String slugA, + @ToolParam(description = "Second page slug") String slugB) { + + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (relationService == null) return error("Relation service not available"); + + RelationExplanation ex = relationService.explain(kbId, slugA, slugB); + if (ex.breakdown().isEmpty()) return slugA + " and " + slugB + " have no detected relation."; + + StringBuilder sb = new StringBuilder("Relation score: ") + .append(String.format("%.2f", ex.totalScore())).append("\n"); + ex.breakdown().forEach(s -> sb.append(" ").append(s.signal()) + .append(": ").append(String.format("%.2f", s.score())).append("\n")); + return sb.toString(); + } + + // ==================== RFC-031: Enrichment tool ==================== + + @Tool(description = """ + Trigger lightweight wikilink enrichment for a specific page. + Does NOT regenerate content — only adds [[wikilink]] cross-references. + """) + public String wiki_enrich_page( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Page slug") String slug) { + + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (jobService == null || eventPublisher == null) return error("Job service not available"); + + WikiPageEntity page = pageService.getBySlug(kbId, slug); + if (page == null) return error("Page not found: " + slug); + + Long rawId = 0L; + try { + List rawIds = objectMapper.readValue( + page.getSourceRawIds() != null ? page.getSourceRawIds() : "[]", + new TypeReference>() {}); + if (!rawIds.isEmpty()) rawId = rawIds.get(0); + } catch (Exception ignored) {} + + WikiProcessingJobEntity job = jobService.createLightEnrich(kbId, rawId); + eventPublisher.publishEvent(new WikiJobCreatedEvent(job.getId())); + return "Wikilink enrichment queued for: " + slug; + } + + // ==================== Helpers ==================== + private Long resolveKbId(Long agentId) { List kbs = kbService.listByAgentId(agentId); return kbs.isEmpty() ? null : kbs.get(0).getId(); } - /** - * 将 sourceRawIds JSON 数组解析为原始文件信息列表 - */ private JSONArray resolveSourceFiles(String sourceRawIdsJson) { JSONArray result = new JSONArray(); if (sourceRawIdsJson == null || sourceRawIdsJson.isBlank()) return result; try { - List rawIds = new ObjectMapper().readValue(sourceRawIdsJson, new TypeReference>() {}); + List rawIds = objectMapper.readValue(sourceRawIdsJson, new TypeReference>() {}); for (Long rawId : rawIds) { WikiRawMaterialEntity raw = rawService.getById(rawId); if (raw != null) { @@ -351,6 +495,30 @@ public class WikiTool { return result; } + /** + * RFC-032: Extract a section from markdown content by heading text. + */ + private String extractSection(String content, String heading) { + if (content == null) return ""; + String escaped = Pattern.quote(heading.trim()); + Pattern p = Pattern.compile( + "(?m)^(#{1,3})\\s+" + escaped + "\\b.*?(?=^#{1,3}\\s|\\Z)", + Pattern.DOTALL | Pattern.MULTILINE + ); + Matcher m = p.matcher(content); + return m.find() ? m.group().strip() : content; + } + + /** + * RFC-032: Truncate content with a helpful message. + */ + private String applyMaxChars(String text, int maxChars) { + if (text == null || text.length() <= maxChars) return text; + return text.substring(0, maxChars) + + "\n\n[Content truncated at " + maxChars + " chars. " + + "Use sectionHeading param to read a specific section.]"; + } + private String error(String message) { return JSONUtil.createObj().set("error", message).toString(); } diff --git a/mateclaw-server/src/main/resources/application.yml b/mateclaw-server/src/main/resources/application.yml index 7863d8ec..a80c2875 100644 --- a/mateclaw-server/src/main/resources/application.yml +++ b/mateclaw-server/src/main/resources/application.yml @@ -133,7 +133,7 @@ mateclaw: enabled: true # 自适应降级(连续 miss 后短路到 NoOp,冷却后恢复) miss-threshold: 5 cool-down-ms: 60000 - # per-provider health tracker for the multi-model failover chain. + # RFC-009 P3.3: per-provider health tracker for the multi-model failover chain. # When a provider hits failure-threshold consecutive failures it enters a cooldown # window during which the chain walker skips it, avoiding repeated 5-retry stalls # against a known-broken provider on every conversation turn. @@ -160,7 +160,7 @@ mate: per-category: shell: 120 web: 30 - # RFC-008 tool-result three-layer budget (per-result spill + per-turn aggregate budget). + # RFC-008 Phase 3: tool-result three-layer budget (per-result spill + per-turn aggregate budget). # Layer 1 (per-tool cap) lives inside individual tools; Layer 2 spills oversized # single results to disk; Layer 3 enforces an aggregate cap on the combined # response size of one tool turn. The full output is preserved on disk and diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V21__provider_fallback_priority.sql b/mateclaw-server/src/main/resources/db/migration/h2/V21__provider_fallback_priority.sql index 9ef9d51e..9e87813c 100644 --- a/mateclaw-server/src/main/resources/db/migration/h2/V21__provider_fallback_priority.sql +++ b/mateclaw-server/src/main/resources/db/migration/h2/V21__provider_fallback_priority.sql @@ -1,4 +1,4 @@ --- ordered multi-provider fallback chain +-- RFC-009 Phase 1: ordered multi-provider fallback chain -- -- `fallback_priority` defines the order in which a provider is tried after the -- primary model exhausts retries: diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V23__wiki_relation_model.sql b/mateclaw-server/src/main/resources/db/migration/h2/V23__wiki_relation_model.sql new file mode 100644 index 00000000..a68e5a4a --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V23__wiki_relation_model.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS mate_wiki_page_citation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + page_id BIGINT NOT NULL, + chunk_id BIGINT NOT NULL, + paragraph_idx INT NOT NULL DEFAULT 0, + anchor_text VARCHAR(512), + confidence DECIMAL(4,3) NOT NULL DEFAULT 1.000, + created_by VARCHAR(32) NOT NULL DEFAULT 'system', + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_wpc_page ON mate_wiki_page_citation (page_id); +CREATE INDEX IF NOT EXISTS idx_wpc_chunk ON mate_wiki_page_citation (chunk_id); + +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS page_type VARCHAR(32) NOT NULL DEFAULT 'concept'; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS purpose_hint TEXT; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V24__wiki_processing_job.sql b/mateclaw-server/src/main/resources/db/migration/h2/V24__wiki_processing_job.sql new file mode 100644 index 00000000..4630a74a --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V24__wiki_processing_job.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS mate_wiki_processing_job ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + kb_id BIGINT NOT NULL, + raw_id BIGINT NOT NULL, + job_type VARCHAR(32) NOT NULL DEFAULT 'heavy_ingest', + stage VARCHAR(64) NOT NULL DEFAULT 'queued', + status VARCHAR(32) NOT NULL DEFAULT 'queued', + primary_model_id BIGINT, + current_model_id BIGINT, + fallback_chain_json TEXT, + retry_count INT NOT NULL DEFAULT 0, + max_retries INT NOT NULL DEFAULT 3, + error_code VARCHAR(64), + error_message TEXT, + resume_from_stage VARCHAR(64), + meta_json TEXT, + started_at DATETIME(3), + finished_at DATETIME(3), + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_wpj_raw ON mate_wiki_processing_job (raw_id); +CREATE INDEX IF NOT EXISTS idx_wpj_status ON mate_wiki_processing_job (status); +CREATE INDEX IF NOT EXISTS idx_wpj_kb ON mate_wiki_processing_job (kb_id, status); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V25__agent_provider_preference.sql b/mateclaw-server/src/main/resources/db/migration/h2/V25__agent_provider_preference.sql new file mode 100644 index 00000000..985f99d2 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V25__agent_provider_preference.sql @@ -0,0 +1,26 @@ +-- RFC-009 Phase 4 PR-3: per-agent provider preferences +-- +-- Lets each agent declare an ordered list of preferred provider ids. Empty +-- table for an agent (no rows) means "use the global fallback chain order" +-- — fully backwards compatible with pre-PR-3 behavior. When rows exist, +-- listed providers are tried in ascending sort_order before any non-listed +-- provider is considered. +-- +-- This is purely a routing hint. The runtime walker still gates each entry +-- through AvailableProviderPool / ProviderHealthTracker — a preferred +-- provider that is HARD-removed or in cooldown is still skipped. + +CREATE TABLE IF NOT EXISTS mate_agent_provider_preference ( + id BIGINT NOT NULL PRIMARY KEY, + agent_id BIGINT NOT NULL, + provider_id VARCHAR(128) NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted INT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_agent_provider + ON mate_agent_provider_preference(agent_id, provider_id); +CREATE INDEX IF NOT EXISTS idx_agent_provider_order + ON mate_agent_provider_preference(agent_id, sort_order); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V21__provider_fallback_priority.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V21__provider_fallback_priority.sql index 0a336863..ddf09921 100644 --- a/mateclaw-server/src/main/resources/db/migration/mysql/V21__provider_fallback_priority.sql +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V21__provider_fallback_priority.sql @@ -1,4 +1,4 @@ --- ordered multi-provider fallback chain +-- RFC-009 Phase 1: ordered multi-provider fallback chain -- -- `fallback_priority` defines the order in which a provider is tried after the -- primary model exhausts retries: diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V23__wiki_relation_model.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V23__wiki_relation_model.sql new file mode 100644 index 00000000..be190cca --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V23__wiki_relation_model.sql @@ -0,0 +1,29 @@ +CREATE TABLE IF NOT EXISTS mate_wiki_page_citation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + page_id BIGINT NOT NULL, + chunk_id BIGINT NOT NULL, + paragraph_idx INT NOT NULL DEFAULT 0, + anchor_text VARCHAR(512), + confidence DECIMAL(4,3) NOT NULL DEFAULT 1.000, + created_by VARCHAR(32) NOT NULL DEFAULT 'system', + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + INDEX idx_wpc_page (page_id), + INDEX idx_wpc_chunk (chunk_id) +); + +SET @c1 = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'page_type'); +SET @s1 = IF(@c1 = 0, + 'ALTER TABLE mate_wiki_page ADD COLUMN page_type VARCHAR(32) NOT NULL DEFAULT ''concept''', + 'SELECT 1'); +PREPARE p1 FROM @s1; EXECUTE p1; DEALLOCATE PREPARE p1; + +SET @c2 = (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'purpose_hint'); +SET @s2 = IF(@c2 = 0, + 'ALTER TABLE mate_wiki_page ADD COLUMN purpose_hint TEXT', + 'SELECT 1'); +PREPARE p2 FROM @s2; EXECUTE p2; DEALLOCATE PREPARE p2; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V24__wiki_processing_job.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V24__wiki_processing_job.sql new file mode 100644 index 00000000..db18e4a9 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V24__wiki_processing_job.sql @@ -0,0 +1,25 @@ +CREATE TABLE IF NOT EXISTS mate_wiki_processing_job ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + kb_id BIGINT NOT NULL, + raw_id BIGINT NOT NULL, + job_type VARCHAR(32) NOT NULL DEFAULT 'heavy_ingest', + stage VARCHAR(64) NOT NULL DEFAULT 'queued', + status VARCHAR(32) NOT NULL DEFAULT 'queued', + primary_model_id BIGINT, + current_model_id BIGINT, + fallback_chain_json TEXT, + retry_count INT NOT NULL DEFAULT 0, + max_retries INT NOT NULL DEFAULT 3, + error_code VARCHAR(64), + error_message TEXT, + resume_from_stage VARCHAR(64), + meta_json TEXT, + started_at DATETIME(3), + finished_at DATETIME(3), + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + INDEX idx_wpj_raw (raw_id), + INDEX idx_wpj_status (status), + INDEX idx_wpj_kb (kb_id, status) +); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V25__agent_provider_preference.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V25__agent_provider_preference.sql new file mode 100644 index 00000000..941f6e2d --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V25__agent_provider_preference.sql @@ -0,0 +1,24 @@ +-- RFC-009 Phase 4 PR-3: per-agent provider preferences +-- +-- Lets each agent declare an ordered list of preferred provider ids. Empty +-- table for an agent (no rows) means "use the global fallback chain order" +-- — fully backwards compatible with pre-PR-3 behavior. When rows exist, +-- listed providers are tried in ascending sort_order before any non-listed +-- provider is considered. +-- +-- This is purely a routing hint. The runtime walker still gates each entry +-- through AvailableProviderPool / ProviderHealthTracker — a preferred +-- provider that is HARD-removed or in cooldown is still skipped. + +CREATE TABLE IF NOT EXISTS mate_agent_provider_preference ( + id BIGINT NOT NULL PRIMARY KEY, + agent_id BIGINT NOT NULL, + provider_id VARCHAR(128) NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + enabled TINYINT(1) NOT NULL DEFAULT 1, + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + deleted INT NOT NULL DEFAULT 0, + UNIQUE KEY uk_agent_provider (agent_id, provider_id), + KEY idx_agent_provider_order (agent_id, sort_order) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java b/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java new file mode 100644 index 00000000..c2c10749 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java @@ -0,0 +1,78 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.model.ModelProviderEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * RFC-009 PR-3 — verifies the agent-preference reorder used by + * {@link AgentGraphBuilder#buildFallbackChain}: listed providers move to the + * front in their declared order; unlisted providers keep their original + * relative order; missing/duplicate preferences are ignored gracefully. + */ +class AgentGraphBuilderPreferenceTest { + + private static ModelProviderEntity p(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + return p; + } + + private static List ids(List ps) { + return ps.stream().map(ModelProviderEntity::getProviderId).toList(); + } + + @Test + @DisplayName("Empty preferences: original order preserved") + void noPreferences() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of()); + assertEquals(List.of("openai", "anthropic", "dashscope"), ids(out)); + } + + @Test + @DisplayName("Single preference: preferred provider moves to front, rest follow original order") + void singlePreferenceFront() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("dashscope")); + assertEquals(List.of("dashscope", "openai", "anthropic"), ids(out)); + } + + @Test + @DisplayName("Multiple preferences: preferred order matches declaration, rest stable") + void multiplePreferencesOrder() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope"), p("kimi")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("kimi", "anthropic")); + // kimi → anthropic → (rest in original order: openai, dashscope) + assertEquals(List.of("kimi", "anthropic", "openai", "dashscope"), ids(out)); + } + + @Test + @DisplayName("Preference references unknown provider: silently skipped") + void preferenceReferencesUnknown() { + var input = List.of(p("openai"), p("anthropic")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("ghost", "anthropic")); + assertEquals(List.of("anthropic", "openai"), ids(out)); + } + + @Test + @DisplayName("Duplicate preferences: each provider appears at most once") + void duplicatePreferencesDeduped() { + var input = List.of(p("openai"), p("anthropic")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("openai", "openai", "anthropic")); + assertEquals(List.of("openai", "anthropic"), ids(out)); + } + + @Test + @DisplayName("All providers preferred: input pure-reordered, no drops") + void allProvidersPreferred() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, + List.of("dashscope", "openai", "anthropic")); + assertEquals(List.of("dashscope", "openai", "anthropic"), ids(out)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java index 58ead87e..5227d185 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java @@ -8,7 +8,7 @@ import java.lang.reflect.Method; import static org.junit.jupiter.api.Assertions.assertEquals; /** - * classification tests for the new error types + * RFC-009 P3.2: classification tests for the new error types * ({@link NodeStreamingChatHelper.ErrorType#BILLING}, * {@link NodeStreamingChatHelper.ErrorType#MODEL_NOT_FOUND}). * diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java index 3d322669..226fa2c6 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java @@ -153,7 +153,7 @@ class NodeStreamingChatHelperFailoverTest { // ============================================================ @Test - @DisplayName("C4 (regression): primary BILLING still triggers fallback (unchanged P3.2)") + @DisplayName("C4 (regression): primary BILLING still triggers fallback (unchanged from RFC-009 P3.2)") void billingStillFallsBack() { ChatModel primary = errorModel(new RuntimeException("402 Payment Required: insufficient_quota")); ChatModel fallback = successModel("recovered via fallback"); diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java index 1765f33b..21107fca 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java @@ -13,7 +13,7 @@ import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.mock; /** - * smoke tests for the multi-model fallback chain wiring on + * RFC-009: smoke tests for the multi-model fallback chain wiring on * {@link NodeStreamingChatHelper}. * *

Full streaming-flow integration (ChatModel.stream / Flux mocking) is left @@ -88,7 +88,7 @@ class NodeStreamingChatHelperFallbackChainTest { } @Test - @DisplayName("EMPTY_RESPONSE / BILLING / MODEL_NOT_FOUND error types exist ") + @DisplayName("EMPTY_RESPONSE / BILLING / MODEL_NOT_FOUND error types exist (RFC-009 fallback triggers)") void fallbackTriggerErrorTypesExist() { // Compile-time safety net: these enum constants the streaming pipeline relies on // must not be renamed or removed without breaking the fallback contract. @@ -98,7 +98,7 @@ class NodeStreamingChatHelperFallbackChainTest { } @Test - @DisplayName("primary providerId is stored when supplied via the full constructor") + @DisplayName("RFC-009 P3.1: primary providerId is stored when supplied via the full constructor") void primaryProviderIdStored() throws Exception { NodeStreamingChatHelper helper = new NodeStreamingChatHelper( streamTracker, List.of(), null, null, "openai"); @@ -110,7 +110,7 @@ class NodeStreamingChatHelperFallbackChainTest { } @Test - @DisplayName("legacy constructors leave primaryProviderId null (tracking disabled)") + @DisplayName("RFC-009 P3.1: legacy constructors leave primaryProviderId null (tracking disabled)") void primaryProviderIdNullForLegacyConstructors() throws Exception { NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker); Field f = NodeStreamingChatHelper.class.getDeclaredField("primaryProviderId"); diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java new file mode 100644 index 00000000..f5ef3903 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java @@ -0,0 +1,269 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.AvailableProviderPool.RemovalSource; +import vip.mate.llm.failover.FallbackEntry; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * RFC-009 Phase 4 — verifies the three pool hooks wired into + * {@link NodeStreamingChatHelper}: + *

    + *
  1. Primary short-circuit when its provider id is not in the pool — + * primary is never even called, fallback runs first.
  2. + *
  3. Walker head filter — out-of-pool fallback entries are skipped.
  4. + *
  5. HARD error → {@code pool.remove}; SOFT error → pool unchanged.
  6. + *
+ * + *

Pool state must remain consistent across these three behaviors so a + * single misconfigured provider can't pollute every conversation turn.

+ */ +class NodeStreamingChatHelperPoolTest { + + private ChatStreamTracker streamTracker; + private ProviderHealthTracker healthTracker; + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + healthTracker = new ProviderHealthTracker(new ProviderHealthProperties()); + pool = new AvailableProviderPool(); + } + + private static ChatModel successModel(String text) { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(text), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + private static ChatModel errorModel(Throwable err) { + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.error(err)); + return m; + } + + /** Stream a single chunk with empty text and no tool calls — triggers EMPTY_RESPONSE (SOFT). */ + private static ChatModel emptyResponseModel() { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(""), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + private NodeStreamingChatHelper helper(List chain, String primary) { + return new NodeStreamingChatHelper(streamTracker, chain, null, healthTracker, primary, pool); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + // ============================================================ + // Hook 1: primary out-of-pool short-circuits the retry loop + // ============================================================ + + @Test + @DisplayName("Primary not in pool: skipped without being called, fallback wins") + void primaryOutOfPoolShortCircuits() { + // openai is HARD-removed from pool before the call + pool.add("dashscope"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "stale 401"); + + ChatModel primary = successModel("primary should never be called"); + ChatModel fallback = successModel("fallback wins"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h1", "reasoning"); + + assertEquals("fallback wins", result.text()); + verify(primary, never()).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Hook 2: walker skips out-of-pool fallback entries + // ============================================================ + + @Test + @DisplayName("Walker skips out-of-pool fallback and lands on the next eligible one") + void walkerSkipsOutOfPoolFallback() { + pool.add("openai"); // primary + pool.remove("anthropic", RemovalSource.BILLING, "402"); // first fallback dead + pool.add("dashscope"); // second fallback alive + + // Use AUTH_ERROR (HARD) on primary — triggers the immediate break-to-walker + // path. Picking SERVER_ERROR would burn 5 retries (~110s) and then exit + // without ever hitting the walker, which is unrelated to the property + // under test here. + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fbAnthropic = successModel("should be skipped"); + ChatModel fbDashscope = successModel("dashscope wins"); + var helper = helper(List.of( + new FallbackEntry("anthropic", fbAnthropic), + new FallbackEntry("dashscope", fbDashscope)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h2", "reasoning"); + + assertEquals("dashscope wins", result.text()); + verify(fbAnthropic, never()).stream(any(Prompt.class)); + verify(fbDashscope, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Hook 3a: primary HARD error evicts from pool + // ============================================================ + + @Test + @DisplayName("Primary AUTH_ERROR HARD-removes openai from pool with AUTH_ERROR source") + void primaryAuthErrorEvictsFromPool() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: bad key")); + ChatModel fallback = successModel("recovered"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3a", "reasoning"); + + assertFalse(pool.contains("openai"), "openai must be removed from pool after AUTH_ERROR"); + var reason = pool.snapshot().get("openai"); + assertNotNull(reason); + assertEquals(RemovalSource.AUTH_ERROR, reason.source()); + assertTrue(pool.contains("dashscope"), "successful fallback stays in pool"); + } + + @Test + @DisplayName("Primary BILLING HARD-removes with BILLING source (distinct from AUTH)") + void primaryBillingEvictsWithBillingSource() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("402 Payment Required: insufficient_quota")); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3b", "reasoning"); + + assertFalse(pool.contains("openai")); + assertEquals(RemovalSource.BILLING, pool.snapshot().get("openai").source()); + } + + @Test + @DisplayName("Primary MODEL_NOT_FOUND HARD-removes with MODEL_NOT_FOUND source") + void primaryModelNotFoundEvictsWithCorrectSource() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("404 model_not_found: gpt-99")); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning"); + + assertFalse(pool.contains("openai")); + assertEquals(RemovalSource.MODEL_NOT_FOUND, pool.snapshot().get("openai").source()); + } + + // ============================================================ + // Hook 3b: SOFT errors do NOT evict from pool + // ============================================================ + + @Test + @DisplayName("Primary EMPTY_RESPONSE (SOFT) keeps provider in pool, only records failure") + void primarySoftErrorKeepsInPool() { + pool.add("openai"); + pool.add("dashscope"); + + // EMPTY_RESPONSE is SOFT and breaks straight to fallback (no 5x retry) + // — keeps the test fast while still exercising the SOFT path. + ChatModel primary = emptyResponseModel(); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3d", "reasoning"); + + assertTrue(pool.contains("openai"), + "SOFT errors must NOT evict — health tracker cooldown handles transient blips"); + assertTrue(healthTracker.snapshot().get("openai").consecutiveFailures() > 0, + "SOFT failure must still be recorded by the health tracker"); + } + + // ============================================================ + // Hook 3c: fallback HARD errors also evict + // ============================================================ + + @Test + @DisplayName("Fallback AUTH_ERROR evicts the fallback provider and walker continues") + void fallbackHardErrorEvictsFallback() { + pool.add("openai"); + pool.add("anthropic"); + pool.add("dashscope"); + + // Use AUTH on primary so we reach the walker without burning 5 retries. + // The behavior under test is fallback eviction, not the primary path. + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: openai key")); + ChatModel fbBad = errorModel(new RuntimeException("401 Unauthorized: anthropic key")); + ChatModel fbGood = successModel("dashscope ok"); + var helper = helper(List.of( + new FallbackEntry("anthropic", fbBad), + new FallbackEntry("dashscope", fbGood)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h3e", "reasoning"); + + assertEquals("dashscope ok", result.text()); + assertFalse(pool.contains("anthropic"), "fallback that failed AUTH must be evicted"); + assertEquals(RemovalSource.AUTH_ERROR, pool.snapshot().get("anthropic").source()); + assertTrue(pool.contains("dashscope")); + } + + // ============================================================ + // Sanity: fail-open mode (null pool) — old call sites unchanged + // ============================================================ + + @Test + @DisplayName("Null pool: helper behaves as before (no NPE, no skipping)") + void nullPoolFailOpen() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fallback = successModel("ok"); + // 5-arg constructor — no pool wired + var helper = new NodeStreamingChatHelper(streamTracker, + List.of(new FallbackEntry("dashscope", fallback)), null, healthTracker, "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-failopen", "reasoning"); + + assertEquals("ok", result.text()); + // No pool to inspect — just confirm we didn't crash and fallback ran. + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java index 8908c8bd..97d3b478 100644 --- a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java @@ -7,7 +7,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; /** - * per-provider failure-count + cooldown logic. + * RFC-009 P3.3: per-provider failure-count + cooldown logic. */ class ProviderHealthTrackerTest { diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index f2f83169..56426cfb 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -282,6 +282,33 @@ export const modelApi = { http.post('/models/embedding/default', { modelId }), } +// ==================== Provider Pool (RFC-009 Phase 4) ==================== +export interface ProviderPoolEntry { + providerId: string + providerName: string + inPool: boolean + removalSource: string | null + removalMessage: string | null + removedAtMs: number | null + inCooldown: boolean + cooldownRemainingMs: number + consecutiveFailures: number +} + +export interface ReprobeResult { + providerId: string + success: boolean + latencyMs: number + errorMessage: string | null + inPool: boolean +} + +export const providerPoolApi = { + snapshot: () => http.get('/llm/provider-pool'), + reprobe: (providerId: string) => + http.post(`/llm/provider-pool/${encodeURIComponent(providerId)}/reprobe`), +} + // ==================== OAuth ==================== export const oauthApi = { authorize: () => http.get('/oauth/openai/authorize'), @@ -407,6 +434,30 @@ export const wikiApi = { // Processing processKB: (kbId: number) => http.post(`/wiki/knowledge-bases/${kbId}/process`), getProcessingStatus: (kbId: number) => http.get(`/wiki/knowledge-bases/${kbId}/processing-status`), + + // RFC-029: Relations + getRelatedPages: (kbId: number, slug: string, topK = 5) => + http.get(`/wiki/kb/${kbId}/pages/${encodeURIComponent(slug)}/related`, { params: { topK } }), + explainRelation: (kbId: number, slugA: string, slugB: string) => + http.get(`/wiki/kb/${kbId}/pages/${encodeURIComponent(slugA)}/relation/${encodeURIComponent(slugB)}`), + getPageCitations: (kbId: number, pageId: number) => + http.get(`/wiki/kb/${kbId}/pages/${pageId}/citations`), + + // RFC-030: Jobs + getWikiJobs: (kbId: number, rawId: number) => + http.get(`/wiki/kb/${kbId}/jobs`, { params: { rawId } }), + getKBStats: (kbId: number) => + http.get(`/wiki/kb/${kbId}/stats`), + + // RFC-031: Enrichment & Repair + enrichPage: (kbId: number, slug: string) => + http.post(`/wiki/kb/${kbId}/pages/${encodeURIComponent(slug)}/enrich`), + repairPage: (kbId: number, slug: string) => + http.post(`/wiki/kb/${kbId}/pages/${encodeURIComponent(slug)}/repair`), + + // RFC-032: Search preview + searchPreview: (kbId: number, data: { query: string; mode?: string; topK?: number }) => + http.post(`/wiki/kb/${kbId}/search-preview`, data), } // ==================== Workspace (Team) ==================== @@ -433,6 +484,11 @@ export const agentBindingApi = { unbindSkill: (agentId: string | number, skillId: number) => http.delete(`/agents/${agentId}/skills/${skillId}`), listTools: (agentId: string | number) => http.get(`/agents/${agentId}/tools`), setTools: (agentId: string | number, toolNames: string[]) => http.put(`/agents/${agentId}/tools`, toolNames), + // RFC-009 PR-3: per-agent provider preference order. Empty list = use global chain order. + listProviderPreferences: (agentId: string | number) => + http.get(`/agents/${agentId}/provider-preferences`), + setProviderPreferences: (agentId: string | number, providerIds: string[]) => + http.put(`/agents/${agentId}/provider-preferences`, providerIds), } // ==================== Dashboard ==================== diff --git a/mateclaw-ui/src/composables/useWikiJobPoller.ts b/mateclaw-ui/src/composables/useWikiJobPoller.ts new file mode 100644 index 00000000..a33f8a4c --- /dev/null +++ b/mateclaw-ui/src/composables/useWikiJobPoller.ts @@ -0,0 +1,51 @@ +import { ref, onMounted, onUnmounted, type Ref } from 'vue' +import { wikiApi } from '@/api/index' + +export interface WikiProcessingJob { + id: number + kbId: number + rawId: number + jobType: string + stage: string + status: string + primaryModelId: number | null + currentModelId: number | null + currentModelName?: string + errorCode: string | null + errorMessage: string | null + retryCount: number + startedAt: string | null + finishedAt: string | null + done?: number + total?: number +} + +/** + * RFC-033: Polls the latest processing job for a given raw material. + * Stops polling when the job reaches a terminal status. + */ +export function useWikiJobPoller(kbId: Ref, rawId: Ref) { + const job = ref(null) + let timer: ReturnType | null = null + + const poll = async () => { + if (!kbId.value || !rawId.value) return + try { + const jobs: any = await wikiApi.getWikiJobs(kbId.value, rawId.value) + const list = jobs.data || jobs || [] + job.value = list[0] ?? null + if (job.value && (job.value.status === 'running' || job.value.status === 'queued')) { + timer = setTimeout(poll, 3000) + } + } catch { + job.value = null + } + } + + onMounted(poll) + onUnmounted(() => { + if (timer) clearTimeout(timer) + }) + + return { job, refresh: poll } +} diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 3c809588..d08101c3 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -332,9 +332,25 @@ export default { protocolGemini: 'Gemini Native', protocolDashScope: 'DashScope Native', advancedHint: 'Use this for generation options such as temperature, max_tokens, and top_p.', - fallbackPriorityHint: 'Failover order after the primary model fails: 0 = excluded, 1 = first in line, 2 = second, and so on. Providers sharing the same value are tried in alphabetical order of their ID.', - fallbackBadge: 'Fallback #{priority}', - fallbackBadgeTitle: 'Position in the multi-model failover chain — lower numbers are tried first', + fallbackPriorityHint: 'Pool try-order (lower wins): 0 = excluded, 1 = first in line, 2 = second, and so on. Providers sharing the same value are tried in alphabetical order of their ID.', + fallbackBadge: 'Preferred #{priority}', + fallbackBadgeTitle: 'Position in the available pool — lower numbers are tried first', + // RFC-009 Phase 4: provider pool status badges + poolBadgeInPool: 'In Pool', + poolBadgeInPoolTitle: 'Passed startup probe; participates in failover', + poolBadgeRemoved: 'Removed', + poolBadgeRemovedTitle: 'Removed from pool ({source}): {message}', + poolBadgeCooldown: 'Cooling Down', + poolBadgeCooldownTitle: 'Recent failures; recovers automatically in {seconds}s', + poolSourceAuthError: 'API key invalid', + poolSourceBilling: 'Billing/quota issue', + poolSourceModelNotFound: 'Model not found', + poolSourceInitProbe: 'Startup probe failed', + poolSourceManual: 'Manually removed', + poolReprobe: 'Reprobe', + poolReprobing: 'Probing...', + poolReprobeOk: 'Probe passed; back in the available pool', + poolReprobeFail: 'Probe failed: {error}', searchHint: 'When enabled, the LLM will use its built-in search engine to retrieve real-time information (DashScope/Kimi/OpenAI supported).', searchStrategyDefault: 'Default', oauthTitle: 'OpenAI OAuth Login', @@ -352,7 +368,7 @@ export default { apiKeyPrefix: 'API Key Prefix', protocol: 'Protocol', generateKwargs: 'Generate Kwargs (JSON)', - fallbackPriority: 'Failover Priority', + fallbackPriority: 'Pool Try-Order', enableSearch: 'Built-in Search', searchStrategy: 'Search Strategy', modelId: 'Model ID', @@ -611,6 +627,7 @@ export default { basic: 'Basic', skills: 'Skills', tools: 'Tools', + providers: 'Providers', context: 'Context', }, columns: { @@ -683,8 +700,11 @@ export default { binding: { skillsHint: 'Select skills this agent can use. Leave empty to use all enabled skills.', toolsHint: 'Select tools this agent can use. Leave empty to use all enabled tools.', + providersHint: 'Preferred provider order for this agent (lower index tried first). Leave empty to use the global available-pool order. Cooling-down or pool-removed providers are still skipped automatically.', + providersAddHint: 'Click a provider below to add it to the preference list:', noSkills: 'No skills available', noTools: 'No tools available', + noProviderPreferences: 'No preferences set — the agent uses the global fallback chain order.', contextHint: 'Manage context files (e.g. AGENT.md) that define this agent\'s behavior, knowledge, and instructions.', goToContext: 'Edit Context Files', }, @@ -1233,6 +1253,64 @@ export default { progress: { preparing: 'Preparing…', }, + jobStage: { + queued: 'Queued', + routing: 'Routing', + phase_a_running: 'Analyzing', + phase_a_done: 'Analyzed', + phase_b_running: 'Generating', + enriching: 'Linking', + embedding: 'Embedding', + completed: 'Completed', + failed: 'Failed', + cancelled: 'Cancelled', + }, + jobInfo: { + currentModel: 'Model', + fallbackActive: 'Fallback active', + elapsedTime: 'Elapsed', + pagesProgress: '{done}/{total} pages', + }, + page: { + type: { + entity: 'Entity', + concept: 'Concept', + source: 'Source', + synthesis: 'Synthesis', + }, + enriched: 'Linked', + notEnriched: 'Not linked', + enrich: 'Add cross-links', + repair: 'Repair page', + citations: '{count} source chunks', + noCitations: 'No source citations found', + }, + relation: { + shared_chunk: 'Shared chunk', + shared_raw: 'Shared source', + direct_link: 'Direct link', + semantic_near: 'Semantic', + relatedPages: 'Related Pages', + explainRelation: 'Explain relation', + }, + configPanel: { + modelStrategy: 'Model Strategy', + stepModel: { + route: 'Analysis / Routing', + create_page: 'Page Creation', + merge_page: 'Page Merge', + enrich: 'Link Annotation', + summary: 'Summary Update', + }, + fallbackModels: 'Fallback Models', + searchPreview: 'Search Preview', + searchPreviewPlaceholder: 'Enter a user message to preview agent search results…', + searchPreviewRun: 'Test', + }, + stats: { + enrichedRatio: '{enriched}/{total} linked', + failedJobs: '{count} failed', + }, }, cronJobs: { kicker: 'Automation', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 51f02dc9..f674ba62 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -322,9 +322,25 @@ export default { protocolGemini: 'Gemini 原生', protocolDashScope: 'DashScope 原生', advancedHint: '用于补充 temperature、max_tokens、top_p 等生成参数。', - fallbackPriorityHint: '主模型失败后的失败切换顺序:0 = 不参与;1 = 第一顺位;2 = 第二顺位,依此类推。多个 provider 共用同一数字时按 ID 字典序。', - fallbackBadge: '兜底 #{priority}', - fallbackBadgeTitle: '该 provider 在多模型失败切换链中,数字越小越先尝试', + fallbackPriorityHint: '池内尝试顺序(数字越小越先):0 = 不参与;1 = 第一顺位;2 = 第二顺位,依此类推。多个 provider 共用同一数字时按 ID 字典序。', + fallbackBadge: '偏好 #{priority}', + fallbackBadgeTitle: '可用池内的尝试顺序,数字越小越先尝试', + // RFC-009 Phase 4: provider pool status badges + poolBadgeInPool: '可用', + poolBadgeInPoolTitle: '已通过启动体检,参与失败切换', + poolBadgeRemoved: '已下线', + poolBadgeRemovedTitle: '已从可用池移除({source}):{message}', + poolBadgeCooldown: '冷却中', + poolBadgeCooldownTitle: '近期连续失败,{seconds} 秒后自动恢复', + poolSourceAuthError: 'API Key 失效', + poolSourceBilling: '余额/计费问题', + poolSourceModelNotFound: '模型不存在', + poolSourceInitProbe: '启动体检失败', + poolSourceManual: '手动移除', + poolReprobe: '重新检测', + poolReprobing: '检测中...', + poolReprobeOk: '检测通过,已重新加入可用池', + poolReprobeFail: '检测失败:{error}', searchHint: '开启后,大模型将在回答时自动调用内置搜索引擎获取实时信息(DashScope/Kimi/OpenAI 支持)。', searchStrategyDefault: '默认', oauthTitle: 'OpenAI OAuth 登录', @@ -342,7 +358,7 @@ export default { apiKeyPrefix: 'API Key 前缀', protocol: '协议', generateKwargs: 'Generate Kwargs (JSON)', - fallbackPriority: '失败切换优先级', + fallbackPriority: '池内尝试顺序', enableSearch: '内置搜索', searchStrategy: '搜索策略', modelId: '模型 ID', @@ -611,6 +627,7 @@ export default { basic: '基本信息', skills: '技能', tools: '工具', + providers: '偏好 Provider', context: '上下文', }, columns: { @@ -683,8 +700,11 @@ export default { binding: { skillsHint: '选择此智能体可使用的技能。留空则使用所有已启用的技能。', toolsHint: '选择此智能体可使用的工具。留空则使用所有已启用的工具。', + providersHint: '此智能体优先使用的 Provider 顺序(数字越小越先尝试)。留空则按全局可用池顺序回退。Provider 进入冷却或被移出池时仍会被自动跳过。', + providersAddHint: '点击下方 Provider 加入偏好列表:', noSkills: '暂无可用技能', noTools: '暂无可用工具', + noProviderPreferences: '尚未配置偏好顺序,将按全局回退链顺序使用。', contextHint: '管理此智能体的上下文文件(如 AGENT.md),定义智能体的行为、知识和指令。', goToContext: '前往编辑上下文', }, @@ -1243,6 +1263,64 @@ export default { progress: { preparing: '准备中…', }, + jobStage: { + queued: '排队中', + routing: '选择模型', + phase_a_running: '分析中', + phase_a_done: '分析完成', + phase_b_running: '生成页面', + enriching: '补充链接', + embedding: '向量化', + completed: '已完成', + failed: '处理失败', + cancelled: '已取消', + }, + jobInfo: { + currentModel: '当前模型', + fallbackActive: '已启用备用模型', + elapsedTime: '已用时', + pagesProgress: '已生成 {done}/{total} 页', + }, + page: { + type: { + entity: '实体', + concept: '概念', + source: '来源', + synthesis: '综合', + }, + enriched: '已链接', + notEnriched: '未链接', + enrich: '添加交叉链接', + repair: '修复页面', + citations: '来源 {count} 个 chunk', + noCitations: '未找到来源引用', + }, + relation: { + shared_chunk: '共享片段', + shared_raw: '共享来源', + direct_link: '直接引用', + semantic_near: '语义相近', + relatedPages: '相关页面', + explainRelation: '查看关联原因', + }, + configPanel: { + modelStrategy: '模型策略', + stepModel: { + route: '分析/路由', + create_page: '页面生成', + merge_page: '页面合并', + enrich: '链接标注', + summary: '摘要更新', + }, + fallbackModels: '备选模型', + searchPreview: '检索测试', + searchPreviewPlaceholder: '输入用户消息,预览 Agent 检索结果…', + searchPreviewRun: '测试', + }, + stats: { + enrichedRatio: '{enriched}/{total} 已链接', + failedJobs: '{count} 个失败', + }, }, cronJobs: { kicker: '自动执行', diff --git a/mateclaw-ui/src/views/Agents.vue b/mateclaw-ui/src/views/Agents.vue index 7eb80611..c048f18c 100644 --- a/mateclaw-ui/src/views/Agents.vue +++ b/mateclaw-ui/src/views/Agents.vue @@ -162,6 +162,10 @@ {{ t('agents.tabs.tools', 'Tools') }} {{ selectedToolNames.length }} + @@ -260,6 +264,38 @@ + + +
+

{{ t('agents.binding.providersHint') }}

+ +
+
+ {{ idx + 1 }} + {{ providerNameById(pid) }} + {{ pid }} + + + +
+
+
{{ t('agents.binding.noProviderPreferences') }}
+ + +
+

{{ t('agents.binding.providersAddHint') }}

+ +
+

{{ provider.id }}

@@ -84,6 +115,17 @@ > {{ t('common.delete') }} + +
@@ -100,11 +142,16 @@ diff --git a/mateclaw-ui/src/views/Wiki/components/JobStageBar.vue b/mateclaw-ui/src/views/Wiki/components/JobStageBar.vue new file mode 100644 index 00000000..91974896 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/JobStageBar.vue @@ -0,0 +1,213 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/PageHeader.vue b/mateclaw-ui/src/views/Wiki/components/PageHeader.vue new file mode 100644 index 00000000..ba4bf011 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/PageHeader.vue @@ -0,0 +1,172 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue index f88ffedc..74b40d94 100644 --- a/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue +++ b/mateclaw-ui/src/views/Wiki/components/RawMaterialPanel.vue @@ -91,7 +91,23 @@
-
+ + + +
+ + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue index 602b927c..f4c6bca2 100644 --- a/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue +++ b/mateclaw-ui/src/views/Wiki/components/WikiConfig.vue @@ -5,25 +5,60 @@

{{ t('wiki.configDesc') }}

- +
+ +
+ {{ t('wiki.configPanel.modelStrategy') }} +
+
+ + +
+
+
+ +
+ + {{ chatModelOptions.find(m => String(m.id) === String(fId))?.name || fId }} + + + +
+
+ +
+ + - - @@ -58,6 +82,10 @@ import { useI18n } from 'vue-i18n' import { useWikiStore, type WikiPage } from '@/stores/useWikiStore' import { wikiApi } from '@/api/index' import { useMarkdownRenderer } from '@/composables/useMarkdownRenderer' +import { Link, SetUp } from '@element-plus/icons-vue' +import PageHeader from './PageHeader.vue' +import RelatedPagesPanel from './RelatedPagesPanel.vue' +import CitationDrawer from './CitationDrawer.vue' const { t } = useI18n() const store = useWikiStore() @@ -66,10 +94,11 @@ const { renderMarkdown } = useMarkdownRenderer() const editing = ref(false) const editContent = ref('') const backlinks = ref([]) +const citationDrawerOpen = ref(false) +const enrichToast = ref('') const renderedContent = computed(() => { if (!store.currentPage?.content) return '' - // Pre-process wiki links [[title]] before markdown rendering const content = store.currentPage.content.replace(/\[\[([^\]]+)\]\]/g, (_match, title) => { const slug = title.trim().toLowerCase().replace(/[^a-z0-9\u4e00-\u9fff\s-]/g, '').replace(/\s+/g, '-') return `${title}` @@ -81,7 +110,6 @@ watch(() => store.currentPage, async (page) => { if (page && store.currentKB) { editing.value = false editContent.value = page.content || '' - // Fetch backlinks try { const res: any = await wikiApi.getBacklinks(store.currentKB.id, page.slug) backlinks.value = res.data || [] @@ -111,12 +139,38 @@ async function handleDelete() { } } +async function handleEnrich() { + if (!store.currentKB || !store.currentPage) return + try { + await wikiApi.enrichPage(store.currentKB.id, store.currentPage.slug) + enrichToast.value = t('wiki.page.enrich') + '…' + setTimeout(() => { enrichToast.value = '' }, 5000) + } catch (e: any) { + console.error('[WikiViewer] Enrich failed:', e) + } +} + +async function handleRepair() { + if (!store.currentKB || !store.currentPage) return + try { + await wikiApi.repairPage(store.currentKB.id, store.currentPage.slug) + enrichToast.value = t('wiki.page.repair') + '…' + setTimeout(async () => { + enrichToast.value = '' + if (store.currentKB && store.currentPage) { + await store.loadPage(store.currentKB.id, store.currentPage.slug) + } + }, 5000) + } catch (e: any) { + console.error('[WikiViewer] Repair failed:', e) + } +} + async function openPage(slug: string) { if (!store.currentKB) return await store.loadPage(store.currentKB.id, slug) } -// Handle wiki link clicks via event delegation onMounted(() => { document.addEventListener('click', (e) => { const target = e.target as HTMLElement @@ -129,7 +183,6 @@ onMounted(() => { diff --git a/mateclaw-ui/src/views/Wiki/components/WikiSearchPreview.vue b/mateclaw-ui/src/views/Wiki/components/WikiSearchPreview.vue new file mode 100644 index 00000000..5b565590 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiSearchPreview.vue @@ -0,0 +1,127 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/index.vue b/mateclaw-ui/src/views/Wiki/index.vue index f553e1c0..7b5381f9 100644 --- a/mateclaw-ui/src/views/Wiki/index.vue +++ b/mateclaw-ui/src/views/Wiki/index.vue @@ -36,6 +36,15 @@ {{ t('wiki.pageCount', { count: kb.pageCount }) }} {{ t('wiki.sourceCount', { count: kb.rawCount }) }}
+ +
+ + ✦ {{ t('wiki.stats.enrichedRatio', { enriched: kbStats[kb.id].enrichedPageCount, total: kbStats[kb.id].pageCount }) }} + + + ⚠ {{ t('wiki.stats.failedJobs', { count: kbStats[kb.id].failedJobCount }) }} + +
{{ t(`wiki.status.${kb.status}`) }}
@@ -166,7 +175,7 @@