diff --git a/.env.example b/.env.example index dbe2a616..ce022f3a 100644 --- a/.env.example +++ b/.env.example @@ -50,6 +50,25 @@ MATECLAW_BROWSER_CDP_URL= MATECLAW_BROWSER_CHROME_PATH= MATECLAW_BROWSER_CHANNEL= +# ==================== OpenAI OAuth(Docker,可选) ==================== +# +# OpenAI ChatGPT OAuth 使用 Codex CLI 的 public client + PKCE / device code, +# 不需要自定义 client secret。 +# +# 默认留空即可。后端会根据访问 Host 自动选择: +# - localhost / 127.0.0.1 / ::1 → LOCAL(PKCE 回调) +# - IP / 域名 / 反向代理访问 → DEVICE_CODE(无缝远程授权) +# +# 本机 Docker 若希望像桌面版一样直接通过宿主机浏览器完成 +# http://localhost:1455/auth/callback 回调,可显式开启 LOCAL,并让容器内 +# 临时回调服务监听 0.0.0.0,以便通过 `1455:1455` 端口映射被宿主机访问到: +# MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE=local +# MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST=0.0.0.0 +# +# 强制模式调试时也可设为:local / device_code / manual_paste +MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE= +MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST= + # ── Maven 镜像(国内加速)───────────────────────────────────────── # 在中国大陆构建时取消注释,将 Aliyun 仓库优先级提前,大幅提速 mvn 拉包。 # 空值(默认)使用 US Maven Central → Google CDN → Aliyun 的顺序。 diff --git a/.gitignore b/.gitignore index 93cd7ff0..dd12de0d 100644 --- a/.gitignore +++ b/.gitignore @@ -101,3 +101,7 @@ CLAUDE.md # Sync tooling local state (generated each run; report is intentionally tracked) scripts/.*-sync-state.json + +# Sandbox / external client work that lives in this directory +# but should not ship in the repo. +outputs/ diff --git a/assets/images/preview.png b/assets/images/preview.png index 9ea30836..d2c3b9fd 100644 Binary files a/assets/images/preview.png and b/assets/images/preview.png differ diff --git a/docker-compose.yml b/docker-compose.yml index 05fea015..69408c32 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,10 @@ services: DB_NAME: ${DB_NAME:-mateclaw} DB_USERNAME: ${DB_USERNAME:-mateclaw} DB_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD is required in .env} - DASHSCOPE_API_KEY: ${DASHSCOPE_API_KEY:-} + # LLM provider keys (DashScope / OpenAI / Anthropic / DeepSeek / Kimi / …) are + # NOT configured via env vars. After startup, add providers in the admin UI: + # Settings → Models → Add Provider + # Keys are stored in mate_model_provider and hot-reloaded. SERPER_API_KEY: ${SERPER_API_KEY:-} JWT_SECRET: ${JWT_SECRET:-} MATECLAW_CORS_ALLOWED_ORIGINS: ${MATECLAW_CORS_ALLOWED_ORIGINS:-} @@ -89,12 +92,17 @@ services: MATECLAW_BROWSER_CDP_URL: ${MATECLAW_BROWSER_CDP_URL:-} MATECLAW_BROWSER_CHROME_PATH: ${MATECLAW_BROWSER_CHROME_PATH:-} MATECLAW_BROWSER_CHANNEL: ${MATECLAW_BROWSER_CHANNEL:-} + # OAuth 模式默认保持 auto:localhost 访问走 LOCAL,IP/域名访问走 DEVICE_CODE。 + # 本机 Docker 若要强制使用 localhost:1455 回调,可在 .env 显式设为 local。 + MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE: ${MATECLAW_OAUTH_OPENAI_DEPLOYMENT_MODE:-} + MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST: ${MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST:-0.0.0.0} # Chromium needs a real /dev/shm. Docker defaults to 64MB which causes # SIGBUS / "Target page closed" errors under load. 2GB is the usual # recommendation for Playwright / headless chrome. shm_size: 2gb ports: - "18080:18088" # host:container — app listens on 18088 inside the container + - "1455:1455" volumes: - server_data:/app/data diff --git a/mateclaw-server/Dockerfile b/mateclaw-server/Dockerfile index 4b8874fb..05ecfb94 100644 --- a/mateclaw-server/Dockerfile +++ b/mateclaw-server/Dockerfile @@ -5,7 +5,11 @@ # build container. These files are later copied into the JAR's classpath so # Spring Boot serves the SPA at the root URL. FROM node:22-alpine AS frontend-builder -RUN npm install -g pnpm --silent +# Pin pnpm to a major version so the Docker build doesn't break when the npm +# `latest` tag jumps majors. pnpm v10+ blocks dependency lifecycle scripts by +# default; the allowed packages live under `pnpm.onlyBuiltDependencies` in +# mateclaw-ui/package.json. +RUN npm install -g pnpm@10 --silent WORKDIR /frontend # Install dependencies first (layer cache) COPY mateclaw-ui/package.json mateclaw-ui/pnpm-lock.yaml ./ @@ -96,4 +100,5 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \ COPY --from=builder /build/target/*.jar app.jar EXPOSE 18088 +EXPOSE 1455 ENTRYPOINT ["java", "-jar", "-Dspring.profiles.active=mysql", "app.jar"] diff --git a/mateclaw-server/pom.xml b/mateclaw-server/pom.xml index a1c2a6b1..6b638658 100644 --- a/mateclaw-server/pom.xml +++ b/mateclaw-server/pom.xml @@ -6,7 +6,7 @@ vip.mate mateclaw-server - 1.2.0 + 1.3.0 jar MateClaw Server @@ -22,10 +22,10 @@ 21 UTF-8 - - 1.1.5 - - 1.1.2.2 + + 1.1.6 + + 1.1.2.3 3.5.16 5.8.26 2.8.16 @@ -351,6 +351,51 @@ 3.0.0 + + + org.xhtmlrenderer + flying-saucer-pdf + 9.13.0 + + + org.commonmark + commonmark + 0.28.0 + + + org.commonmark + commonmark-ext-gfm-tables + 0.28.0 + + + org.commonmark + commonmark-ext-yaml-front-matter + 0.28.0 + + + org.commonmark + commonmark-ext-gfm-strikethrough + 0.28.0 + + + org.commonmark + commonmark-ext-autolink + 0.28.0 + + org.flywaydb @@ -413,6 +458,17 @@ pdfbox 3.0.3 + + + + io.pebbletemplates + pebble + 3.2.2 + @@ -567,5 +623,27 @@ + + + + media-gen + + + + org.apache.maven.plugins + maven-surefire-plugin + + media-gen + + + + + diff --git a/mateclaw-server/src/main/java/vip/mate/MateClawApplication.java b/mateclaw-server/src/main/java/vip/mate/MateClawApplication.java index 0d699cbb..c0eaf1c5 100644 --- a/mateclaw-server/src/main/java/vip/mate/MateClawApplication.java +++ b/mateclaw-server/src/main/java/vip/mate/MateClawApplication.java @@ -15,13 +15,19 @@ import org.springframework.scheduling.annotation.EnableScheduling; * @author MateClaw Team */ @SpringBootApplication(exclude = { - // 禁用 Spring AI MCP Client 自动配置(由 McpClientManager 自行管理生命周期) + // Disable Spring AI MCP Client auto-configuration (lifecycle owned by McpClientManager). org.springframework.ai.mcp.client.common.autoconfigure.McpClientAutoConfiguration.class, org.springframework.ai.mcp.client.common.autoconfigure.McpToolCallbackAutoConfiguration.class, org.springframework.ai.mcp.client.common.autoconfigure.StdioTransportAutoConfiguration.class, org.springframework.ai.mcp.client.common.autoconfigure.annotations.McpClientAnnotationScannerAutoConfiguration.class, org.springframework.ai.mcp.client.httpclient.autoconfigure.SseHttpClientTransportAutoConfiguration.class, org.springframework.ai.mcp.client.httpclient.autoconfigure.StreamableHttpHttpClientTransportAutoConfiguration.class, + // DashScopeAgent is the Bailian "Application Agent" (Bailian-hosted prompt+tool app), + // not the chat model. We don't use it — model configuration is admin-UI driven and + // built by AgentDashScopeChatModelBuilder. Its auto-config strictly requires + // spring.ai.dashscope.api-key to be non-empty at startup, which makes the whole + // ApplicationContext fail when users deploy via Docker without setting the key. + com.alibaba.cloud.ai.autoconfigure.dashscope.DashScopeAgentAutoConfiguration.class, }) @EnableScheduling @MapperScan("vip.mate.**.repository") diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java index 15be0dc1..053544a2 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentGraphBuilder.java @@ -131,8 +131,11 @@ public class AgentGraphBuilder { private final vip.mate.llm.failover.ProviderHealthTracker providerHealthTracker; private final vip.mate.llm.chatmodel.ProviderChatModelFactory chatModelFactory; private final vip.mate.llm.failover.AvailableProviderPool providerPool; + private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; /** PR-0b: DashScope-specific construction lives here now; we only call into it for the search-on log. */ private final vip.mate.agent.chatmodel.AgentDashScopeChatModelBuilder dashScopeBuilder; + private final vip.mate.llm.routing.MultimodalRouter multimodalRouter; + private final vip.mate.llm.routing.MediaCaptionService mediaCaptionService; /** * Optional audit pipeline. Setter injection (rather than a constructor @@ -293,6 +296,11 @@ public class AgentGraphBuilder { agent.modelCapabilities = modelCapabilityService.resolve( runtimeModel.getModelName(), runtimeModel.getModalities()); agent.runtimeProviderId = provider != null ? provider.getProviderId() : ""; + agent.runtimeModelConfig = runtimeModel; + agent.toolSet = toolSet; + agent.multimodalRouter = multimodalRouter; + agent.mediaCaptionService = mediaCaptionService; + agent.userLocale = resolveLocale(); agent.temperature = runtimeModel.getTemperature(); agent.maxTokens = runtimeModel.getMaxTokens(); agent.maxInputTokens = runtimeModel.getMaxInputTokens(); @@ -450,6 +458,8 @@ public class AgentGraphBuilder { // 丢这个键,evidence_insufficient 检查会"静默地不生效" —— // StateKeyRegistrationCoverageTest 专门兜这条。 .addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE) + // Multimodal sidecar routing decision for the current turn. + .addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE) .build(); // Graph 拓扑: @@ -556,7 +566,7 @@ public class AgentGraphBuilder { ObservationNode observationNode = new ObservationNode(observationProcessor, streamTracker); SummarizingNode summarizingNode = new SummarizingNode(chatModel, streamingHelper, streamTracker); LimitExceededNode limitExceededNode = new LimitExceededNode(chatModel, observationProcessor, streamingHelper, i18nService); - FinalAnswerNode finalAnswerNode = new FinalAnswerNode(); + FinalAnswerNode finalAnswerNode = new FinalAnswerNode(generatedFileCache); KeyStrategyFactory keyStrategyFactory = KeyStrategy.builder() // 输入字段 @@ -636,6 +646,8 @@ public class AgentGraphBuilder { // 丢这个键,evidence_insufficient 检查会"静默地不生效" —— // StateKeyRegistrationCoverageTest 专门兜这条。 .addStrategy(MateClawStateKeys.SOURCE_EVIDENCE_LEDGER, KeyStrategy.REPLACE) + // Multimodal sidecar routing decision for the current turn. + .addStrategy(MateClawStateKeys.ROUTING_DECISION, KeyStrategy.REPLACE) .build(); StateGraph graph = new StateGraph("react-agent-v2", keyStrategyFactory) @@ -701,6 +713,21 @@ public class AgentGraphBuilder { return buildRuntimeChatModel(runtimeModel, this.retryTemplate); } + /** + * Resolve the user-facing locale used for sidecar caption prompts. + * Reads {@code language} from system settings; falls back to + * {@code zh-CN} so CN deployments stay consistent with the chat UI. + */ + private java.util.Locale resolveLocale() { + try { + String lang = systemSettingService.getLanguage(); + if (lang == null || lang.isBlank()) return java.util.Locale.SIMPLIFIED_CHINESE; + return java.util.Locale.forLanguageTag(lang); + } catch (Exception e) { + return java.util.Locale.SIMPLIFIED_CHINESE; + } + } + /** * 构建运行时 ChatModel,并指定自定义的 Spring AI {@link RetryTemplate}。 *

@@ -1584,6 +1611,7 @@ public class AgentGraphBuilder { private RestClient.Builder applyHttpTimeouts(RestClient.Builder builder, Integer readTimeoutOverride) { HttpClient httpClient = HttpClient.newBuilder() .connectTimeout(vip.mate.llm.chatmodel.HttpTimeouts.CONNECT_TIMEOUT) + .version(HttpClient.Version.HTTP_1_1) .build(); JdkClientHttpRequestFactory rf = new JdkClientHttpRequestFactory(httpClient); rf.setReadTimeout(vip.mate.llm.chatmodel.HttpTimeouts.resolveReadTimeout(readTimeoutOverride)); @@ -1614,8 +1642,13 @@ public class AgentGraphBuilder { * {@link #applyHttpTimeouts(RestClient.Builder, Integer)}. */ private WebClient.Builder applyHttpTimeoutsToWebClient(WebClient.Builder builder, Integer readTimeoutOverride) { + // Pin HTTP/1.1: many self-hosted OpenAI-compatible servers (vLLM, lmstudio, + // llama.cpp, ollama — all uvicorn/ASGI based) only speak HTTP/1.1 over + // cleartext and slam the socket on the JDK client's default H2C upgrade + // probe, surfacing as "header parser received no bytes" with no body sent. HttpClient httpClient = HttpClient.newBuilder() .connectTimeout(vip.mate.llm.chatmodel.HttpTimeouts.CONNECT_TIMEOUT) + .version(HttpClient.Version.HTTP_1_1) .build(); org.springframework.http.client.reactive.JdkClientHttpConnector connector = new org.springframework.http.client.reactive.JdkClientHttpConnector(httpClient); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/AgentService.java b/mateclaw-server/src/main/java/vip/mate/agent/AgentService.java index 235c25e0..c101eb6c 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/AgentService.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/AgentService.java @@ -3,12 +3,15 @@ package vip.mate.agent; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import reactor.core.publisher.Flux; import vip.mate.agent.context.ChatOrigin; import vip.mate.agent.context.ChatOriginHolder; +import vip.mate.agent.event.AgentLifecycleEvent; import vip.mate.agent.model.AgentEntity; import vip.mate.agent.repository.AgentMapper; import vip.mate.exception.MateClawException; @@ -43,6 +46,11 @@ public class AgentService { private final MemoryLifecycleMediator lifecycleMediator; private final MemoryProperties memoryProperties; + /** Field-injected publisher for agent_lifecycle trigger events; the + * trigger module's bridge listens and forwards into ingest. */ + @Autowired(required = false) + private ApplicationEventPublisher events; + /** 运行时 Agent 实例缓存(agentId -> BaseAgent) */ private final Map agentInstances = new ConcurrentHashMap<>(); @@ -57,9 +65,26 @@ public class AgentService { * 按工作区列出 Agent */ public List listAgentsByWorkspace(Long workspaceId) { - return agentMapper.selectList(new LambdaQueryWrapper() - .eq(AgentEntity::getWorkspaceId, workspaceId) - .orderByDesc(AgentEntity::getCreateTime)); + return listAgentsByWorkspace(workspaceId, null); + } + + /** + * 按工作区列出 Agent,可选过滤启用状态。 + * + * @param enabled non-null restricts the result set to agents whose + * {@code enabled} column matches the given value. + * Pass {@code true} from chat selectors so disabled + * agents disappear from the picker; the admin + * management page passes {@code null} to keep + * disabled rows visible for re-enabling. + */ + public List listAgentsByWorkspace(Long workspaceId, Boolean enabled) { + LambdaQueryWrapper q = new LambdaQueryWrapper() + .eq(AgentEntity::getWorkspaceId, workspaceId); + if (enabled != null) { + q.eq(AgentEntity::getEnabled, enabled); + } + return agentMapper.selectList(q.orderByDesc(AgentEntity::getCreateTime)); } public AgentEntity getAgent(Long id) { @@ -75,19 +100,100 @@ public class AgentService { if (agent.getAgentType() == null) { agent.setAgentType("react"); } + requireUniqueName(agent, null); agentMapper.insert(agent); + publishLifecycle(agent, "spawned"); return agent; } public AgentEntity updateAgent(AgentEntity agent) { + // Detect enabled-flag flip so the lifecycle event reflects the + // intent rather than every metadata edit. Reading the prior row + // is cheap and gives us a clean diff source. + AgentEntity prior = agentMapper.selectById(agent.getId()); + // Only re-validate uniqueness when the name actually changes — + // a pure metadata edit (icon, prompt, ...) shouldn't pay the + // SELECT cost or risk a false positive against the row itself. + if (prior != null + && agent.getName() != null + && !agent.getName().equals(prior.getName())) { + // Workspace cannot be moved (Controller pins it to prior.workspaceId), + // so reuse it for the lookup even if the incoming DTO left it null. + if (agent.getWorkspaceId() == null) { + agent.setWorkspaceId(prior.getWorkspaceId()); + } + requireUniqueName(agent, agent.getId()); + } agentMapper.updateById(agent); agentInstances.remove(agent.getId()); + if (prior != null && prior.getEnabled() != null + && !prior.getEnabled().equals(agent.getEnabled())) { + publishLifecycle(agent, + Boolean.TRUE.equals(agent.getEnabled()) ? "enabled" : "disabled"); + } return agent; } + /** + * Friendly business-code surface for the {@code (workspace_id, name)} + * unique index added in V102. + * + *

The wire shape is the project-wide R<T> envelope: HTTP status + * stays 200 (per the convention in {@code R.fail} and the axios + * interceptor in {@code mateclaw-ui/src/api/index.ts}); the 409 lives in + * the response body's {@code code} field so the front-end can branch + * without breaking on an axios error. Without this pre-check the + * duplicate save would surface as an opaque + * {@code DataIntegrityViolation} stack trace. + * + * @param excludeId when non-null, skip this row in the lookup so + * {@link #updateAgent} doesn't mistake the row for its + * own duplicate. + */ + private void requireUniqueName(AgentEntity agent, Long excludeId) { + if (agent.getName() == null || agent.getName().isBlank()) { + throw new MateClawException("err.agent.name_required", 400, "Agent 名称不能为空"); + } + Long workspaceId = agent.getWorkspaceId() == null ? 1L : agent.getWorkspaceId(); + LambdaQueryWrapper q = new LambdaQueryWrapper() + .eq(AgentEntity::getWorkspaceId, workspaceId) + .eq(AgentEntity::getName, agent.getName()); + if (excludeId != null) { + q.ne(AgentEntity::getId, excludeId); + } + Long count = agentMapper.selectCount(q); + if (count != null && count > 0) { + throw new MateClawException("err.agent.duplicate_name", 409, + "工作区内已存在同名 Agent: " + agent.getName()); + } + } + public void deleteAgent(Long id) { + AgentEntity prior = agentMapper.selectById(id); agentMapper.deleteById(id); agentInstances.remove(id); + if (prior != null) publishLifecycle(prior, "terminated"); + } + + /** + * Best-effort publish of an {@link AgentLifecycleEvent}. A publish + * failure must never roll back the agent CRUD that just succeeded — + * the agent_lifecycle trigger surface is observability, not the + * canonical record. + */ + private void publishLifecycle(AgentEntity agent, String phase) { + if (events == null || agent == null) return; + try { + events.publishEvent(new AgentLifecycleEvent( + agent.getWorkspaceId() == null ? 0L : agent.getWorkspaceId(), + agent.getId() == null ? 0L : agent.getId(), + agent.getName(), + phase, + System.currentTimeMillis())); + } catch (Exception e) { + log.warn("[AgentService] lifecycle publish failed for agent {} ({}): {}", + agent.getId(), phase, e.getMessage()); + } } /** diff --git a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java index bcbe8d38..4fae2e5a 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/BaseAgent.java @@ -10,7 +10,12 @@ import org.springframework.core.io.FileSystemResource; import org.springframework.util.MimeType; import reactor.core.publisher.Flux; import vip.mate.approval.ApprovalPlaceholderUtil; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.MediaCaptionService; +import vip.mate.llm.routing.MultimodalRouter; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; import vip.mate.llm.service.ModelCapabilityService; +import org.springframework.ai.chat.messages.ToolResponseMessage; import vip.mate.workspace.conversation.ConversationService; import vip.mate.workspace.conversation.model.MessageContentPart; import vip.mate.workspace.conversation.model.MessageEntity; @@ -20,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; @@ -85,6 +91,31 @@ public abstract class BaseAgent { /** 构建时使用的 provider ID(运行时快照) */ protected String runtimeProviderId; + /** + * Full runtime model configuration used by the multimodal router to + * decide whether the primary model can handle attachments natively. + * Set by {@code AgentGraphBuilder} alongside {@link #modelCapabilities}. + */ + protected ModelConfigEntity runtimeModelConfig; + + /** + * The agent's effective tool set. Lifted from subclasses so + * {@link #buildUserMessage} can ask whether the agent has any media-capable + * tool when the primary model rejects an attachment. + */ + protected vip.mate.agent.AgentToolSet toolSet; + + /** + * Optional sidecar routing services. Null when not wired (e.g. tests with + * minimal builders); the routing path then degrades to the legacy + * skip-with-text-hint behavior without any extra LLM calls. + */ + protected MultimodalRouter multimodalRouter; + protected MediaCaptionService mediaCaptionService; + + /** Locale used when prompting the vision sidecar. Defaults to zh-CN when unset. */ + protected java.util.Locale userLocale = java.util.Locale.SIMPLIFIED_CHINESE; + protected BaseAgent(ChatClient chatClient, ConversationService conversationService) { this.chatClient = chatClient; @@ -206,17 +237,46 @@ public abstract class BaseAgent { agentName, history.size(), totalCount, windowSize); } - // ===== 识别持久化的压缩摘要:从摘要位置开始,跳过更早消息 ===== - for (int i = 0; i < history.size(); i++) { + // ===== Slice from the LATEST compression boundary, not the first ===== + // A long-running conversation can accumulate several boundaries; the + // newest one is the only relevant cut-off because every earlier + // boundary's content is already folded into the newer summary. Walking + // forward and breaking on the first boundary kept everything between + // boundaries — the very redundancy compaction was supposed to remove. + boolean boundaryFoundInWindow = false; + for (int i = history.size() - 1; i >= 0; i--) { MessageEntity msg = history.get(i); if ("system".equals(msg.getRole()) && isCompressionSummary(msg)) { history = new ArrayList<>(history.subList(i, history.size())); - log.info("[{}] Found compression summary, loading from index {} ({} messages)", + boundaryFoundInWindow = true; + log.info("[{}] Found latest compression boundary at index {}; loading {} messages forward", agentName, i, history.size()); break; } } + // ===== Latest boundary may live OUTSIDE the recent window ===== + // On a long conversation that compacted hours/days ago and has paged + // fewer than `windowSize` new messages since, `listRecentMessages` + // returns only the raw tail — the boundary sat at index 0 of the + // original list and never made it into `history`. Without prepending + // it, the model would forget the original goal even though we already + // paid the LLM cost to produce a structured summary. + if (!boundaryFoundInWindow && totalCount > windowSize) { + try { + MessageEntity latestBoundary = conversationService.findLatestCompressionBoundary(conversationId); + if (latestBoundary != null) { + history = new ArrayList<>(history); + history.add(0, latestBoundary); + log.info("[{}] Prepended out-of-window compression boundary id={} so the model keeps the summary context", + agentName, latestBoundary.getId()); + } + } catch (Exception e) { + log.warn("[{}] findLatestCompressionBoundary failed; loading recent window without boundary: {}", + agentName, e.getMessage()); + } + } + // ===== 转换为 Spring AI Message 对象 ===== int limit = history.size(); if (limit > 0) { @@ -270,9 +330,125 @@ public abstract class BaseAgent { while (!messages.isEmpty() && messages.get(messages.size() - 1) instanceof UserMessage) { messages.remove(messages.size() - 1); } + + // Head guard — orphan tool-response strip. + // + // Independent of the compaction pair-safe boundary in + // ConversationWindowManager: that one protects the *compaction* cut, + // this one protects the *pagination* cut. listRecentMessages returns + // the last N rows verbatim, and the first row of that page can be a + // ToolResponseMessage whose owning AssistantMessage sat one row + // earlier — i.e. outside the page. Sending such a sequence to any + // OpenAI-compatible provider returns 400 because every tool response + // must be preceded by an assistant message issuing that tool_call_id. + // + // The boundary prepend earlier inserts a SystemMessage at the head; + // the orphan, if present, sits at index 1 in that case. Skip leading + // SystemMessages and drop any leading ToolResponseMessage whose + // response ids are not all issued by a *preceding* AssistantMessage + // — i.e. one we have already walked past in this scan. Provider + // validity is order-sensitive; a later same-id assistant deeper in + // the window does NOT redeem an earlier orphan. See + // stripHeadOrphanToolResponses below for the forward-scan details. + // + // Dropping is correct rather than expanding backward to fetch the + // missing assistant: if the AssistantMessage is outside the window, + // its content is already lost to the model anyway, and the boundary + // summary (if any) covers it. Keeping the orphan would just trade a + // dropped row for a 400. + stripHeadOrphanToolResponses(messages, agentName); return messages; } + /** + * Drop leading {@link ToolResponseMessage}s whose owning + * {@link AssistantMessage} sits before them in this list. Provider + * validity is order-sensitive: a tool response must follow the assistant + * that issued the tool_call_id; an unrelated later AssistantMessage that + * happens to carry the same id does not redeem an earlier orphan. + * + *

Algorithm: forward scan with a {@code seenIssuedIds} set. Leading + * {@link SystemMessage}s (boundary rows, system prompts) pass through + * untouched but contribute no ids. The first {@link AssistantMessage} or + * {@link UserMessage} we hit stops the repair walk — by that point we're + * out of head-orphan territory. Every {@link ToolResponseMessage} we + * encounter before that stop is checked against {@code seenIssuedIds}; + * if every response id is unseen, the message is dropped and the scan + * re-examines the new head. A response whose ids are all in the seen + * set (e.g. {@code [system, assistant(X), toolResponse(X), ...]} when + * the assistant fell at index 1 of the slice) is left in place. + * + *

Mixed responses (some ids matched, some not) inside a single + * leading {@code ToolResponseMessage} are dropped wholesale rather than + * surgically rewritten — the provider would reject partially-broken + * sequences anyway, and the mixed case implies an upstream invariant + * violation that surfaces in logs. + * + *

Package-private + static so unit tests can drive it without standing + * up a full BaseAgent subclass. + */ + static int stripHeadOrphanToolResponses(List messages, String agentName) { + if (messages.isEmpty()) return 0; + + // Built up as we walk; only assistants we've already passed count + // toward "preceding". An assistant that sits behind a head orphan is + // irrelevant: provider order-validity asks "was this tool_call id + // issued BEFORE this response?", not "anywhere in the prompt". + Set seenIssuedIds = new HashSet<>(); + + int dropped = 0; + int i = 0; + while (i < messages.size()) { + Message m = messages.get(i); + if (m instanceof SystemMessage) { + // Boundary rows / system prompts pass through; advance and + // keep looking for orphan tool responses that sit behind them. + i++; + continue; + } + if (m instanceof AssistantMessage am) { + // Reached a preceding assistant — head danger is over. The + // tool_call ids it issued are valid for any tool responses + // that follow, but we stop the repair walk here either way. + if (am.getToolCalls() != null) { + for (AssistantMessage.ToolCall tc : am.getToolCalls()) { + if (tc.id() != null && !tc.id().isEmpty()) { + seenIssuedIds.add(tc.id()); + } + } + } + break; + } + if (m instanceof ToolResponseMessage trm) { + // Every response id must have been issued by a preceding + // assistant we already walked through. If any single id is + // missing from seenIssuedIds, the message is invalid in + // place. Empty / null ids don't count for or against. + boolean anyUnmatched = trm.getResponses().stream() + .map(ToolResponseMessage.ToolResponse::id) + .filter(id -> id != null && !id.isEmpty()) + .anyMatch(id -> !seenIssuedIds.contains(id)); + if (anyUnmatched) { + messages.remove(i); + dropped++; + continue; // re-examine the new messages[i] + } + // All ids match a preceding assistant — keep, and stop the + // repair walk. Anything past here is well-formed by + // construction (provider validates each subsequent pair as + // we go). + break; + } + // UserMessage (or anything else) — past the head danger. Stop. + break; + } + if (dropped > 0) { + log.info("[{}] Stripped {} leading orphan ToolResponseMessage(s) — no preceding AssistantMessage in scope", + agentName, dropped); + } + return dropped; + } + /** * History sanitization entry point. Encapsulates *all* steps applied to a * persisted message before it reaches an LLM prompt. Returns {@code null} @@ -510,34 +686,87 @@ public abstract class BaseAgent { } /** - * 构建 UserMessage,支持 multimodal:如果消息包含图片/视频附件,直接注入 Spring AI Media 对象, - * 让模型在 prompt 中直接看到媒体内容,不需要再调 MCP read_media_file 工具。 + * Build a {@link UserMessage} for the current turn, including any image/video + * media the agent's primary model can handle natively. Returns the message + * paired with the {@link MultimodalRoutingDecision} taken so the caller can + * persist it as message metadata and emit a routing event. */ - protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { - return buildUserMessage(message, renderedContent, true); + protected CurrentTurnUserMessage buildUserMessageForCurrentTurn(MessageEntity message, String renderedContent) { + return buildUserMessageInternal(message, renderedContent, true); } /** - * @param injectMedia when {@code false} (history replay), skip the Media-loading - * branch entirely and return text-only — providers like Zhipu - * GLM-5V cap at 1 video per request, so re-injecting historical - * attachments on every turn breaks the call. + * History-replay variant: text-only, no media reinjected, no routing decision. + * Many providers cap at one video per request, so re-injecting old attachments + * on every replay would break the call. */ + protected UserMessage buildUserMessage(MessageEntity message, String renderedContent) { + return buildUserMessageInternal(message, renderedContent, true).userMessage(); + } + protected UserMessage buildUserMessage(MessageEntity message, String renderedContent, boolean injectMedia) { + return buildUserMessageInternal(message, renderedContent, injectMedia).userMessage(); + } + + private CurrentTurnUserMessage buildUserMessageInternal(MessageEntity message, String renderedContent, boolean injectMedia) { if (!injectMedia) { - return new UserMessage(renderedContent == null ? "" : renderedContent); + return new CurrentTurnUserMessage( + new UserMessage(renderedContent == null ? "" : renderedContent), + null); } List parts = conversationService.parseMessageParts(message); + + // Sidecar routing — runs first so caption text gets folded into finalText + // before native media injection considers the same parts again. + MultimodalRoutingDecision decision = multimodalRouter != null + ? multimodalRouter.route(parts, runtimeModelConfig) + : MultimodalRoutingDecision.none(); + + StringBuilder textBuilder = new StringBuilder(renderedContent == null ? "" : renderedContent); + java.util.Set sidecarHandledIdentifiers = new java.util.HashSet<>(); + if (decision.strategy() == MultimodalRoutingDecision.Strategy.SIDECAR + && mediaCaptionService != null + && decision.sidecarModel() != null) { + for (MessageContentPart part : parts) { + if (part == null) continue; + String contentType = part.getContentType(); + boolean isImage = ("image".equals(part.getType()) || "file".equals(part.getType())) + && contentType != null && contentType.startsWith("image/") + && !contentType.contains("svg"); + if (!isImage) continue; + MediaCaptionService.CaptionResult result = mediaCaptionService.caption( + decision.sidecarModel(), part, userLocale); + if (result.isFailure()) { + log.warn("[{}] Sidecar caption failed for {}: {}", + agentName, part.getFileName(), result.failure().getMessage()); + textBuilder.append("\n\n[系统提示] 视觉模型未能解析附件 ") + .append(part.getFileName()) + .append(",请稍后重试或在「设置 → 模型」检查视觉模型配置。"); + continue; + } + textBuilder.append("\n\n[图片附件描述: ") + .append(part.getFileName() == null ? "image" : part.getFileName()) + .append("]\n") + .append(result.description()) + .append("\n[/图片附件描述]"); + String identifier = identifyPart(part); + if (identifier != null) sidecarHandledIdentifiers.add(identifier); + } + } + List mediaList = new ArrayList<>(); - // Reasons for attachments that the model cannot consume — surfaced to the agent - // via the user message text so it does not hallucinate a tool call to read them. - // See issue #44. List skippedAttachments = new ArrayList<>(); boolean videoSupported = modelSupportsVideo(); boolean visionSupported = modelSupportsVision(); for (MessageContentPart part : parts) { if (part == null) continue; + // Sidecar already produced text for this image; never inject the + // raw bytes — the primary model would receive them and try to + // process natively, defeating the cost-saving purpose. + String identifier = identifyPart(part); + if (identifier != null && sidecarHandledIdentifiers.contains(identifier)) continue; + String partType = part.getType(); String contentType = part.getContentType(); // image 类型的 part 可能没有精确 contentType,补全为 image/jpeg @@ -607,21 +836,88 @@ public abstract class BaseAgent { } } - String finalText = renderedContent; if (!skippedAttachments.isEmpty()) { - finalText = (renderedContent == null ? "" : renderedContent) - + "\n\n[系统提示] 以下附件未能传入当前模型:" + String.join("、", skippedAttachments) - + "。\n请用对话语言清晰、友好地告诉用户:当前模型无法处理这类附件,建议切换到具备相应能力的多模态模型(图片需视觉模型,视频需视频理解模型)后重新上传。" - + "不要调用任何工具(包括 ffmpeg、浏览器、文件读取等)尝试解析这些附件。"; + textBuilder.append("\n\n[系统提示] 以下附件未能传入当前模型:") + .append(String.join("、", skippedAttachments)) + .append("。"); + // Only suggest switching models when no media-capable tool is bound + // either. With a media tool the LLM may legitimately choose to + // delegate to the tool — never instruct it not to use tools. + if (!hasMediaCapableTools()) { + textBuilder.append("\n请用对话语言清晰、友好地告诉用户:当前模型无法处理这类附件,建议切换到具备相应能力的多模态模型,或在「设置 → 模型」中配置视觉/视频模型作为旁路。"); + } } - if (mediaList.isEmpty()) { - return new UserMessage(finalText); + String finalText = textBuilder.toString(); + UserMessage built = mediaList.isEmpty() + ? new UserMessage(finalText) + : UserMessage.builder().text(finalText).media(mediaList).build(); + return new CurrentTurnUserMessage(built, decision); + } + + /** + * Stable identifier for de-duplicating parts already handled by the sidecar + * pass. Falls back across {@code path → mediaId → fileName} since not every + * channel populates the same field. + */ + private static String identifyPart(MessageContentPart part) { + if (part == null) return null; + if (part.getPath() != null && !part.getPath().isBlank()) return "p:" + part.getPath(); + if (part.getMediaId() != null && !part.getMediaId().isBlank()) return "m:" + part.getMediaId(); + if (part.getFileName() != null && !part.getFileName().isBlank()) return "f:" + part.getFileName(); + return null; + } + + /** + * True if the agent has at least one tool whose name or description + * suggests it can read images / video / audio. The check is intentionally + * loose — false positives just mean the agent is allowed to attempt media + * processing on its own, which is the safer default. + */ + private static final Set MEDIA_TOOL_KEYWORDS = Set.of( + "image", "图片", "vision", "视觉", + "video", "视频", "ffmpeg", + "ocr", "caption", "media", "audio", "音频"); + + private boolean hasMediaCapableTools() { + if (toolSet == null) return false; + var callbacks = toolSet.callbacks(); + if (callbacks == null || callbacks.isEmpty()) return false; + return callbacks.stream().anyMatch(cb -> { + try { + String name = String.valueOf(cb.getToolDefinition().name()).toLowerCase(); + String desc = String.valueOf(cb.getToolDefinition().description()).toLowerCase(); + return MEDIA_TOOL_KEYWORDS.stream().anyMatch(k -> name.contains(k) || desc.contains(k)); + } catch (Exception e) { + return false; + } + }); + } + + /** + * Pair returned from the current-turn user message build path: the assembled + * {@link UserMessage} and the routing decision the caller should persist as + * {@code metadata.routing} and surface to the SSE consumer. + */ + public record CurrentTurnUserMessage(UserMessage userMessage, MultimodalRoutingDecision routingDecision) {} + + /** + * Extract a routing-decision payload from the graph input map (placed there + * by {@code buildInitialState}) and turn it into a startup + * {@link vip.mate.agent.AgentService.StreamDelta} the SSE accumulator can + * persist. Returns an empty Flux when no routing happened this turn so we + * don't emit zero-value events. + */ + @SuppressWarnings("unchecked") + public static reactor.core.publisher.Flux routingStartupDelta( + java.util.Map inputs) { + Object decision = inputs.get(vip.mate.agent.graph.state.MateClawStateKeys.ROUTING_DECISION); + if (decision instanceof java.util.Map map && !map.isEmpty()) { + return reactor.core.publisher.Flux.just(vip.mate.agent.AgentService.StreamDelta.event( + vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION, + (java.util.Map) map)); } - return UserMessage.builder() - .text(finalText) - .media(mediaList) - .build(); + return reactor.core.publisher.Flux.empty(); } /** @@ -642,6 +938,17 @@ public abstract class BaseAgent { * @return 带图片 Media 的 UserMessage(如果有图片附件),否则纯文本 UserMessage */ protected UserMessage buildCurrentUserMessage(String conversationId, String userMessageText) { + return buildCurrentUserMessageWithRouting(conversationId, userMessageText).userMessage(); + } + + /** + * Same as {@link #buildCurrentUserMessage} but also returns the multimodal + * routing decision taken for this turn so the caller can persist it as + * {@code metadata.routing} and emit a SSE-side event for the chat UI. + * Returns a decision with NONE strategy when the message has no attachments + * the primary model can't already handle. + */ + protected CurrentTurnUserMessage buildCurrentUserMessageWithRouting(String conversationId, String userMessageText) { try { List history = conversationService.listMessages(conversationId); // 倒序取最后一条 user 消息(buildInitialState 在 saveMessage 后调用,所以最后一条就是当前消息) @@ -650,14 +957,14 @@ public abstract class BaseAgent { if ("user".equals(msg.getRole())) { // 用 DB 中的实际内容(可能包含 contentParts),不用传入的 text String content = conversationService.renderMessageContent(msg); - return buildUserMessage(msg, content != null && !content.isBlank() ? content : userMessageText); + return buildUserMessageForCurrentTurn(msg, content != null && !content.isBlank() ? content : userMessageText); } } } catch (Exception e) { log.debug("[{}] Failed to load current user message parts for multimodal: {}", agentName, e.getMessage()); } - return new UserMessage(userMessageText); + return new CurrentTurnUserMessage(new UserMessage(userMessageText), null); } protected Path resolveImagePath(String relativePath) { diff --git a/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java b/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java index 278d747b..1cc8b5f1 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/GraphEventPublisher.java @@ -45,6 +45,31 @@ public final class GraphEventPublisher { */ public static final String EVENT_FINISH_REASON = "finish_reason"; + /** + * User-facing recovery affordances offered after a turn ends in a + * non-transient error. Carries the error type + message + a + * data-driven list of actions ({@code retry}, {@code regenerate}, + * {@code report}) so the frontend can render the right buttons + * without hard-coding which categories deserve which actions. + * + *

Sibling to {@link #EVENT_FINISH_REASON} (which only carries the + * machine-readable reason). The two are kept separate so legacy + * consumers of {@code finish_reason} don't have to learn a new + * payload shape — and so a future graph branch (e.g. evidence- + * insufficient → "rerun with the listed files attached") can emit + * feedback affordances without abusing the finish_reason channel. + */ + public static final String EVENT_FEEDBACK = "feedback_event"; + + /** + * Multimodal sidecar routing decision for the current turn. Emitted once + * per turn before the graph starts streaming; the channel-side accumulator + * stores it under {@code metadata.routing} so the chat UI can show which + * sidecar (if any) was invoked. Underscore-prefixed name keeps it out of + * IM channel rebroadcast (see {@code ChannelMessageRouter}). + */ + public static final String EVENT_ROUTING_DECISION = "_routing_decision"; + /** * 事件记录 */ @@ -217,6 +242,30 @@ public final class GraphEventPublisher { ), ts); } + /** + * Emit a recovery-affordance event for the frontend. {@code errorType} + * mirrors the {@code NodeStreamingChatHelper.ErrorType} value (e.g. + * {@code AUTH_ERROR}, {@code BILLING}, {@code MODEL_NOT_FOUND}, or + * the generic {@code UNKNOWN}); {@code errorMessage} is the + * user-friendly text already displayed in the bubble; {@code actions} + * is the ordered list of buttons to render. Default offering is the + * standard {@code retry / regenerate / report} triad — call sites + * can narrow this if a category has limitations (e.g. AUTH_ERROR + * shouldn't offer "retry" until the key is fixed). + */ + public static GraphEvent feedback(String errorType, String errorMessage, + java.util.List actions) { + long ts = System.currentTimeMillis(); + return new GraphEvent(EVENT_FEEDBACK, Map.of( + "errorType", errorType != null ? errorType : "", + "errorMessage", errorMessage != null ? errorMessage : "", + "actions", actions != null && !actions.isEmpty() + ? actions + : java.util.List.of("retry", "regenerate", "report"), + "timestamp", ts + ), ts); + } + // ===== 提取方法 ===== /** diff --git a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java index 9ecf1166..ea5391b4 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/binding/service/AgentBindingService.java @@ -12,8 +12,17 @@ import vip.mate.agent.binding.model.AgentToolBinding; import vip.mate.agent.binding.repository.AgentProviderPreferenceMapper; import vip.mate.agent.binding.repository.AgentSkillBindingMapper; import vip.mate.agent.binding.repository.AgentToolBindingMapper; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.exception.MateClawException; +import vip.mate.skill.acp.AcpSkillBridge; +import vip.mate.skill.mcp.McpSkillBridge; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.repository.SkillMapper; import vip.mate.skill.runtime.SkillRuntimeService; import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.service.AvailableToolService; import java.util.Collections; import java.util.LinkedHashSet; @@ -43,16 +52,51 @@ public class AgentBindingService { * graph when SkillRuntimeService initializes after binding. */ private final SkillRuntimeService skillRuntimeService; + /** + * Source of truth for what the picker can offer (built-in + MCP). Used + * by {@link #setToolBindings} to refuse new tool names that the runtime + * couldn't resolve anyway — closes the gap where a UI-disabled row + * could still be saved by hitting the API directly. + */ + private final AvailableToolService availableToolService; + /** + * Direct mapper access (instead of {@code AgentService}) to look up an + * agent's workspace before binding a skill. {@code AgentService} pulls + * in {@code AgentGraphBuilder}, which itself depends on + * {@code AgentBindingService} — going through the service would create a + * boot-time cycle. The mapper has no such transitive dependency. + */ + private final AgentMapper agentMapper; + /** Same reasoning as {@link #agentMapper}: skill workspace lookup. */ + private final SkillMapper skillMapper; + /** + * ACP virtual skills aren't rows in {@code mate_skill}; the bridge + * synthesizes them from {@code mate_acp_endpoint}. We need this to + * answer "what workspace does this virtual id belong to?" when an + * agent tries to bind one. MCP virtual skills don't need a bridge + * reference — {@link McpSkillBridge#isVirtualMcpSkillId(Long)} is a + * static range check, and MCP servers carry no workspace today, so + * binding any MCP virtual id is allowed for any agent. + */ + private final AcpSkillBridge acpSkillBridge; @Autowired public AgentBindingService(AgentSkillBindingMapper skillBindingMapper, AgentToolBindingMapper toolBindingMapper, AgentProviderPreferenceMapper providerPreferenceMapper, - @Lazy SkillRuntimeService skillRuntimeService) { + @Lazy SkillRuntimeService skillRuntimeService, + AvailableToolService availableToolService, + AgentMapper agentMapper, + SkillMapper skillMapper, + AcpSkillBridge acpSkillBridge) { this.skillBindingMapper = skillBindingMapper; this.toolBindingMapper = toolBindingMapper; this.providerPreferenceMapper = providerPreferenceMapper; this.skillRuntimeService = skillRuntimeService; + this.availableToolService = availableToolService; + this.agentMapper = agentMapper; + this.skillMapper = skillMapper; + this.acpSkillBridge = acpSkillBridge; } // ==================== Skill Bindings ==================== @@ -80,6 +124,7 @@ public class AgentBindingService { } public AgentSkillBinding bindSkill(Long agentId, Long skillId) { + requireSameWorkspace(agentId, skillId); // 检查是否已绑定 AgentSkillBinding existing = skillBindingMapper.selectOne( new LambdaQueryWrapper() @@ -109,6 +154,15 @@ public class AgentBindingService { * 批量设置 Agent 的 skill 绑定(替换模式) */ public void setSkillBindings(Long agentId, List skillIds) { + // Validate every incoming skill BEFORE touching the binding rows; + // a half-applied save (old bindings dropped, new set rejected + // mid-loop) would leave the agent silently un-bound from skills it + // had a moment ago. + if (skillIds != null) { + for (Long skillId : skillIds) { + requireSameWorkspace(agentId, skillId); + } + } // 删除旧绑定 skillBindingMapper.delete( new LambdaQueryWrapper() @@ -125,6 +179,78 @@ public class AgentBindingService { } } + /** + * Refuse to bind a skill that doesn't share the agent's workspace. + * Skills are per-workspace installable artifacts (each workspace has + * its own catalog under {@code mate_skill.workspace_id}); letting + * workspace A's agent bind workspace B's skill would leak capabilities + * — and prompt content — across the tenancy boundary. + * + *

Three skill id flavors to handle: + *

+ * + *

Most {@code mate_skill} rows currently sit in the default workspace + * (id=1) because skill creation doesn't yet honor the + * {@code X-Workspace-Id} header; the real-skill branch is therefore + * defense-in-depth right now and flips on automatically the moment + * workspace-scoped skill creation lands. ACP enforcement is live today. + * + * @throws MateClawException 404 if the agent or skill doesn't exist; + * 403 on a workspace mismatch. + */ + private void requireSameWorkspace(Long agentId, Long skillId) { + if (agentId == null) { + throw new MateClawException("err.agent.not_found", 404, "Agent ID is required"); + } + if (skillId == null) { + throw new MateClawException("err.skill.not_found", 404, "Skill ID is required"); + } + AgentEntity agent = agentMapper.selectById(agentId); + if (agent == null) { + throw new MateClawException("err.agent.not_found", 404, "Agent 不存在: " + agentId); + } + // MCP virtual: no workspace on McpServerEntity — globally bindable. + if (McpSkillBridge.isVirtualMcpSkillId(skillId)) { + return; + } + SkillEntity skill; + if (AcpSkillBridge.isVirtualAcpSkillId(skillId)) { + // ACP virtual: synthesize from the bridge so workspace_id flows + // through from mate_acp_endpoint. A null reply here means the + // backing endpoint was deleted or disabled between picker render + // and save — same surface as a deleted real skill. + skill = acpSkillBridge.findEntityById(skillId); + if (skill == null) { + throw new MateClawException("err.skill.not_found", 404, + "ACP endpoint backing skill " + skillId + " is gone or disabled"); + } + } else { + skill = skillMapper.selectById(skillId); + if (skill == null) { + throw new MateClawException("err.skill.not_found", 404, "Skill 不存在: " + skillId); + } + } + long agentWs = agent.getWorkspaceId() == null ? 1L : agent.getWorkspaceId(); + long skillWs = skill.getWorkspaceId() == null ? 1L : skill.getWorkspaceId(); + if (agentWs != skillWs) { + throw new MateClawException("err.skill.cross_workspace_binding", 403, + "Skill " + skillId + " (workspace=" + skillWs + + ") cannot be bound to Agent " + agentId + + " (workspace=" + agentWs + ")"); + } + } + // ==================== Tool Bindings ==================== public List listToolBindings(Long agentId) { @@ -175,6 +301,19 @@ public class AgentBindingService { * → contribute nothing through this path; legacy SKILL.md prompt * enhancement still runs separately. * + * + *

Auto-included on every non-null result, in addition to the bound + * tools and skill-expanded tools: + *

*/ public Set getEffectiveToolNames(Long agentId) { Set boundSkillIds = getBoundSkillIds(agentId); @@ -216,9 +355,41 @@ public class AgentBindingService { // (the LLM stops being able to write to LESSONS.md / MEMORY.md). merged.addAll(SYSTEM_LEVEL_TOOLS); + // Enabled MCP server tools auto-join the allowlist for the same + // reason SYSTEM_LEVEL_TOOLS does: MCP servers are an + // administrator-enabled capability, not a per-agent opt-in. Without + // this union, an agent with any skill or built-in tool bound would + // silently lose every MCP tool — users hit this when they bound one + // built-in tool, didn't tick the MCP rows, and observed "only + // built-in tools work". Operators who need to hide a specific MCP + // tool from a specific agent still have the tool-guard deny path + // (AgentGraphBuilder applies withDeniedToolsFiltered before this). + merged.addAll(getEnabledMcpToolNames()); + return merged; } + /** + * Names of every currently-bindable MCP tool, sourced from the same + * picker that the agent edit screen reads. Failures (picker outage, + * cache parse error) yield an empty set so the caller's allowlist is + * strictly narrower, never wider, than the picker — never throws. + */ + private Set getEnabledMcpToolNames() { + try { + return availableToolService.listAvailable().stream() + .filter(t -> "mcp".equals(t.getSource())) + .filter(AvailableToolDTO::isAvailable) + .map(AvailableToolDTO::getName) + .filter(n -> n != null && !n.isBlank()) + .collect(Collectors.toCollection(LinkedHashSet::new)); + } catch (Exception e) { + log.warn("AvailableToolService unavailable while computing effective tool allowlist; " + + "MCP tools will be excluded for this resolve cycle: {}", e.getMessage()); + return Collections.emptySet(); + } + } + /** * RFC-090 §11 — tools that exist outside the skill scope and must * survive any agent-level skill binding restriction. @@ -277,6 +448,11 @@ public class AgentBindingService { "image_generate", "music_generate", "video_generate", + // HTML → PNG rasteriser. Closes the loop for HTML-producing skills + // (architecture-diagram, infographics, dashboards) so IM channels + // can deliver the artifact as a native image instead of a file or + // a dead markdown link. + "render_html_image", // Universal capabilities the global system prompts (SOUL.md / // AGENTS.md / "Web Search Capability" / "File Reading Guidelines") // explicitly tell the LLM exist. Pre-Phase-2b they were globally @@ -335,9 +511,27 @@ public class AgentBindingService { } /** - * 批量设置 Agent 的 tool 绑定(替换模式) + * Replace the agent's tool binding set. + * + *

Validation rule for each incoming name: + *

    + *
  • Already in the existing binding → always allowed (so the + * user can keep a previously-bound tool whose upstream MCP server + * is currently stale or even removed; the client just keeps what + * it already had).
  • + *
  • New addition (not in existing binding) → must appear in + * {@link AvailableToolService#listAvailable()} with + * {@code available == true}. Names that are unknown + * (typos / legacy unprefixed MCP names / hand-crafted strings) or + * that the picker marked unavailable (hash collision, etc.) are + * rejected — saving them would put a {@code mate_agent_tool} row + * in the database that the runtime can never resolve, which then + * silently drops the tool when the agent runs.
  • + *
*/ public void setToolBindings(Long agentId, List toolNames) { + validateNewToolBindings(agentId, toolNames); + toolBindingMapper.delete( new LambdaQueryWrapper() .eq(AgentToolBinding::getAgentId, agentId)); @@ -352,6 +546,56 @@ public class AgentBindingService { } } + /** + * Refuse the save when any *newly-added* tool name doesn't resolve to + * an {@code available=true} row in the picker. Names already in the + * existing binding are exempt so that subsequent edits (especially + * "remove this stale tool") still succeed even if upstream state has + * drifted. + */ + private void validateNewToolBindings(Long agentId, List incoming) { + if (incoming == null || incoming.isEmpty()) { + return; + } + Set existing = listToolBindings(agentId).stream() + .map(AgentToolBinding::getToolName) + .collect(Collectors.toSet()); + Set bindable; + try { + bindable = availableToolService.listAvailable().stream() + .filter(AvailableToolDTO::isAvailable) + .map(AvailableToolDTO::getName) + .collect(Collectors.toSet()); + } catch (Exception e) { + // The picker source briefly failing must not block the user + // from saving a binding that's still in their existing set. + // Re-validate everything against just the existing set — + // strictly conservative: only allow keeps, refuse adds. + log.warn("AvailableToolService unavailable during binding validation, falling back to existing-only: {}", + e.getMessage()); + bindable = Set.of(); + } + + List rejected = new java.util.ArrayList<>(); + for (String name : incoming) { + if (name == null || name.isBlank()) { + rejected.add(""); + continue; + } + if (existing.contains(name)) continue; // keeps are always allowed + if (!bindable.contains(name)) rejected.add(name); + } + if (!rejected.isEmpty()) { + String preview = rejected.size() <= 5 + ? String.join(", ", rejected) + : String.join(", ", rejected.subList(0, 5)) + " (+" + (rejected.size() - 5) + " more)"; + throw new MateClawException("err.agent.tool_binding_unbindable", + "Tool name(s) cannot be bound: " + preview + + ". Either the name is unknown or the picker marked it unavailable " + + "(e.g. hash collision, upstream server removed)."); + } + } + // ==================== Provider Preferences (RFC-009 PR-3) ==================== /** Raw rows for the agent edit form. Sorted by sort_order ascending. */ diff --git a/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java index 4ec000e6..e512681a 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/chatmodel/AgentDashScopeChatModelBuilder.java @@ -172,7 +172,7 @@ public class AgentDashScopeChatModelBuilder implements ChatModelBuilder { } if (!modelProviderService.hasUsableApiKey(apiKey)) { throw new MateClawException("err.agent.dashscope_key_missing", - "DashScope API Key 未配置,请在模型设置中填写 dashscope 的 API Key,或设置 DASHSCOPE_API_KEY 环境变量"); + "DashScope API Key 未配置,请在「设置 → 模型 → 添加供应商」中为 dashscope 填写 API Key"); } builder.apiKey(apiKey.trim()); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/context/ConversationWindowManager.java b/mateclaw-server/src/main/java/vip/mate/agent/context/ConversationWindowManager.java index bda69ee9..2f4b9f43 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/context/ConversationWindowManager.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/context/ConversationWindowManager.java @@ -14,6 +14,7 @@ import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.tool.ToolCallback; import com.alibaba.cloud.ai.dashscope.chat.DashScopeChatOptions; import org.springframework.stereotype.Component; +import vip.mate.agent.graph.executor.ToolResultStorage; import vip.mate.agent.prompt.PromptLoader; import vip.mate.config.ConversationWindowProperties; import vip.mate.memory.spi.MemoryManager; @@ -21,6 +22,7 @@ import vip.mate.workspace.conversation.ConversationService; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; /** @@ -61,18 +63,35 @@ public class ConversationWindowManager { /** 迭代更新:合并旧摘要 + 新轮次 */ private static final String STRUCTURED_SUMMARY_UPDATE = PromptLoader.loadPrompt("context/structured-summary-update"); - /** 摘要注入前缀 */ - private static final String SUMMARY_PREFIX = + /** 摘要注入前缀 (package-private for test assertions) */ + static final String SUMMARY_PREFIX = "[上下文压缩] 更早的对话轮次已被压缩为摘要以节省上下文空间。" + "以下摘要描述了已完成的工作,当前会话状态可能已反映这些变更。" + "请基于摘要和当前状态继续,避免重复已完成的工作:\n\n"; + /** + * Marker prefix used by the first-user anchor. Lets compaction skip + * previously-injected anchors when looking for the "real" first user + * message in a subsequent round. + * + *

Package-private so unit tests can assert on the marker. + */ + static final String ANCHOR_PREFIX = "[Original goal]\n"; + // ==================== 序列化截断参数 ==================== private static final int CONTENT_MAX = 6000; private static final int CONTENT_HEAD = 4000; private static final int CONTENT_TAIL = 1500; - private static final int OLD_TOOL_RESULT_SUMMARY_THRESHOLD = 500; + + /** + * Minimum body size at which the duplicate-output placeholder is preferred + * over keeping the verbatim copy. Below this size the placeholder text + * (~80 chars) is comparable to the body itself, so deduplication only + * complicates the prompt without saving meaningful tokens. Above this + * size the dedup placeholder is a real win. + */ + private static final int DEDUP_MIN_CHARS = 500; /** * Tool names whose results must never be compacted into a one-line @@ -99,6 +118,35 @@ public class ConversationWindowManager { private final MemoryManager memoryManager; private final ConversationService conversationService; + /** + * Optional spill store, injected via setter so unit tests and the two + * existing 3-arg constructor callers in tests stay source-compatible. + * When {@code null}, prune falls back to "keep originals verbatim" — no + * lossy summary rewrite is ever applied. Spring autowires this when + * {@link ToolResultStorage} is on the context. + */ + private ToolResultStorage toolResultStorage; + + @org.springframework.beans.factory.annotation.Autowired(required = false) + public void setToolResultStorage(ToolResultStorage toolResultStorage) { + this.toolResultStorage = toolResultStorage; + } + + /** + * Optional stream tracker for broadcasting {@code compact_status} + * SSE events. Wired via setter so unit tests can leave it {@code null} + * without dragging in the channel layer. When present, every + * compaction emits start/skipped/summarize/done events so the + * frontend can render a boundary card and a status line in real + * time. + */ + private vip.mate.channel.web.ChatStreamTracker streamTracker; + + @org.springframework.beans.factory.annotation.Autowired(required = false) + public void setStreamTracker(vip.mate.channel.web.ChatStreamTracker streamTracker) { + this.streamTracker = streamTracker; + } + // ==================== 状态 ==================== /** 摘要缓存:key = "conversationId:oldMessageCount" */ @@ -133,7 +181,7 @@ public class ConversationWindowManager { Integer maxInputTokens, ChatModel chatModel, String conversationId, Long agentId) { return fitToWindow(messages, systemPrompt, currentUserMessage, - maxInputTokens, chatModel, conversationId, agentId, null); + maxInputTokens, chatModel, conversationId, agentId, null, null); } /** @@ -149,10 +197,33 @@ public class ConversationWindowManager { Integer maxInputTokens, ChatModel chatModel, String conversationId, Long agentId, java.util.Collection toolCallbacks) { + return fitToWindow(messages, systemPrompt, currentUserMessage, + maxInputTokens, chatModel, conversationId, agentId, toolCallbacks, null); + } + + /** + * Most comprehensive overload — adds {@code workspaceBasePath} so the + * pre-pass that prunes old tool results can route oversized bodies to + * the agent's workspace spill directory via {@link ToolResultStorage}. + * + *

When {@code workspaceBasePath} is {@code null}, spill files land in + * the configured base dir, or the JVM tmpdir as last resort (see + * {@link ToolResultStorage#resolveBaseDir(String)}). Workspace-aware + * callers should always pass the path so historical spill files stay + * grouped with the workspace that produced them. + */ + public List fitToWindow(List messages, String systemPrompt, + String currentUserMessage, + Integer maxInputTokens, ChatModel chatModel, + String conversationId, Long agentId, + java.util.Collection toolCallbacks, + String workspaceBasePath) { if (messages == null || messages.isEmpty()) { return messages; } - messages = pruneOldToolResultsForModelInput(messages); + long spillsAtEntry = (toolResultStorage != null) ? toolResultStorage.getSpillCount() : 0L; + + messages = pruneOldToolResultsForModelInput(messages, conversationId, workspaceBasePath); int effectiveMax = (maxInputTokens != null && maxInputTokens > 0) ? maxInputTokens : properties.getDefaultMaxInputTokens(); @@ -176,7 +247,7 @@ public class ConversationWindowManager { // 可用于历史的 token 预算 = max - system - currentMsg - tools - 安全余量 int reservedTokens = systemTokens + currentMsgTokens + toolsTokens + (int) (effectiveMax * 0.05); - // RFC-025 Change 1: reserve 硬封顶到 effectiveMax 的 50%。 + // 预留 reserve 硬封顶到 effectiveMax 的 50%。 // 小上下文模型(Ollama 16K、本地 8K)下,systemTokens + currentMsgTokens 很容易 // 接近或超过 effectiveMax,不封顶会让 historyBudget 变负数导致死循环压缩 // (压缩目标比压缩前还大 → 压缩后又触发压缩)。 @@ -191,7 +262,8 @@ public class ConversationWindowManager { // 尾部保护 token 预算:阈值的 20%(与 Hermes 一致) int tailTokenBudget = (int) (triggerThreshold * 0.20); - return compactMessages(messages, historyBudget, tailTokenBudget, chatModel, conversationId, agentId); + return compactMessages(messages, historyBudget, tailTokenBudget, chatModel, + conversationId, agentId, totalTokens, spillsAtEntry); } /** @@ -207,18 +279,62 @@ public class ConversationWindowManager { // ==================== 核心压缩逻辑 ==================== + /** Broadcast a single compact_status event; silent no-op when no tracker is wired. */ + private void broadcastCompactStatus(String conversationId, String status, Map extra) { + if (streamTracker == null || conversationId == null || conversationId.isEmpty()) { + return; + } + try { + Map payload = new java.util.LinkedHashMap<>(); + payload.put("status", status); + payload.put("timestamp", System.currentTimeMillis()); + if (extra != null) payload.putAll(extra); + streamTracker.broadcastObject(conversationId, "compact_status", payload); + } catch (Exception e) { + log.debug("[ConversationWindow] broadcast compact_status failed: {}", e.getMessage()); + } + } + private List compactMessages(List messages, int historyBudget, int tailTokenBudget, ChatModel chatModel, - String conversationId, Long agentId) { + String conversationId, Long agentId, + int preTokens, long spillsAtEntry) { + broadcastCompactStatus(conversationId, "start", Map.of( + "preTokens", preTokens, + "messagesIn", messages.size(), + "trigger", "token_threshold" + )); + // 动态计算尾部保护边界(替代固定 preserveRecentPairs) int headEnd = 0; // 头部保护:暂不保护(system prompt 已在外部计算) int tailStart = findTailBoundary(messages, headEnd, tailTokenBudget); if (tailStart <= headEnd) { log.debug("[ConversationWindow] 消息数不足以拆分,跳过压缩"); + broadcastCompactStatus(conversationId, "skipped", + Map.of("reason", "insufficient_messages")); return messages; } + // Pair safety: never split an AssistantMessage's tool_calls from its + // matching ToolResponseMessages. The cut may walk forward (i.e. the + // tail grows) until every call/response cluster lives on one side of + // the boundary. If no safe cut survives the walk, skip compaction — + // a broken pair would 400 every OpenAI-compatible provider, which is + // strictly worse than letting context cross the budget by one extra + // turn. + int pairSafeCut = enforcePairSafeBoundary(messages, headEnd, tailStart); + if (pairSafeCut <= headEnd) { + broadcastCompactStatus(conversationId, "skipped", + Map.of("reason", "pair_boundary_collapsed")); + return messages; + } + if (pairSafeCut != tailStart) { + broadcastCompactStatus(conversationId, "pair_safe", Map.of( + "movedFrom", tailStart, "movedTo", pairSafeCut)); + } + tailStart = pairSafeCut; + List oldMessages = new ArrayList<>(messages.subList(headEnd, tailStart)); List recentMessages = messages.subList(tailStart, messages.size()); @@ -274,13 +390,20 @@ public class ConversationWindowManager { // 计算动态摘要预算 int summaryBudget = computeSummaryBudget(forSummary); + broadcastCompactStatus(conversationId, "summarize", Map.of( + "messagesToSummarize", oldMessages.size(), + "summaryBudget", summaryBudget + )); + // 检查缓存 String cacheKey = conversationId + ":" + oldMessages.size(); CachedSummary cached = summaryCache.get(cacheKey); String summary; + boolean fromCache = false; if (cached != null && !cached.isExpired(CACHE_TTL_MS)) { summary = cached.summary(); + fromCache = true; log.debug("[ConversationWindow] 命中摘要缓存, conv={}", conversationId); } else { summary = generateSummary(forSummary, chatModel, conversationId, summaryBudget, memoryExtraContext); @@ -289,27 +412,32 @@ public class ConversationWindowManager { int count = compressionCounts.merge(conversationId, 1, Integer::sum); log.info("[ConversationWindow] 生成结构化摘要 ({} 字符, 第 {} 次压缩), 压缩 {} 条旧消息, conv={}", summary.length(), count, oldMessages.size(), conversationId); - - // 持久化摘要到 DB:下次加载历史时可直接从摘要位置开始,跳过重复压缩 - if (conversationService != null) { - try { - conversationService.saveCompressionSummary( - conversationId, SUMMARY_PREFIX + summary, oldMessages.size()); - } catch (Exception e) { - log.warn("[ConversationWindow] Failed to persist compression summary: {}", e.getMessage()); - } - } } } // 组装结果 List result = new ArrayList<>(); + boolean anchored = false; if (summary != null && !summary.isBlank()) { result.add(new UserMessage(SUMMARY_PREFIX + summary)); + + // Anchor the original user goal so a long task that paged through + // dozens of turns can still see what was originally asked. Always + // as a UserMessage — promoting historical user input to a + // SystemMessage would be a privilege-escalation risk. + Message anchor = buildFirstUserAnchor(oldMessages); + if (anchor != null) { + result.add(anchor); + anchored = true; + } } else if (!oldMessages.isEmpty()) { log.warn("[ConversationWindow] 摘要生成失败,降级为保留最近 4 条旧消息, conv={}", conversationId); int fallbackKeep = Math.min(4, oldMessages.size()); result.addAll(oldMessages.subList(oldMessages.size() - fallbackKeep, oldMessages.size())); + broadcastCompactStatus(conversationId, "failed", Map.of( + "reason", "summary_generation_failed", + "fallbackKept", fallbackKeep + )); } result.addAll(recentMessages); @@ -318,6 +446,48 @@ public class ConversationWindowManager { if (resultTokens > historyBudget && result.size() > 2) { log.warn("[ConversationWindow] 压缩后仍超预算: {} > {}, 执行二次裁剪", resultTokens, historyBudget); result = trimToFit(result, historyBudget); + resultTokens = TokenEstimator.estimateTokens(result); + } + + // Persist the boundary + announce completion only when the summary + // actually wrote a row. Failed-summary fallback already broadcast + // its own event above. + if (summary != null && !summary.isBlank() && conversationService != null && !fromCache) { + long spillsThisTurn = (toolResultStorage != null) + ? Math.max(0L, toolResultStorage.getSpillCount() - spillsAtEntry) + : 0L; + Map boundaryMetadata = new java.util.LinkedHashMap<>(); + boundaryMetadata.put("trigger", "token_threshold"); + boundaryMetadata.put("preTokens", preTokens); + boundaryMetadata.put("postTokens", resultTokens); + boundaryMetadata.put("messagesSummarized", oldMessages.size()); + boundaryMetadata.put("tailKept", recentMessages.size()); + boundaryMetadata.put("toolResultsSpilled", spillsThisTurn); + boundaryMetadata.put("anchored", anchored); + Long summaryId = null; + try { + summaryId = conversationService.saveCompressionSummaryReturningId( + conversationId, SUMMARY_PREFIX + summary, oldMessages.size(), + boundaryMetadata); + } catch (Exception e) { + log.warn("[ConversationWindow] Failed to persist compression boundary: {}", e.getMessage()); + } + if (summaryId != null) { + // Mirror the DB row's metadata: the SSE consumer needs the id + // to deep-link the boundary card without having to refetch. + boundaryMetadata.put("summaryId", summaryId); + } + broadcastCompactStatus(conversationId, "done", boundaryMetadata); + } else if (summary != null && !summary.isBlank() && fromCache) { + // Cached summary path — no new DB row, but emit done so the + // frontend status bar still updates. + broadcastCompactStatus(conversationId, "done", Map.of( + "preTokens", preTokens, + "postTokens", resultTokens, + "messagesSummarized", oldMessages.size(), + "tailKept", recentMessages.size(), + "fromCache", true + )); } return result; @@ -362,6 +532,201 @@ public class ConversationWindowManager { return Math.max(cutIdx, headEnd + 1); } + /** + * Adjust the candidate boundary so an {@link AssistantMessage}'s + * {@code toolCalls} are never separated from their matching + * {@link ToolResponseMessage}s. + * + *

Walks forward, collecting every {@code tool_call_id}'s assistant + * index and the indices of its matching responses. Whenever an + * assistant in the prefix has at least one response in the tail, the + * cut moves backward to that assistant — pulling the whole cluster + * into the tail. The walk repeats until convergence because moving + * the cut can expose pairs that were previously fully in the tail. + * + *

The method preserves pair integrity above any other concern. If + * the cut collapses all the way to {@code headEnd}, callers must + * interpret the return as "skip compaction this turn" — splitting a + * pair would produce HTTP 400 on every OpenAI-compatible provider, + * which is a worse failure mode than letting context grow by one turn. + * + *

An orphan {@code ToolResponseMessage} (id matching no + * assistant in scope) does not trigger movement; the upstream code + * paths should never produce one, and logging at WARN gives us a + * breadcrumb if they ever do. + * + * @return adjusted cut index, or {@code headEnd} when no pair-safe + * cut larger than {@code headEnd} can be produced. + */ + // Package-private so unit tests in the same package can drive it directly + // without standing up a ChatModel + the rest of the compactMessages pipeline. + int enforcePairSafeBoundary(List messages, int headEnd, int tailStart) { + if (tailStart <= headEnd || tailStart >= messages.size()) { + return tailStart; + } + int cut = tailStart; + int safety = messages.size() + 1; // hard guard against pathological loops + while (safety-- > 0) { + // Map: tool_call_id -> earliest assistant index that issued it. + java.util.Map assistantIdxById = new java.util.HashMap<>(); + // Map: tool_call_id -> max response index closing it. + java.util.Map latestResponseIdxById = new java.util.HashMap<>(); + + for (int i = headEnd; i < messages.size(); i++) { + Message m = messages.get(i); + if (m instanceof AssistantMessage am && am.getToolCalls() != null) { + for (AssistantMessage.ToolCall tc : am.getToolCalls()) { + String tid = tc.id(); + if (tid == null || tid.isEmpty()) continue; + // Keep the first occurrence so the cut "snaps" to the + // earliest assistant for any duplicated ids; the same + // id should never repeat anyway. + assistantIdxById.putIfAbsent(tid, i); + } + } else if (m instanceof ToolResponseMessage trm) { + for (ToolResponseMessage.ToolResponse r : trm.getResponses()) { + String tid = r.id(); + if (tid == null || tid.isEmpty()) continue; + latestResponseIdxById.merge(tid, i, Math::max); + } + } + } + + // Find the earliest in-prefix assistant whose pair is split. + int earliestSplitAssistant = Integer.MAX_VALUE; + for (var e : assistantIdxById.entrySet()) { + String id = e.getKey(); + int aIdx = e.getValue(); + Integer rIdx = latestResponseIdxById.get(id); + if (rIdx == null) { + // Assistant issued a call but no response — orphan call, + // would already break the provider. Not a pair-split, ignore. + continue; + } + if (aIdx < cut && rIdx >= cut && aIdx < earliestSplitAssistant) { + earliestSplitAssistant = aIdx; + } + if (aIdx >= cut && rIdx < cut) { + log.warn("[ConversationWindow] Orphan tool response in prefix without preceding assistant in tail (id={}); leaving boundary alone", + id); + } + } + + if (earliestSplitAssistant == Integer.MAX_VALUE) { + break; // converged: no splits remain + } + cut = earliestSplitAssistant; + } + + if (cut <= headEnd) { + log.info("[ConversationWindow] Pair-safe boundary collapsed to {} for conv: skipping compaction this turn to avoid splitting a tool_call ↔ tool_response pair", + headEnd); + return headEnd; + } + + int prefixSize = cut - headEnd; + int minPrefix = Math.max(0, properties.getPairSafeMinPrefixToCompact()); + if (prefixSize < minPrefix) { + log.info("[ConversationWindow] Pair-safe boundary left {} prefix message(s) (< minPrefix={}); skipping compaction", + prefixSize, minPrefix); + return headEnd; + } + + if (cut != tailStart) { + log.info("[ConversationWindow] Pair-safe boundary moved {} -> {} to keep tool_call ↔ tool_response pairs intact", + tailStart, cut); + } + return cut; + } + + /** + * Build an anchor message replaying the first real user input + * found in the compressed prefix. "Real" here excludes prior + * compaction artifacts ({@link #SUMMARY_PREFIX} / {@link #ANCHOR_PREFIX} + * messages from earlier rounds), because anchoring the previous + * summary defeats the purpose — the model would just see "[Original + * goal] [上下文压缩] …" pointing at compressor output, not at the user's + * actual request. + * + *

Sizing rules: + *

    + *
  • ≤ {@code firstUserAnchorMaxTokens}: keep the original text verbatim.
  • + *
  • ≤ 3× the budget: head+tail truncate to the budget so most of + * the prompt-cache benefit survives.
  • + *
  • > 3× the budget: degrade to a 200-char pointer line so we + * don't blow prompt cache or the summary budget on a single + * message that was probably a pasted spec the model can re-read + * from the workspace anyway.
  • + *
+ * + *

Always returns a {@link UserMessage}. {@code null} when anchoring + * is disabled, no real first user exists in the prefix, or the body is + * blank. + * + *

Package-private for direct unit testing — the surrounding + * {@link #compactMessages} path needs a ChatModel and the whole + * structured-summary pipeline, which the anchor logic does not. + */ + Message buildFirstUserAnchor(List oldMessages) { + if (!properties.isFirstUserAnchorEnabled()) { + return null; + } + UserMessage firstUser = null; + for (Message m : oldMessages) { + if (!(m instanceof UserMessage um)) continue; + String text = um.getText(); + if (text == null) continue; + // Skip synthetic prior-round artifacts. + if (text.startsWith(SUMMARY_PREFIX) || text.startsWith(ANCHOR_PREFIX)) { + continue; + } + firstUser = um; + break; + } + if (firstUser == null) return null; + + String text = firstUser.getText(); + if (text == null || text.isBlank()) return null; + + int maxAnchorTokens = Math.max(40, properties.getFirstUserAnchorMaxTokens()); + int textTokens = TokenEstimator.estimateTokens(text); + + if (textTokens <= maxAnchorTokens) { + return new UserMessage(ANCHOR_PREFIX + text); + } + + // > 3× budget: cheap pointer line so we don't pay token tax for a + // gigantic pasted spec. The model still knows the original goal + // existed without seeing the full body. + if (textTokens > maxAnchorTokens * 3L) { + int pointerChars = Math.min(text.length(), 200); + String pointer = text.substring(0, pointerChars).stripTrailing() + + (text.length() > pointerChars ? "..." : ""); + log.info("[ConversationWindow] First-user anchor downgraded to pointer ({} tokens > 3× budget {})", + textTokens, maxAnchorTokens); + return new UserMessage(ANCHOR_PREFIX + pointer); + } + + // Within 3× — head+tail truncate to the budget. The 2 chars/token + // ratio is a deliberate over-estimate so the anchor never inflates + // past the configured budget on ASCII-heavy input. + int budgetChars = Math.max(160, maxAnchorTokens * 2); + if (budgetChars >= text.length()) { + return new UserMessage(ANCHOR_PREFIX + text); + } + int headLen = (int) (budgetChars * 0.6); + int tailLen = Math.max(40, budgetChars - headLen - 40); + if (headLen + tailLen >= text.length()) { + return new UserMessage(ANCHOR_PREFIX + text); + } + String truncated = text.substring(0, headLen) + + "\n...[" + (text.length() - headLen - tailLen) + " chars truncated]...\n" + + text.substring(text.length() - tailLen); + log.info("[ConversationWindow] First-user anchor head+tail truncated ({} -> ~{} chars)", + text.length(), truncated.length()); + return new UserMessage(ANCHOR_PREFIX + truncated); + } + /** * 计算摘要字数预算:被压缩内容 token 的 20%,不低于 500、不超过 3000。 */ @@ -374,7 +739,54 @@ public class ConversationWindowManager { // ==================== 工具结果处理 ==================== + /** + * Backwards-compatible overload — older tool results that are oversized + * stay verbatim because no {@link ToolResultStorage} target is in + * scope. New call sites should use the 3-arg overload with explicit + * {@code conversationId} and {@code workspaceBasePath} so oversized + * bodies can be spilled to disk and recovered via {@code read_file}. + */ public List pruneOldToolResultsForModelInput(List messages) { + return pruneOldToolResultsForModelInput(messages, null, null); + } + + /** + * Walk the messages newest-to-oldest, keeping the latest tool response + * verbatim and applying space-saving rewrites to older ones: + * + *

    + *
  1. Bodies already starting with {@link ToolResultStorage#SPILL_MARKER_PREFIX} + * were spilled at tool-execution time — pass through untouched.
  2. + *
  3. If a body matches an identical body already seen in a newer turn, + * replace it with a short "duplicate tool output omitted" placeholder + * (only above {@link #DEDUP_MIN_CHARS} so we don't bloat tiny acks).
  4. + *
  5. Otherwise, when a {@link ToolResultStorage} is wired and a + * conversation id is available, try + * {@link ToolResultStorage#persistIfOversized} to spill the raw + * bytes to disk and replace the inline body with a preview + path + * so the model can read_file the original on demand.
  6. + *
  7. If none of the above apply, leave the body verbatim. Bodies + * under the spill threshold or running without a storage hook are + * preserved exactly — the lossy "summarized for model context" + * single-liner that used to fire here destroyed enough context + * on long tasks to be the wrong default.
  8. + *
+ * + *

The {@link #PRUNE_EXEMPT_TOOLS} set still bypasses everything: + * sub-agent delegations are not replayable, so their full transcript + * stays in context. + * + * @param messages full conversation in chronological order + * @param conversationId used to scope spill files; {@code null} disables spill + * @param workspaceBasePath used to locate the spill directory; {@code null} + * falls back through the storage's resolveBaseDir chain + */ + public List pruneOldToolResultsForModelInput(List messages, + String conversationId, + String workspaceBasePath) { + if (messages == null || messages.isEmpty()) { + return messages; + } int latestToolResponseIndex = -1; for (int i = messages.size() - 1; i >= 0; i--) { if (messages.get(i) instanceof ToolResponseMessage) { @@ -386,9 +798,13 @@ public class ConversationWindowManager { return messages; } + boolean canSpill = toolResultStorage != null + && conversationId != null && !conversationId.isEmpty(); + List pruned = new ArrayList<>(messages); java.util.Set seenLargeOutputs = new java.util.HashSet<>(); int changed = 0; + int spilled = 0; for (int i = pruned.size() - 1; i >= 0; i--) { if (!(pruned.get(i) instanceof ToolResponseMessage trm)) { continue; @@ -398,23 +814,53 @@ public class ConversationWindowManager { boolean messageChanged = false; for (ToolResponseMessage.ToolResponse r : trm.getResponses()) { String data = r.responseData(); - boolean exempt = r.name() != null && PRUNE_EXEMPT_TOOLS.contains(r.name()); - if (keepFull || exempt || data == null || data.length() <= OLD_TOOL_RESULT_SUMMARY_THRESHOLD) { + String name = r.name(); + boolean exempt = name != null && PRUNE_EXEMPT_TOOLS.contains(name); + boolean alreadySpilled = data != null + && data.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX); + + // Pass through: the latest response, exempt tools, empty bodies, + // already-spilled previews — none should be rewritten. + if (keepFull || exempt || data == null || data.isEmpty() || alreadySpilled) { newResponses.add(r); - if (data != null && data.length() > OLD_TOOL_RESULT_SUMMARY_THRESHOLD) { + if (data != null && data.length() > DEDUP_MIN_CHARS) { seenLargeOutputs.add(data); } continue; } - String replacement; - if (seenLargeOutputs.contains(data)) { - replacement = "[" + r.name() + "] duplicate tool output omitted; same content appeared later."; - } else { - replacement = summarizeToolResponse(r.name(), data); + + // Dedup: identical body seen in a later turn already. + if (data.length() > DEDUP_MIN_CHARS && seenLargeOutputs.contains(data)) { + String replacement = "[" + name + + "] duplicate tool output omitted; same content appeared later."; + newResponses.add(new ToolResponseMessage.ToolResponse(r.id(), name, replacement)); + messageChanged = true; + continue; + } + + // Spill on demand: route oversized bodies to disk so the model + // can read_file them rather than losing them to a lossy summary. + if (canSpill) { + String candidate = toolResultStorage.persistIfOversized( + data, name, r.id(), conversationId, workspaceBasePath); + if (candidate != null + && candidate.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)) { + newResponses.add(new ToolResponseMessage.ToolResponse(r.id(), name, candidate)); + seenLargeOutputs.add(data); + messageChanged = true; + spilled++; + continue; + } + // returned unchanged: under threshold, excluded tool, or write failed. + // Fall through to "keep verbatim". + } + + // Default: keep the body verbatim. Better to send a few extra + // tokens than to silently destroy data the model might need. + newResponses.add(r); + if (data.length() > DEDUP_MIN_CHARS) { seenLargeOutputs.add(data); } - newResponses.add(new ToolResponseMessage.ToolResponse(r.id(), r.name(), replacement)); - messageChanged = true; } if (messageChanged) { pruned.set(i, ToolResponseMessage.builder().responses(newResponses).build()); @@ -422,47 +868,43 @@ public class ConversationWindowManager { } } if (changed > 0) { - log.info("[ConversationWindow] Pruned {} older tool response message(s) before model request", changed); + log.info("[ConversationWindow] Pruned {} older tool response message(s) ({} spilled to disk) before model request", + changed, spilled); } return changed > 0 ? pruned : messages; } - private static String summarizeToolResponse(String toolName, String data) { - int chars = data.length(); - int lines = data.isBlank() ? 0 : data.split("\\R", -1).length; - String firstLine = firstNonBlankLine(data); - if (firstLine.length() > 160) { - firstLine = firstLine.substring(0, 160) + "..."; - } - StringBuilder sb = new StringBuilder(); - sb.append('[').append(toolName).append("] previous tool output summarized for model context: ") - .append(chars).append(" chars, ").append(lines).append(" lines"); - if (!firstLine.isBlank()) { - sb.append(". First line: ").append(firstLine); - } - return sb.toString(); - } - - private static String firstNonBlankLine(String data) { - for (String line : data.split("\\R")) { - String trimmed = line.trim(); - if (!trimmed.isBlank()) { - return trimmed.replace('|', '/'); - } - } - return ""; + /** + * Spill-marker responses already point at an on-disk full copy via + * {@code path=...} in their body. Trimming, replacing, or pre-pruning + * them would destroy the very pointer the model needs to recover the + * original output with {@code read_file} — which is the whole reason + * we spilled in the first place. All three compaction phases consult + * this guard before touching a response. + */ + static boolean isSpillMarker(ToolResponseMessage.ToolResponse r) { + return r != null + && r.responseData() != null + && r.responseData().startsWith(ToolResultStorage.SPILL_MARKER_PREFIX); } /** * Phase 1 - Soft trim:对工具结果做 head+tail 裁剪(保留首尾各 200 字符)。 + *

Spill-marker responses are left untouched so their on-disk pointer + * survives intact across compaction. */ - private int softTrimToolResults(List messages) { + int softTrimToolResults(List messages) { int trimmed = 0; for (int i = 0; i < messages.size(); i++) { if (messages.get(i) instanceof ToolResponseMessage trm) { List newResponses = new ArrayList<>(); boolean changed = false; for (ToolResponseMessage.ToolResponse r : trm.getResponses()) { + if (isSpillMarker(r)) { + // Pointer + preview already; trimming would lose the path. + newResponses.add(r); + continue; + } String data = r.responseData(); if (data != null && data.length() > 500) { String head = data.substring(0, 200); @@ -485,16 +927,28 @@ public class ConversationWindowManager { /** * Phase 2 - Hard clear:将所有旧工具结果替换为占位符。 + *

Spill-marker responses are left untouched so the on-disk pointer + * survives — a placeholder here would force the model to abandon a + * tool output it could otherwise recover via {@code read_file}. */ - private int hardClearToolResults(List messages) { + int hardClearToolResults(List messages) { int cleared = 0; for (int i = 0; i < messages.size(); i++) { if (messages.get(i) instanceof ToolResponseMessage trm) { - List placeholders = trm.getResponses().stream() - .map(r -> new ToolResponseMessage.ToolResponse(r.id(), r.name(), "[tool result removed]")) - .toList(); - messages.set(i, ToolResponseMessage.builder().responses(placeholders).build()); - cleared++; + boolean changed = false; + List replaced = new ArrayList<>(trm.getResponses().size()); + for (ToolResponseMessage.ToolResponse r : trm.getResponses()) { + if (isSpillMarker(r)) { + replaced.add(r); + continue; + } + replaced.add(new ToolResponseMessage.ToolResponse(r.id(), r.name(), "[tool result removed]")); + changed = true; + } + if (changed) { + messages.set(i, ToolResponseMessage.builder().responses(replaced).build()); + cleared++; + } } } return cleared; @@ -502,18 +956,27 @@ public class ConversationWindowManager { /** * Phase 3 Pre-prune:在 LLM 摘要前,将工具输出替换为占位符(减少摘要输入 token)。 + *

Spill-marker responses are left untouched so the summary input + * still has the on-disk path the model might cite back in its summary. */ - private int prePruneForSummary(List messages) { + int prePruneForSummary(List messages) { int pruned = 0; for (int i = 0; i < messages.size(); i++) { if (messages.get(i) instanceof ToolResponseMessage trm) { boolean hasSubstantial = trm.getResponses().stream() - .anyMatch(r -> r.responseData() != null && r.responseData().length() > 200); + .anyMatch(r -> !isSpillMarker(r) + && r.responseData() != null + && r.responseData().length() > 200); if (hasSubstantial) { - List placeholders = trm.getResponses().stream() - .map(r -> new ToolResponseMessage.ToolResponse(r.id(), r.name(), - "[旧工具输出已清理以节省上下文空间]")) - .toList(); + List placeholders = new ArrayList<>(trm.getResponses().size()); + for (ToolResponseMessage.ToolResponse r : trm.getResponses()) { + if (isSpillMarker(r)) { + placeholders.add(r); + continue; + } + placeholders.add(new ToolResponseMessage.ToolResponse(r.id(), r.name(), + "[旧工具输出已清理以节省上下文空间]")); + } messages.set(i, ToolResponseMessage.builder().responses(placeholders).build()); pruned++; } diff --git a/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java b/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java index 1526041b..f2d6aac9 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/controller/AgentController.java @@ -11,7 +11,13 @@ import vip.mate.channel.web.Utf8SseEmitter; import vip.mate.agent.AgentService; import vip.mate.agent.AgentState; import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.vo.AgentCapabilitiesVO; import vip.mate.audit.service.AuditEventService; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; import vip.mate.auth.model.UserEntity; import vip.mate.auth.service.AuthService; import vip.mate.common.result.R; @@ -40,16 +46,22 @@ public class AgentController { private final AuditEventService auditEventService; private final AuthService authService; private final WorkspaceService workspaceService; + private final ModelConfigService modelConfigService; + private final ModelCapabilityService modelCapabilityService; + private final SystemSettingService systemSettingService; private final ExecutorService sseExecutor = Executors.newCachedThreadPool(); @Operation(summary = "获取Agent列表") @GetMapping @RequireWorkspaceRole("viewer") public R> list( - @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId, + @RequestParam(value = "enabled", required = false) Boolean enabled) { // 无 header 时强制使用默认 workspace,不返回全局数据 long wsId = workspaceId != null ? workspaceId : 1L; - return R.ok(agentService.listAgentsByWorkspace(wsId)); + // enabled=true: chat selectors hide disabled agents. + // enabled=null: admin management page sees enabled + disabled. + return R.ok(agentService.listAgentsByWorkspace(wsId, enabled)); } @Operation(summary = "获取Agent详情") @@ -62,6 +74,57 @@ public class AgentController { return R.ok(agent); } + @Operation(summary = "获取Agent当前能力(modality 集合 + sidecar 配置),用于聊天页提示条") + @GetMapping("/{id}/capabilities") + @RequireWorkspaceRole("viewer") + public R capabilities( + @PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + AgentEntity agent = agentService.getAgent(id); + verifyResourceWorkspace(agent.getWorkspaceId(), workspaceId); + + ModelConfigEntity primary; + try { + primary = modelConfigService.resolveModel(agent.getModelName()); + } catch (Exception e) { + // No default model configured yet — return a capabilities snapshot that + // tells the UI "we can't say anything about this agent's modalities". + return R.ok(AgentCapabilitiesVO.builder() + .agentId(id) + .modelName("") + .providerId("") + .modalities(List.of()) + .build()); + } + java.util.Set modalities = + modelCapabilityService.resolve(primary.getModelName(), primary.getModalities()); + + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long visionId = settings.getDefaultVisionModelId(); + Long videoId = settings.getDefaultVideoModelId(); + + return R.ok(AgentCapabilitiesVO.builder() + .agentId(id) + .modelName(primary.getModelName()) + .providerId(primary.getProvider()) + .modalities(modalities.stream().map(Enum::name).toList()) + .defaultVisionModelId(visionId) + .defaultVisionModelLabel(resolveSidecarLabel(visionId)) + .defaultVideoModelId(videoId) + .defaultVideoModelLabel(resolveSidecarLabel(videoId)) + .build()); + } + + private String resolveSidecarLabel(Long modelId) { + if (modelId == null) return null; + try { + ModelConfigEntity m = modelConfigService.getModel(modelId); + return m == null ? null : m.getProvider() + " / " + m.getModelName(); + } catch (Exception e) { + return null; + } + } + @Operation(summary = "创建Agent") @PostMapping @RequireWorkspaceRole("member") @@ -127,6 +190,7 @@ public class AgentController { @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { AgentEntity agent = agentService.getAgent(id); verifyResourceWorkspace(agent != null ? agent.getWorkspaceId() : null, workspaceId); + verifyAgentEnabled(agent); // RFC-058 PR-1: Utf8SseEmitter 显式 charset=UTF-8,防止中文 SSE 乱码 SseEmitter emitter = new Utf8SseEmitter(5 * 60 * 1000L); @@ -166,6 +230,7 @@ public class AgentController { @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { AgentEntity agent = agentService.getAgent(id); verifyResourceWorkspace(agent != null ? agent.getWorkspaceId() : null, workspaceId); + verifyAgentEnabled(agent); return R.ok(agentService.chat(id, request.getMessage(), request.getConversationId())); } @@ -178,6 +243,7 @@ public class AgentController { @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { AgentEntity agent = agentService.getAgent(id); verifyResourceWorkspace(agent != null ? agent.getWorkspaceId() : null, workspaceId); + verifyAgentEnabled(agent); return R.ok(agentService.execute(id, request.getMessage(), request.getConversationId())); } @@ -208,6 +274,21 @@ public class AgentController { } } + /** + * Block runtime calls against an agent flagged as disabled. + * + *

{@code AgentService#getOrBuildAgent} also checks the flag, but only on + * a cache miss — once the {@code BaseAgent} instance is warm, a flip to + * disabled would silently keep serving requests until something else + * invalidates the cache. Enforcing here at the controller closes that gap + * for every external entry point. + */ + private void verifyAgentEnabled(AgentEntity agent) { + if (agent != null && !Boolean.TRUE.equals(agent.getEnabled())) { + throw new MateClawException("err.agent.disabled", "Agent 已禁用: " + agent.getName()); + } + } + private Long resolveUserId(Authentication auth) { if (auth == null) { throw new MateClawException("err.auth.unauthenticated", 401, "Not authenticated"); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/event/AgentLifecycleEvent.java b/mateclaw-server/src/main/java/vip/mate/agent/event/AgentLifecycleEvent.java new file mode 100644 index 00000000..b7fecd88 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/event/AgentLifecycleEvent.java @@ -0,0 +1,25 @@ +package vip.mate.agent.event; + +/** + * Spring application event fired when an agent's lifecycle state changes. + * The trigger module subscribes via {@code @EventListener} and forwards + * the payload through {@code TriggerEventIngestService} so triggers of + * pattern type {@code agent_lifecycle} can fan out to workflows. + * + *

{@code phase} matches the matcher's vocabulary: {@code spawned} for + * a fresh create, {@code enabled} / {@code disabled} for a flag flip, + * {@code terminated} for a delete. {@code crashed} is reserved for v1 + * once the agent runtime grows a structured error hook. + * + *

The dedup key downstream is {@code phase + ":" + agentId + ":" + + * timestamp}; that's stable across retries of the same operation but + * lets the same agent flip enabled/disabled repeatedly without the + * trigger pipeline collapsing the events. + */ +public record AgentLifecycleEvent( + long workspaceId, + long agentId, + String agentName, + String phase, + long timestamp +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java index bf3d37cb..7b3fba58 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/NodeStreamingChatHelper.java @@ -1,5 +1,6 @@ package vip.mate.agent.graph; +import com.fasterxml.jackson.databind.ObjectMapper; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.chat.messages.AssistantMessage; import org.springframework.ai.chat.messages.Message; @@ -283,6 +284,35 @@ public class NodeStreamingChatHelper { */ private static final int THINKING_ONLY_HARD_CAP_CHARS = 32768; + /** + * Narrow content-repetition guard — fires when the buffer ends with + * the same period-sized chunk repeated {@link + * #CONTENT_REPEAT_MAX_OCCURRENCES}+ times in a row. Picked to catch + * the specific failure mode where reasoning-mode models (qwen3.6, + * deepseek-r1) get into a "Wait, I should X. → 写答案 → Wait, I + * should Y. → 写同一份答案 → …" self-arguing loop and emit the same + * final-answer paragraph dozens of times until {@code max_tokens} + * runs out. + * + *

Tests probe sizes from {@link #CONTENT_REPEAT_MIN_PERIOD} up + * to {@link #CONTENT_REPEAT_MAX_PERIOD}; the smallest period that + * yields the required consecutive copies trips the guard. 4 + * verbatim consecutive copies of any 24+ char unit is a near- + * impossible coincidence in real text, so false positives are very + * rare. Not as exhaustive as the previous {@code RepetitionDetector} + * (removed at 42d406ff for being brittle on legitimate long-form + * content), just the cheap specific check that catches this loop. + */ + public static final int CONTENT_REPEAT_MIN_PERIOD = 24; + public static final int CONTENT_REPEAT_MAX_PERIOD = 240; + private static final int CONTENT_REPEAT_MAX_OCCURRENCES = 4; + /** + * Re-scan every N chars of new content. Smaller = faster reaction, + * larger = less CPU. The probe loop is O(period_range × occurrences) + * char comparisons per scan — cheap even at 400-char intervals. + */ + private static final int CONTENT_REPEAT_CHECK_INTERVAL = 200; + private static final int MAX_RETRIES = 5; // RATE_LIMIT: fail fast to failover chain — staying on the same // provider during a rate-limit window wastes time without recovery. @@ -291,6 +321,8 @@ public class NodeStreamingChatHelper { private static final long BACKOFF_BASE_MS = 3000; private static final long BACKOFF_CAP_MS = 60_000; + private static final ObjectMapper TOOL_ARG_JSON_MAPPER = new ObjectMapper(); + /** * 判断错误是否可重试(基于状态码/异常类型) */ @@ -373,11 +405,31 @@ public class NodeStreamingChatHelper { || msg.contains("invalid_request_error") || msg.contains("unsupported")) { return ErrorType.CLIENT_ERROR; } - // Server errors + // Server errors and transient TLS / socket-level network hiccups. + // Without the TLS-specific patterns, a single SSL fatal alert + // (e.g. bad_record_mac during long-running streams) falls through to + // UNKNOWN — non-retryable — so one transient handshake glitch surfaces + // to the user as "LLM 调用失败" with no recovery attempt. These are + // network-layer transients that almost always succeed on retry, so + // they belong in the same retryable bucket as 5xx/timeouts. if (msg.contains("500") || msg.contains("502") || msg.contains("503") || msg.contains("504") || msg.contains("APITimeoutError") || msg.contains("APIConnectionError") || msg.contains("Connection reset") || msg.contains("Connection refused") - || msg.contains("timeout") || msg.contains("Timeout")) { + || msg.contains("timeout") || msg.contains("Timeout") + // TLS-layer transients: bad_record_mac (RFC 5246 §7.2.2 fatal + // alert 20), aborted handshakes, mid-stream protocol errors. + || msg.contains("SSLException") || msg.contains("SSLHandshakeException") + || msg.contains("SSLProtocolException") || msg.contains("bad_record_mac") + // Socket-level transients: a peer closing the TCP connection + // mid-response, or the OS reporting a half-closed pipe. + || msg.contains("SocketException") || msg.contains("Broken pipe") + || msg.contains("Premature close") || msg.contains("PrematureCloseException") + || msg.contains("Connection prematurely closed") + || msg.contains("Connection closed prematurely") + // Reactor Netty wraps the raw socket cause in WebClientRequestException; + // surface that wrapper too so retries fire even when the cause chain + // string is "WebClientRequestException ...; nested ... SSLException". + || msg.contains("WebClientRequestException")) { return ErrorType.SERVER_ERROR; } return ErrorType.UNKNOWN; @@ -718,6 +770,16 @@ public class NodeStreamingChatHelper { // 仅保留 thinking-only 这条体积兜底,处理 volcengine-plan 等 provider // 在 thinking 通道堆字符不出 content 的死循环(生产 trace c1eefa45)。 AtomicBoolean thinkingOnlyCapTriggered = new AtomicBoolean(false); + // Content-repetition guard: trips when the same paragraph-sized + // suffix appears CONTENT_REPEAT_MAX_OCCURRENCES+ times in + // contentAccum. The outer poll loop disposes the upstream + // subscription within 500ms once flipped — same pattern as the + // thinking-only cap above. + AtomicBoolean contentRepeatCapTriggered = new AtomicBoolean(false); + // Last contentAccum length at which we ran the repetition scan. + // Throttles the O(n) substring scan so it runs at most once per + // CONTENT_REPEAT_CHECK_INTERVAL chars, not on every chunk. + AtomicInteger lastContentRepeatCheckLen = new AtomicInteger(0); // Lifecycle events emitted at most once per call so consumers can // pivot the UI between "thinking" and "drafting" without inspecting @@ -760,7 +822,7 @@ public class NodeStreamingChatHelper { lastAssistantMessage.set(msg); // thinking-only soft cap 已触发 → 跳过一切处理(等外层 dispose) - if (thinkingOnlyCapTriggered.get()) { + if (thinkingOnlyCapTriggered.get() || contentRepeatCapTriggered.get()) { return; } @@ -847,6 +909,36 @@ public class NodeStreamingChatHelper { return; } + // 5. Content-repetition guard. Some reasoning-mode models + // (qwen3.6, deepseek-r1) get stuck in a "Wait, I should X + // → 写答案 → Wait, I should Y → 写同一份答案 → ..." loop + // and emit the same final-answer paragraph dozens of times + // until max_tokens runs out. Without this, the user sees a + // wall of duplicated text and the bot never actually finishes. + // Throttled to one scan per CONTENT_REPEAT_CHECK_INTERVAL + // chars of new content — the probe loop is cheap but no + // need to run on every chunk. + int currentLen = contentAccum.length(); + int floor = CONTENT_REPEAT_MIN_PERIOD * CONTENT_REPEAT_MAX_OCCURRENCES; + if (currentLen >= floor + && currentLen - lastContentRepeatCheckLen.get() >= CONTENT_REPEAT_CHECK_INTERVAL) { + lastContentRepeatCheckLen.set(currentLen); + if (hasRepeatingSuffix(contentAccum, CONTENT_REPEAT_MIN_PERIOD, + CONTENT_REPEAT_MAX_PERIOD, + CONTENT_REPEAT_MAX_OCCURRENCES)) { + log.warn("[{}] Content-repetition cap reached " + + "({} chars, tail repeated {}+ times) " + + "— disposing stream for conversation {}", + phase, currentLen, CONTENT_REPEAT_MAX_OCCURRENCES, + conversationId); + broadcastContentTruncated(conversationId, + "content_repetition", + currentLen); + contentRepeatCapTriggered.set(true); + return; + } + } + // 4. 提取 token usage(通常最后一个 chunk 携带完整 usage) if (chatResponse.getMetadata() != null && chatResponse.getMetadata().getUsage() != null) { var usage = chatResponse.getMetadata().getUsage(); @@ -884,6 +976,20 @@ public class NodeStreamingChatHelper { // dispose 后 latch 可能不会 countDown,直接跳出 break; } + if (contentRepeatCapTriggered.get()) { + // Same dispose pattern as thinking-only cap. The + // accumulated content is preserved (it's the looping + // text — at least the user gets the FIRST occurrence + // as a partial answer instead of waiting for max_tokens). + log.warn("[{}] Stream guard tripped (content_repetition), disposing " + + "upstream subscription for conversation {}", phase, conversationId); + subscription.dispose(); + if (broadcast) { + broadcastDelta(conversationId, "warning", + buildDeltaJson("检测到回答内容反复重复,已自动截断")); + } + break; + } if (streamTracker.isStopRequested(conversationId)) { // 用户主动停止 — 也 dispose 上游 subscription.dispose(); @@ -979,21 +1085,26 @@ public class NodeStreamingChatHelper { conversationId, phase, errorType); } - // ===== 成功(检查是否因 thinking-only 软上限被截断) ===== + // ===== 成功(检查是否因 thinking-only 软上限或内容重复被截断) ===== boolean truncatedByThinkingCap = thinkingOnlyCapTriggered.get(); + boolean truncatedByContentRepeat = contentRepeatCapTriggered.get(); + boolean truncated = truncatedByThinkingCap || truncatedByContentRepeat; if (truncatedByThinkingCap) { log.warn("[{}] LLM stream disposed: thinking-only soft cap reached for conversation {}", phase, conversationId); + } else if (truncatedByContentRepeat) { + log.warn("[{}] LLM stream disposed: content-repetition cap reached for conversation {}", + phase, conversationId); } // RFC-009: guard against silent empty responses. Some providers return // HTTP 200 with an empty body under soft-failure conditions (rate-limit // capacity, context filter, upstream overload). Treat this as a failure // signal so streamCallInternal can hand off to the fallback chain. - // Only fire when the thinking-only cap didn't fire (which deliberately - // produces thinking-only output) and there are no tool calls + // Only fire when neither truncation cap fired (those deliberately + // produce non-empty output) and there are no tool calls // (tool-only responses are legitimately empty-text). - if (!truncatedByThinkingCap + if (!truncated && contentAccum.length() == 0 && thinkingAccum.length() == 0 && toolCallAccumulators.isEmpty()) { @@ -1001,11 +1112,14 @@ public class NodeStreamingChatHelper { return buildErrorResultWithType("LLM 返回空响应", conversationId, phase, ErrorType.EMPTY_RESPONSE); } + String truncationReason = truncatedByThinkingCap ? "thinking_only_no_content" + : truncatedByContentRepeat ? "content_repetition" + : null; return assembleResult(contentAccum, thinkingAccum, toolCallAccumulators, promptTokens.get(), completionTokens.get(), cacheReadTokens.get(), cacheWriteTokens.get(), phase, - truncatedByThinkingCap, - truncatedByThinkingCap ? "thinking_only_no_content" : null); + truncated, + truncationReason); } /** 组装 stopped partial 结果(用户主动停止,有已累积内容) */ @@ -1516,6 +1630,99 @@ public class NodeStreamingChatHelper { } } + /** + * Collapse a content buffer's trailing run of verbatim repeats to a + * single copy. Used to clean up the persisted final answer after + * {@link #hasRepeatingSuffix} fires — the streamed text already + * contains the duplicates (SSE chunks can't be unsent), but the + * DB-persisted message and the IM channel reply should show ONE + * clean copy of the looping unit, not a wall. + * + *

Algorithm: find the smallest period in {@code [minPeriod, + * maxPeriod]} where the buffer ends with that unit repeated 2+ + * times consecutively, then return everything up to (and including) + * the FIRST copy of that unit. Conservative — if no period yields + * 2+ consecutive matches, returns the buffer unchanged. + * + *

Public for unit-testing alongside {@link #hasRepeatingSuffix}. + */ + public static String dedupTrailingRepeats(String content, int minPeriod, int maxPeriod) { + if (content == null || content.isEmpty()) return content; + int len = content.length(); + if (minPeriod <= 0 || maxPeriod < minPeriod) return content; + int periodCap = Math.min(maxPeriod, len / 2); + for (int p = minPeriod; p <= periodCap; p++) { + int unitStart = len - p; + // Walk backward as far as the unit keeps matching. + int copies = 1; + int blockStart = unitStart - p; + while (blockStart >= 0 + && content.regionMatches(blockStart, content, unitStart, p)) { + copies++; + blockStart -= p; + } + if (copies >= 2) { + // Keep prefix + ONE copy. The first copy starts at + // (blockStart + p) since the loop walked back one step + // past the last match. + int firstCopyStart = blockStart + p; + int trimEnd = firstCopyStart + p; + return content.substring(0, trimEnd); + } + } + return content; + } + + /** + * Detect whether {@code accum} ends with the same {@code period}-sized + * unit repeated at least {@code minOccurrences} times consecutively, + * for some {@code period} in {@code [minPeriod, maxPeriod]}. Returns + * true when the model is stuck in a "self-arguing" loop emitting the + * same final-answer chunk over and over. + * + *

Algorithm: probe period sizes from small to large. For each + * candidate period {@code p}, take the last {@code p} chars as the + * unit and check whether the {@code minOccurrences-1} preceding + * blocks of length {@code p} are byte-identical. The smallest period + * that yields the required consecutive copies trips the guard. We + * iterate small→large because tighter periods are more specific: + * a 30-char unit repeated 4× is a stronger signal than a 200-char + * unit happening to appear once. + * + *

Cost: O(periodRange × occurrences × period) char comparisons. + * For default thresholds (~200 × 4 × 100) that's ~80K comparisons + * per scan — microseconds against an LLM call. Throttled by the + * caller via {@code lastContentRepeatCheckLen} so the scan amortizes. + * + *

Package-private + static for unit-testing the threshold without + * spinning up a full {@code StreamResult}. + */ + static boolean hasRepeatingSuffix(CharSequence accum, int minPeriod, int maxPeriod, + int minOccurrences) { + if (accum == null) return false; + int len = accum.length(); + if (minPeriod <= 0 || minOccurrences <= 1 || maxPeriod < minPeriod) return false; + if (len < minPeriod * minOccurrences) return false; + String s = accum.toString(); + int periodCap = Math.min(maxPeriod, len / minOccurrences); + for (int p = minPeriod; p <= periodCap; p++) { + // Unit = last p chars. Check prior (minOccurrences - 1) + // blocks of length p match the unit byte-for-byte. + int unitStart = len - p; + boolean allMatch = true; + for (int k = 2; k <= minOccurrences; k++) { + int blockStart = len - k * p; + if (blockStart < 0) { allMatch = false; break; } + if (!s.regionMatches(blockStart, s, unitStart, p)) { + allMatch = false; + break; + } + } + if (allMatch) return true; + } + return false; + } + /** * Best-effort character count of the outbound prompt for the * {@code context_prepared} event. Cheaper than tokenizing and only used @@ -1637,11 +1844,49 @@ public class NodeStreamingChatHelper { acc.id, acc.type != null ? acc.type : "function", acc.name, - acc.arguments.toString())); + sanitizeToolCallArguments(acc.name, acc.arguments.toString()))); } return result; } + /** + * Ensure {@code function.arguments} is always a well-formed JSON string. + *

+ * Some providers (e.g. aliyun-codingplan) reject the entire follow-up + * request with HTTP 400 when the assistant message in history carries a + * tool call whose {@code arguments} is not parseable JSON. Streaming + * accumulation can produce such payloads when: + *

    + *
  • The model emits zero-argument tool calls as {@code ""} instead + * of {@code "{}"}.
  • + *
  • The upstream stream is truncated mid-token, leaving a partial + * JSON fragment like {@code "{\"a\":"}.
  • + *
+ * Both cases are normalized to {@code "{}"} so the chat-completions + * round-trip stays valid. Tool execution downstream still re-validates + * arguments and surfaces a per-tool error if the empty payload is wrong + * for that tool. + */ + private static String sanitizeToolCallArguments(String toolName, String arguments) { + if (arguments == null || arguments.isBlank()) { + return "{}"; + } + try { + TOOL_ARG_JSON_MAPPER.readTree(arguments); + return arguments; + } catch (Exception e) { + log.warn("Tool '{}' arguments are not valid JSON after stream aggregation " + + "(len={}, head={}); replacing with empty object so the " + + "follow-up chat-completions request stays well-formed. " + + "Parse error: {}", + toolName, + arguments.length(), + arguments.substring(0, Math.min(80, arguments.length())), + e.getMessage()); + return "{}"; + } + } + private static class ToolCallAccumulator { String id; String type; diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java index aa040600..f845d9a3 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/StateGraphReActAgent.java @@ -198,7 +198,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC AtomicInteger lastSoftCap = new AtomicInteger(0); AtomicBoolean sawLegitimateExit = new AtomicBoolean(false); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); List allEvents = GraphEventPublisher.extractEvents(output); @@ -263,7 +263,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> { setState(AgentState.IDLE); if (!sawLegitimateExit.get()) { @@ -326,7 +326,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC AtomicInteger lastSoftCap = new AtomicInteger(0); AtomicBoolean sawLegitimateExit = new AtomicBoolean(false); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); // 1. 提取所有累积的事件,只发送新增部分 @@ -402,7 +402,7 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> { setState(AgentState.IDLE); if (!sawLegitimateExit.get()) { @@ -439,12 +439,15 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC chatModel, conversationId, parsedAgentId, - toolSet != null ? toolSet.callbacks() : null); + toolSet != null ? toolSet.callbacks() : null, + workspaceBasePath); } List messages = new ArrayList<>(historyMessages); // 构建当前用户消息:支持 multimodal(如果有图片附件,直接注入 Media) - messages.add(buildCurrentUserMessage(conversationId, userMessage)); + // 同步获取 routing decision,写入 state 供后续节点 / accumulator 读取。 + BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage); + messages.add(currentTurn.userMessage()); Map inputs = new HashMap<>(); // 输入 @@ -482,6 +485,15 @@ public class StateGraphReActAgent extends BaseAgent implements StructuredStreamC inputs.put(RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : ""); inputs.put(TRACE_ID, UUID.randomUUID().toString().substring(0, 8)); + // Multimodal sidecar routing — null when the turn carries no media or + // the primary model already covers the modalities. Stored as a Map so + // graph state stays JSON-friendly. + if (currentTurn.routingDecision() != null + && currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE + || (currentTurn.routingDecision() != null && !currentTurn.routingDecision().skipped().isEmpty())) { + inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap()); + } + // RFC-063r §2.5: enrich the originating ChatOrigin with this agent's id // and workspace, then write it into graph state so ActionNode + // StepExecutionNode can forward it to ToolExecutionExecutor → ToolContext. diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java index f0bd059d..aba6c73a 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolExecutionExecutor.java @@ -79,22 +79,70 @@ public class ToolExecutionExecutor { ); /** - * Layer 1 — hard truncation cap applied to every tool result before it - * reaches ToolResultStorage (Layer 2 spill) or the LLM prompt. + * Inline hard-truncate cap for a single tool result. Acts as the fallback + * when raw-first spill cannot run (storage disabled, tool excluded, body + * already at-or-below the spill threshold, or disk write failed). * - *

Two-level budget chain (RFC-008 / RFC-06 D-5): + *

Per-tool-result handling chain: *

-     *   raw tool result
-     *     → truncateToolResult(..., MAX_TOOL_RESULT_CHARS=8000)   // Layer 1: hard cap
-     *     → persistIfOversized(..., perResultThresholdChars=16000) // Layer 2: spill to disk
-     *     → enforceTurnBudget(..., perTurnBudgetChars=32000)      // Layer 3: per-turn aggregate
+     *   raw tool result (full bytes)
+     *     → spillRawOrTruncate(...)
+     *         ├─ persistIfOversized(...) tries to write the raw body to disk
+     *         │     when size > perResultThresholdChars and tool is not
+     *         │     in the spill exclusion list. Returns a SPILL_MARKER preview
+     *         │     on success, or the original string otherwise.
+     *         └─ if no SPILL_MARKER on the return, truncateToolResult(...)
+     *               caps inline to MAX_TOOL_RESULT_CHARS so a multi-MB raw
+     *               body never enters the model prompt.
+     *     → enforceTurnBudget(..., perTurnBudgetChars=32000)   // per-turn aggregate
      * 
- * Layer 1 runs first and is intentionally kept at 8000 to prevent oversized - * results from inflating the prompt. Layers 2/3 thresholds are configured in - * {@link ToolResultProperties} and application.yml. + * Spill must see the RAW result so the full output is preserved on disk + * and the model can call {@code read_file} on the spill path. Truncating + * before spilling would write a pre-shortened blob to disk, defeating the + * "ground truth on disk" guarantee. {@link ToolResultProperties} controls + * the thresholds; this constant stays in code because it is the safety + * net for the failure case and should not vary by deployment. */ private static final int MAX_TOOL_RESULT_CHARS = 8000; + /** + * Raw-first spill: try to write the full result to disk via the spill + * store; only fall back to inline hard-truncate when no spill marker + * comes back. Caller distinguishes spill success from "returned + * unchanged" by checking {@link ToolResultStorage#SPILL_MARKER_PREFIX} + * on the returned string — otherwise an IO failure or under-threshold + * body would slip through indistinguishable from a successful spill, + * and a multi-MB raw body could end up in the model prompt. + * + *

Package-private + static so the spill/truncate decision is unit + * testable in isolation from the rest of the executor. + * + * @param storage spill store; {@code null} skips the spill attempt + * @param maxTruncateChars fallback inline hard cap + * @param result raw tool output (may be {@code null}) + * @param toolName used in the spill preview header + * @param toolUseId unique within the conversation; becomes the file name + * @param conversationId spill files are scoped per conversation; blank/null falls back to "unknown" + * @param workspaceBasePath where the spill directory lives when set + * @return the SPILL_MARKER preview when spill succeeded, otherwise the + * original string (when ≤ threshold) or the inline-truncated string. + */ + static String spillRawOrTruncate(ToolResultStorage storage, int maxTruncateChars, + String result, String toolName, String toolUseId, + String conversationId, String workspaceBasePath) { + if (result == null) return null; + if (storage != null) { + String safeConv = conversationId != null && !conversationId.isEmpty() + ? conversationId : "unknown"; + String candidate = storage.persistIfOversized( + result, toolName, toolUseId, safeConv, workspaceBasePath); + if (candidate != null && candidate.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)) { + return candidate; + } + } + return truncateToolResult(result, maxTruncateChars); + } + /** 尾部错误模式检测 */ private static final java.util.regex.Pattern ERROR_TAIL_PATTERN = java.util.regex.Pattern.compile( "(?i)\\b(error|exception|traceback|failed|fatal|panic|stack.?trace|errno)\\b"); @@ -443,6 +491,17 @@ public class ToolExecutionExecutor { } ToolCallback callback = toolCallbackMap.get(toolName); if (callback == null) { + SkillRedirect redirect = tryAutoRedirectSkillCall(toolName, arguments, safeOrigin); + if (redirect != null) { + // Auto-redirect succeeds with success=true on the SSE event so the + // model treats the SKILL.md content as the answer to a different, + // valid question (rather than as another failed call to recover from). + events.add(GraphEventPublisher.toolComplete( + toolCall.id(), toolName, redirect.response(), true)); + allResponses.add(new ToolResponseMessage.ToolResponse( + toolCall.id(), toolName, redirect.response())); + continue; + } String msg = skillAwareNotFoundMessage(toolName); log.warn("[ToolExecutor] {}", msg); events.add(GraphEventPublisher.toolComplete(toolCall.id(), toolName, msg, false)); @@ -521,6 +580,18 @@ public class ToolExecutionExecutor { ToolCallback callback = toolCallbackMap.get(toolName); if (callback == null) { + // Same auto-redirect for pre-approved replays — a stale skill-as-tool + // approval shouldn't dead-end the conversation either. + ChatOrigin replayOriginForRedirect = ChatOrigin.EMPTY + .withConversationId(conversationId) + .withWorkspace(null, workspaceBasePath); + SkillRedirect redirect = tryAutoRedirectSkillCall(toolName, callArguments, replayOriginForRedirect); + if (redirect != null) { + events.add(GraphEventPublisher.toolComplete( + toolCall.id(), toolName, redirect.response(), true)); + return new ToolResponseMessage.ToolResponse( + toolCall.id(), toolName, redirect.response()); + } String msg = skillAwareNotFoundMessage(toolName); log.warn("[ToolExecutor] Pre-approved {}", msg); events.add(GraphEventPublisher.toolComplete(toolCall.id(), toolName, msg, false)); @@ -558,16 +629,13 @@ public class ToolExecutionExecutor { toolCall.id(), toolName, DIRECT_TOOL_PLACEHOLDER); } - // RFC-008 Layer 1 first, then Layer 2 — match the non-replay path - // in executeSingleTool so behavior stays symmetric across approval - // replays. The caller-supplied conversationId scopes spill files - // into the same per-conversation directory layout. - result = truncateToolResult(result, MAX_TOOL_RESULT_CHARS); - if (resultStorage != null && result != null) { - String spillConv = conversationId != null && !conversationId.isEmpty() ? conversationId : "unknown"; - result = resultStorage.persistIfOversized( - result, toolName, toolCall.id(), spillConv, workspaceBasePath); - } + // Raw-first spill, inline truncate as fallback. Symmetric with the + // non-replay path in executeSingleTool. The caller-supplied + // conversationId scopes spill files into the per-conversation + // directory layout. See spillRawOrTruncate javadoc for why the + // order matters. + result = spillRawOrTruncate(resultStorage, MAX_TOOL_RESULT_CHARS, + result, toolName, toolCall.id(), conversationId, workspaceBasePath); log.info("[ToolExecutor] Pre-approved tool {} returned {} chars{}", toolName, rawLen, result != null && result.length() < rawLen ? " (now " + result.length() + " after spill/truncate)" : ""); events.add(GraphEventPublisher.toolComplete(toolCall.id(), toolName, result, true)); @@ -783,20 +851,16 @@ public class ToolExecutionExecutor { } } - // RFC-008 Layer 1: hard truncation cap to prevent oversized results - // from inflating the prompt. Runs FIRST (before spill) so the spill - // store doesn't need to handle multi-MB writes for run-of-the-mill - // greps that happen to spit out a long stdout. - result = truncateToolResult(result, MAX_TOOL_RESULT_CHARS); - // RFC-008 Layer 2: spill oversized results to disk and replace - // with preview + path. Falls back to truncation when spilling is - // disabled or fails. Spill preserves the full output (read_file can - // retrieve it); the Layer 1 truncation above already capped the - // inline portion, so this layer mostly catches near-cap residues. - if (resultStorage != null && result != null) { - result = resultStorage.persistIfOversized( - result, toolName, pc.toolCall.id(), pc.conversationId, pc.workspaceBasePath); - } + // Raw-first spill: write the full output to disk and replace + // with preview + path so the model can call read_file for the + // ground truth. Fall back to inline truncate only when spilling + // is disabled, the tool is on the exclusion list, the body is + // already under the spill threshold, or the disk write fails. + // Truncating before spilling would persist a pre-shortened body + // to disk and silently lose data the model could otherwise + // recover. + result = spillRawOrTruncate(resultStorage, MAX_TOOL_RESULT_CHARS, + result, toolName, pc.toolCall.id(), pc.conversationId, pc.workspaceBasePath); log.info("[ToolExecutor] Tool {} returned {} chars{}", toolName, rawLen, result != null && result.length() < rawLen ? " (now " + result.length() + " after spill/truncate)" : ""); events.add(GraphEventPublisher.toolComplete(pc.toolCall.id(), toolName, result, true)); @@ -1049,6 +1113,79 @@ public class ToolExecutionExecutor { return "Tool not found: " + toolName; } + /** + * Holder for an auto-redirect outcome: the SKILL.md content (wrapped + * with a one-line nudge) that we substitute as the tool response when + * the LLM mistakenly calls a skill name as if it were a tool. + * + *

{@code success=true} on the substituted response so the model + * doesn't read it as "tool failed, try harder" — semantically we + * answered a different question than the one it asked, and we want + * the model to follow the redirect rather than thrash. + */ + private record SkillRedirect(String response) {} + + /** + * When the LLM calls a skill name as if it were a tool, transparently + * fetch its SKILL.md and return that as the tool response. Smaller + * models (qwen-turbo et al.) often can't act on a "not a tool — go + * read X first" hint; they keep emitting the same wrong call until the + * iteration cap. With auto-redirect, the model receives runnable + * instructions on the very first attempt and can copy the runSkillScript + * shape from SKILL.md verbatim. + * + *

Returns {@code null} if {@code toolName} isn't a registered skill, + * if {@code readSkillFile} isn't available in this agent's tool set, or + * if the redirect call itself errored — the caller then falls through + * to the usual {@code skillAwareNotFoundMessage} hint. + */ + private SkillRedirect tryAutoRedirectSkillCall(String toolName, String originalArgs, ChatOrigin origin) { + if (skillRuntimeService == null || toolName == null || toolName.isBlank()) return null; + try { + boolean isSkill = skillRuntimeService.getActiveSkills().stream() + .anyMatch(s -> s.getName() != null && s.getName().equalsIgnoreCase(toolName)); + if (!isSkill) return null; + } catch (Exception e) { + log.debug("[ToolExecutor] auto-redirect skill lookup failed: {}", e.getMessage()); + return null; + } + + ToolCallback readSkillFile = toolCallbackMap.get("readSkillFile"); + if (readSkillFile == null) { + log.debug("[ToolExecutor] readSkillFile not bound to this agent — cannot auto-redirect '{}'", toolName); + return null; + } + + String redirectArgs = "{\"skillName\":\"" + + jsonStringEscape(toolName) + + "\",\"filePath\":\"SKILL.md\"}"; + String skillMd; + try { + ToolContext ctx = (origin != null ? origin : ChatOrigin.EMPTY).toToolContext(); + skillMd = readSkillFile.call(redirectArgs, ctx); + } catch (Exception e) { + log.warn("[ToolExecutor] Auto-redirect readSkillFile failed for '{}': {}", toolName, e.getMessage()); + return null; + } + + log.info("[ToolExecutor] Auto-redirected skill-as-tool call '{}' → readSkillFile (returned {} chars)", + toolName, skillMd != null ? skillMd.length() : 0); + + String safeArgs = originalArgs == null || originalArgs.isBlank() ? "{}" : originalArgs; + String response = String.format( + "[auto-redirect] You called '%s' as a tool, but it's a Skill (documentation package). " + + "Its SKILL.md is loaded below — read the script invocation example, then call " + + "`runSkillScript(skillName=\"%s\", scriptPath=\"scripts/\", args=[...])` " + + "to actually run it. Your original payload was: %s%n%n---%n%s", + toolName, toolName, safeArgs, skillMd == null ? "" : skillMd); + return new SkillRedirect(response); + } + + /** Minimal JSON string escaping for the synthetic readSkillFile arg payload. */ + private static String jsonStringEscape(String s) { + return s.replace("\\", "\\\\").replace("\"", "\\\""); + } + // ==================== 内部数据类 ==================== private record PreparedToolCall( diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultProperties.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultProperties.java index d4c90dd4..e53c5fac 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultProperties.java @@ -40,12 +40,26 @@ public class ToolResultProperties { private boolean enabled = true; /** - * Layer 2 — a single tool result larger than this is spilled to disk. - * The executor evaluates this against the raw result before applying the - * final inline cap, so oversized content is preserved before it is shortened - * for the model request. + * Per-result spill threshold. A single tool result larger than this is + * spilled to disk and the in-context view is replaced with a short + * preview + path so the model can call {@code read_file} on demand. + * + *

Aligned with {@code ToolExecutionExecutor.MAX_TOOL_RESULT_CHARS} + * (8000): the executor now tries to spill the RAW result first; only + * when spilling is disabled, the tool is on {@link #excludedTools}, the + * body is under this threshold, or the disk write fails, does it fall + * back to truncating inline to 8000 chars. Keeping the threshold equal + * to the truncate cap yields a single semantic ladder — above the + * threshold means "preserved on disk", at-or-below means "stays inline + * verbatim". + * + *

If you want to keep more text inline before spilling, raise this + * value AND raise the executor's hard cap together; otherwise the + * 8000-char fallback truncate would silently shorten anything between + * this threshold and 8000 even when spill is disabled, defeating the + * intent. */ - private int perResultThresholdChars = 16000; // was 4000 — prevents WebSearch spill-to-disk + private int perResultThresholdChars = 8000; /** * Layer 3 — aggregate cap on combined response size in one tool turn. @@ -83,6 +97,28 @@ public class ToolResultProperties { */ private List excludedTools = List.of("read_file", "read_workspace_memory_file"); + /** + * Days to retain spill files before the scheduled cleanup deletes them. + *

Default 0 means time-based cleanup is disabled — spill files + * stay on disk until the owning conversation is explicitly deleted (which + * fires {@code purgeConversation} via {@code ConversationService}). + * This preserves the "recoverable" invariant: a summary or preview that + * cites a spill path will keep working for the whole life of the + * conversation, no matter how long it sits dormant. + *

Set to a positive value if disk pressure outweighs recoverability + * for your deployment. The scheduled sweep will then delete files whose + * mtime falls outside the retention horizon. + */ + private int retentionDays = 0; + + /** + * Cron expression for the spill-cleanup task. Defaults to once a day at + * 03:00 server-local time so cleanup runs during quiet hours. Set this + * to a Spring-recognised value (six-field cron) or change the bean + * wiring to disable it entirely. + */ + private String cleanupCron = "0 0 3 * * ?"; + public boolean isEnabled() { return enabled; } public void setEnabled(boolean enabled) { this.enabled = enabled; } @@ -116,6 +152,14 @@ public class ToolResultProperties { this.excludedTools = excludedTools == null ? List.of() : excludedTools; } + public int getRetentionDays() { return retentionDays; } + public void setRetentionDays(int retentionDays) { this.retentionDays = retentionDays; } + + public String getCleanupCron() { return cleanupCron; } + public void setCleanupCron(String cleanupCron) { + this.cleanupCron = cleanupCron == null ? "" : cleanupCron; + } + /** O(1) membership test for the exclusion list, used on every tool result. */ public Set excludedToolsSet() { return Set.copyOf(excludedTools); diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultRetentionScheduler.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultRetentionScheduler.java new file mode 100644 index 00000000..fab5ef88 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultRetentionScheduler.java @@ -0,0 +1,54 @@ +package vip.mate.agent.graph.executor; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +/** + * Drives {@link ToolResultStorage#cleanupExpired()} on a cron schedule so + * spill files don't accumulate forever. Kept in its own class instead of + * inlined into {@link ToolResultStorage} for two reasons: + * + *

    + *
  • Tests can exercise {@code cleanupExpired()} directly without + * fighting the Spring scheduler.
  • + *
  • Deployments that want to disable the schedule entirely can simply + * leave this component out of the autoconfigure path.
  • + *
+ * + *

The cron expression comes from + * {@link ToolResultProperties#getCleanupCron()} (default {@code 0 0 3 * * ?}, + * i.e. once a day at 03:00 server-local time). The retention horizon comes + * from {@link ToolResultProperties#getRetentionDays()}. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ToolResultRetentionScheduler { + + private final ToolResultStorage storage; + private final ToolResultProperties props; + + /** + * Cron-fired hook. Failures are logged at WARN so they show up in + * standard log scrapes without aborting the scheduler thread — losing + * a single sweep is fine, the next one will catch the same files. + */ + @Scheduled(cron = "${mate.agent.tool-result.cleanup-cron:0 0 3 * * ?}") + public void cleanup() { + if (props.getRetentionDays() <= 0) { + log.debug("[ToolResultRetentionScheduler] retentionDays<=0, skipping sweep"); + return; + } + try { + int deleted = storage.cleanupExpired(); + if (deleted > 0) { + log.info("[ToolResultRetentionScheduler] sweep deleted {} spill file(s) older than {} days", + deleted, props.getRetentionDays()); + } + } catch (Exception e) { + log.warn("[ToolResultRetentionScheduler] sweep failed: {}", e.getMessage(), e); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultStorage.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultStorage.java index 558d43bb..88aeb301 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultStorage.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/executor/ToolResultStorage.java @@ -58,6 +58,15 @@ public class ToolResultStorage { /** D-6: monotonically increasing spill counter for observability. */ private final java.util.concurrent.atomic.AtomicLong spillCount = new java.util.concurrent.atomic.AtomicLong(); + /** + * Workspace roots observed during this JVM's lifetime. Populated every + * time a successful spill resolves a base directory; consulted by the + * scheduled retention sweep and by {@link #purgeConversation} so we + * don't have to query the database for every workspace path. Cross-JVM + * orphans are not covered — that is documented in the cleanup javadoc. + */ + private final java.util.Set observedRoots = java.util.concurrent.ConcurrentHashMap.newKeySet(); + public ToolResultStorage(ToolResultProperties props) { this.props = props; this.excludedToolsSnapshot = props.excludedToolsSet(); @@ -270,15 +279,162 @@ public class ToolResultStorage { } private Path resolveBaseDir(String workspaceBasePath) { + Path base; if (!props.getStorageBaseDir().isEmpty()) { - return Paths.get(props.getStorageBaseDir()); + base = Paths.get(props.getStorageBaseDir()); + } else if (workspaceBasePath != null && !workspaceBasePath.isBlank()) { + base = Paths.get(workspaceBasePath, ".mateclaw", "tool-results"); + } else { + String tmp = System.getProperty("java.io.tmpdir"); + if (tmp == null || tmp.isEmpty()) return null; + base = Paths.get(tmp, "mateclaw", "tool-results"); } - if (workspaceBasePath != null && !workspaceBasePath.isBlank()) { - return Paths.get(workspaceBasePath, ".mateclaw", "tool-results"); + // Register so the retention sweep and conversation-delete hook can + // reach this root even when the workspace path is no longer in scope. + observedRoots.add(base); + return base; + } + + /** + * Roots currently known to this instance. Exposed package-private so the + * scheduled retention sweep and unit tests can enumerate them without + * touching the underlying set directly. + */ + java.util.Set getObservedRoots() { + return java.util.Collections.unmodifiableSet(observedRoots); + } + + /** + * Best-effort: delete every spill file and per-conversation directory + * older than {@link ToolResultProperties#getRetentionDays()} across all + * roots this storage has seen, plus the configured base dir and the + * tmpdir fallback. Returns the number of files deleted. + * + *

Workspaces that never received a spill in this JVM's lifetime are + * not covered. Persisting an observed-roots registry across restarts + * could fix that, but is intentionally out of scope — the operator-side + * remedy is to run a one-off cleanup with {@code storage-base-dir} + * pointed at the historical workspace. + */ + public int cleanupExpired() { + if (props.getRetentionDays() <= 0) { + return 0; + } + long cutoffEpochMillis = System.currentTimeMillis() + - (long) props.getRetentionDays() * 24L * 60L * 60L * 1000L; + + java.util.Set roots = new java.util.LinkedHashSet<>(observedRoots); + if (!props.getStorageBaseDir().isEmpty()) { + roots.add(Paths.get(props.getStorageBaseDir())); } String tmp = System.getProperty("java.io.tmpdir"); - if (tmp == null || tmp.isEmpty()) return null; - return Paths.get(tmp, "mateclaw", "tool-results"); + if (tmp != null && !tmp.isEmpty()) { + roots.add(Paths.get(tmp, "mateclaw", "tool-results")); + } + + int deleted = 0; + for (Path root : roots) { + deleted += deleteExpiredUnder(root, cutoffEpochMillis); + } + if (deleted > 0) { + log.info("[ToolResultStorage] cleanup: {} spill files removed across {} root(s)", + deleted, roots.size()); + } + return deleted; + } + + private int deleteExpiredUnder(Path root, long cutoffEpochMillis) { + if (root == null || !java.nio.file.Files.isDirectory(root)) { + return 0; + } + int deleted = 0; + try (java.util.stream.Stream stream = java.nio.file.Files.walk(root, 2)) { + for (Path p : (Iterable) stream::iterator) { + if (p.equals(root)) continue; + if (!java.nio.file.Files.isRegularFile(p)) continue; + try { + long mtime = java.nio.file.Files.getLastModifiedTime(p).toMillis(); + if (mtime < cutoffEpochMillis) { + java.nio.file.Files.deleteIfExists(p); + deleted++; + } + } catch (java.io.IOException ioe) { + log.warn("[ToolResultStorage] failed to inspect spill file {}: {}", p, ioe.getMessage()); + } + } + } catch (java.io.IOException ioe) { + log.warn("[ToolResultStorage] cleanup walk failed under {}: {}", root, ioe.getMessage()); + return deleted; + } + // Best-effort: remove emptied per-conversation directories. + try (java.util.stream.Stream stream = java.nio.file.Files.list(root)) { + for (Path child : (Iterable) stream::iterator) { + if (!java.nio.file.Files.isDirectory(child)) continue; + try (java.util.stream.Stream kids = java.nio.file.Files.list(child)) { + if (kids.findAny().isEmpty()) { + java.nio.file.Files.deleteIfExists(child); + } + } catch (java.io.IOException ignored) { + // empty-check failure is not fatal — leave the directory alone + } + } + } catch (java.io.IOException ioe) { + log.warn("[ToolResultStorage] empty-dir sweep failed under {}: {}", root, ioe.getMessage()); + } + return deleted; + } + + /** + * Delete every spill file produced for {@code conversationId} across + * all observed roots, plus the configured base and tmpdir fallback. + * Called by {@code ConversationService.deleteConversation} so spill + * directories don't outlive the conversation that owns them. + * + *

Silently no-ops when nothing matches — a conversation that never + * spilled, or one whose workspace root was never observed in this JVM, + * is simply left alone. Returns the number of files deleted. + */ + public int purgeConversation(String conversationId) { + if (conversationId == null || conversationId.isEmpty()) { + return 0; + } + String safeConv = sanitize(conversationId); + java.util.Set roots = new java.util.LinkedHashSet<>(observedRoots); + if (!props.getStorageBaseDir().isEmpty()) { + roots.add(Paths.get(props.getStorageBaseDir())); + } + String tmp = System.getProperty("java.io.tmpdir"); + if (tmp != null && !tmp.isEmpty()) { + roots.add(Paths.get(tmp, "mateclaw", "tool-results")); + } + int deleted = 0; + for (Path root : roots) { + Path convDir = root.resolve(safeConv); + if (!java.nio.file.Files.isDirectory(convDir)) continue; + try (java.util.stream.Stream stream = java.nio.file.Files.list(convDir)) { + for (Path p : (Iterable) stream::iterator) { + try { + if (java.nio.file.Files.isRegularFile(p)) { + java.nio.file.Files.deleteIfExists(p); + deleted++; + } + } catch (java.io.IOException ioe) { + log.warn("[ToolResultStorage] failed to delete spill file {}: {}", p, ioe.getMessage()); + } + } + } catch (java.io.IOException ioe) { + log.warn("[ToolResultStorage] purge walk failed under {}: {}", convDir, ioe.getMessage()); + } + try { + java.nio.file.Files.deleteIfExists(convDir); + } catch (java.io.IOException ignored) { + // non-empty after deletes (another writer raced us) — fine, leave it + } + } + if (deleted > 0) { + log.info("[ToolResultStorage] purged {} spill file(s) for conversation {}", deleted, conversationId); + } + return deleted; } /** Strip path separators and reserved characters so user-supplied IDs cannot escape the directory. */ diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/FinalAnswerNode.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/FinalAnswerNode.java index aeee304d..6792e0a6 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/FinalAnswerNode.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/FinalAnswerNode.java @@ -8,6 +8,7 @@ import vip.mate.agent.graph.state.DirectToolOutput; import vip.mate.agent.graph.state.FinishReason; import vip.mate.agent.graph.state.MateClawStateAccessor; import vip.mate.agent.graph.state.SourceEvidenceLedger; +import vip.mate.tool.document.GeneratedFileCache; import java.util.List; import java.util.Map; @@ -30,6 +31,22 @@ import java.util.Map; @Slf4j public class FinalAnswerNode implements NodeAction { + /** + * Cache used to vet {@code /api/v1/files/generated/{id}} URLs the LLM + * may have written into the final answer. {@code null} disables the + * guard (legacy callers, narrow unit tests that don't exercise file + * outputs). + */ + private final GeneratedFileCache generatedFileCache; + + public FinalAnswerNode() { + this(null); + } + + public FinalAnswerNode(GeneratedFileCache generatedFileCache) { + this.generatedFileCache = generatedFileCache; + } + @Override public Map apply(OverAllState state) throws Exception { MateClawStateAccessor accessor = new MateClawStateAccessor(state); @@ -47,7 +64,7 @@ public class FinalAnswerNode implements NodeAction { if (accessor.returnDirectTriggered()) { List outputs = accessor.directToolOutputs(); if (!outputs.isEmpty()) { - String assembled = assembleDirectAnswer(outputs); + String assembled = scrubFakeUrls(assembleDirectAnswer(outputs)); String currentThinking = accessor.currentThinking(); String existingThinking = accessor.finalThinking(); String preservedThinking = !currentThinking.isEmpty() ? currentThinking : existingThinking; @@ -70,7 +87,7 @@ public class FinalAnswerNode implements NodeAction { // 审批等待路径:Graph 因 AWAITING_APPROVAL 终止,保留已流式推送的内容用于持久化 if (accessor.awaitingApproval()) { - String preservedContent = accessor.streamedContent(); + String preservedContent = scrubFakeUrls(accessor.streamedContent()); String preservedThinking = !accessor.streamedThinking().isEmpty() ? accessor.streamedThinking() : accessor.currentThinking(); log.info("[FinalAnswerNode] AWAITING_APPROVAL — preserving streamed content " + @@ -139,6 +156,12 @@ public class FinalAnswerNode implements NodeAction { } } + // Scrub hallucinated `/api/v1/files/generated/{id}` URLs whose ids + // were never inserted into the cache. Done before evidence + // validation so the validator sees the user-visible warning rather + // than treating the fake link as a "reference". + finalAnswer = scrubFakeUrls(finalAnswer); + SourceEvidenceLedger.Validation validation = accessor.sourceEvidenceLedger().validateAnswer(finalAnswer); if (finishReason == FinishReason.NORMAL && !validation.valid()) { finishReason = FinishReason.EVIDENCE_INSUFFICIENT; @@ -147,6 +170,24 @@ public class FinalAnswerNode implements NodeAction { validation.unsupportedReferences()); } + // Build the event list. Always carries the finish_reason event so + // downstream consumers (memory gate, channel accumulator, message + // metadata persistence) see a machine-readable status. When the + // turn ended in a non-transient error, also attach a + // feedback_event so the frontend can render retry/regenerate/ + // report affordances next to the red "[错误] ..." bubble — without + // this, fatal errors leave the user staring at error text with no + // way to recover short of retyping the whole prompt. + List events = + new java.util.ArrayList<>(2); + events.add(GraphEventPublisher.finishReason(finishReason.getValue())); + if (finishReason == FinishReason.ERROR_FALLBACK) { + events.add(GraphEventPublisher.feedback( + "ERROR_FALLBACK", + finalAnswer, + List.of("retry", "regenerate", "report"))); + } + // 不重置 CONTENT_STREAMED/THINKING_STREAMED,保留上游节点的标志 var builder = MateClawStateAccessor.output() .finalAnswer(finalAnswer) @@ -160,7 +201,7 @@ public class FinalAnswerNode implements NodeAction { // signal. APPEND-strategy on PENDING_EVENTS means this // composes safely with any earlier events upstream nodes // attached. - .events(List.of(GraphEventPublisher.finishReason(finishReason.getValue()))); + .events(events); if (!finalThinking.isEmpty()) { builder.finalThinking(finalThinking); @@ -196,6 +237,16 @@ public class FinalAnswerNode implements NodeAction { return sb.toString(); } + /** + * Replace fake {@code /api/v1/files/generated/{id}} URLs (cache-miss) + * with a user-visible warning. No-op when no cache is wired (legacy + * tests) or when the answer is empty. + */ + private String scrubFakeUrls(String text) { + if (generatedFileCache == null || text == null || text.isEmpty()) return text; + return generatedFileCache.scrubMissingReferences(text); + } + private FinishReason parseFinishReason(String reason) { if (reason == null || reason.isEmpty()) { return FinishReason.NORMAL; diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java index 7c7bacf4..acfb4a1c 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/node/ReasoningNode.java @@ -348,7 +348,12 @@ public class ReasoningNode implements NodeAction { } if (conversationWindowManager != null) { - messages = conversationWindowManager.pruneOldToolResultsForModelInput(messages); + // Pass conversationId + workspaceBasePath so oversized older + // tool results can be spilled to the workspace spill directory + // (preserving the full body for read_file recovery) instead of + // being rewritten into a lossy single-line summary. + messages = conversationWindowManager.pruneOldToolResultsForModelInput( + messages, conversationId, workspaceBasePath); } promptMessages.addAll(messages); @@ -489,6 +494,46 @@ public class ReasoningNode implements NodeAction { return builder.build(); } + if (result.partial() && "content_repetition".equals(result.errorMessage())) { + // Reasoning loop: the helper disposed the stream because the + // model emitted the same paragraph 4+ times in a row (qwen3.6 + // / deepseek-r1 self-arguing pattern). The streamed text + // already showed the duplicates to the user — we can't unsend + // SSE chunks — but the persisted finalAnswer should be ONE + // clean copy so the IM channel reply and any page-reload + // history don't show the wall of repetition. Skip + // FinalAnswerNode's evidence validation: the answer is + // already truncated, applying validateAnswer on top would + // double-stamp warnings on something the user already knows + // is incomplete. + String rawContent = result.text() != null ? result.text() : ""; + String dedupedAnswer = NodeStreamingChatHelper.dedupTrailingRepeats( + rawContent, + NodeStreamingChatHelper.CONTENT_REPEAT_MIN_PERIOD, + NodeStreamingChatHelper.CONTENT_REPEAT_MAX_PERIOD); + log.warn("[ReasoningNode] Content-repetition cap hit (raw={} chars → deduped={} chars); " + + "INCOMPLETE", + rawContent.length(), dedupedAnswer.length()); + var builder = reasonOutput() + .needsToolCall(false) + .shouldSummarize(false) + .finalAnswer(dedupedAnswer.isEmpty() + ? "(模型反复输出同一段内容,已自动截断。请尝试重新生成或换个问法。)" + : dedupedAnswer) + .llmCallCount(nextLlmCallCount) + .finishReason(FinishReason.INCOMPLETE) + // contentStreamed=true because the user already saw + // the looping text in their bubble; persisting again + // via streamedContent would replay it. + .contentStreamed(true) + .thinkingStreamed(result.thinking() != null && !result.thinking().isEmpty()) + .mergeUsage(state, result); + if (result.thinking() != null && !result.thinking().isEmpty()) { + builder.finalThinking(result.thinking()); + } + return builder.build(); + } + // Fatal error:直接设置 finalAnswer 为错误文案 + ERROR_FALLBACK, // 不走 LimitExceededNode(后者会再发一次 LLM 调用,语义不对且对认证/配额错误会再失败)。 // ReasoningDispatcher 看到 !needsToolCall && !shouldSummarize → finalAnswerNode, diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java index 39cd4ecf..cf04df6b 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/StateGraphPlanExecuteAgent.java @@ -148,7 +148,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS AtomicReference lastPersistedStepResult = new AtomicReference<>(""); AtomicReference lastPersistedStepThinking = new AtomicReference<>(""); - return compiledGraph.stream(inputs, config) + return BaseAgent.routingStartupDelta(inputs).concatWith(compiledGraph.stream(inputs, config) .flatMapIterable(output -> { List deltas = new ArrayList<>(); // 1. 提取事件(只发送新增部分) @@ -218,7 +218,7 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS )); } return null; - }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty())) + }).flatMapMany(d -> d != null ? Flux.just(d) : Flux.empty()))) .doOnComplete(() -> setState(AgentState.IDLE)) .doOnError(e -> { log.error("[{}] Plan-Execute stream error: {}", agentName, e.getMessage()); @@ -267,11 +267,13 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS chatModel, conversationId, parsedAgentId, - toolSet != null ? toolSet.callbacks() : null); + toolSet != null ? toolSet.callbacks() : null, + workspaceBasePath); } List messages = new ArrayList<>(historyMessages); - messages.add(buildCurrentUserMessage(conversationId, userMessage)); + BaseAgent.CurrentTurnUserMessage currentTurn = buildCurrentUserMessageWithRouting(conversationId, userMessage); + messages.add(currentTurn.userMessage()); // 构建 working context:对历史消息做受控长度摘要 String workingContext = buildWorkingContext(historyMessages, List.of()); @@ -299,6 +301,12 @@ public class StateGraphPlanExecuteAgent extends BaseAgent implements StructuredS inputs.put(MateClawStateKeys.RUNTIME_PROVIDER_ID, runtimeProviderId != null ? runtimeProviderId : ""); inputs.put(MateClawStateKeys.TRACE_ID, UUID.randomUUID().toString().substring(0, 8)); + if (currentTurn.routingDecision() != null + && (currentTurn.routingDecision().strategy() != vip.mate.llm.routing.model.MultimodalRoutingDecision.Strategy.NONE + || !currentTurn.routingDecision().skipped().isEmpty())) { + inputs.put(MateClawStateKeys.ROUTING_DECISION, currentTurn.routingDecision().toMap()); + } + // RFC-063r §2.5: same as ReAct path — enrich and store the ChatOrigin // so StepExecutionNode (and any sub-graphs spawned via DelegateAgentTool) // can read it back from state. diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/node/StepExecutionNode.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/node/StepExecutionNode.java index 1b095cb7..b7f60d13 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/node/StepExecutionNode.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/plan/node/StepExecutionNode.java @@ -211,7 +211,11 @@ public class StepExecutionNode implements NodeAction { ChatOptions options = oaiOpts; if (conversationWindowManager != null) { - messages = conversationWindowManager.pruneOldToolResultsForModelInput(messages); + // Pass conversationId + workspaceBasePath so oversized + // older tool results can be spilled to disk instead of + // being rewritten into a lossy single-line summary. + messages = conversationWindowManager.pruneOldToolResultsForModelInput( + messages, conversationId, workspaceBasePath); } NodeStreamingChatHelper.StreamResult result = streamingHelper.streamCall( diff --git a/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java b/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java index d75f3dad..30e44678 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/graph/state/MateClawStateKeys.java @@ -83,6 +83,15 @@ public final class MateClawStateKeys { // ===== 事件流(APPEND 策略)===== public static final String PENDING_EVENTS = "pending_events"; + /** + * Multimodal routing decision for the current turn (REPLACE strategy). + * Stored as a Map ready for JSON serialization. Set by BaseAgent before + * the reasoning node runs; read back by FinalAnswerNode and (separately) + * emitted as a graph event for the SSE accumulator to write into the + * persisted message metadata under {@code metadata.routing}. + */ + public static final String ROUTING_DECISION = "routing_decision"; + // ===== 阶段标记(REPLACE 策略)===== public static final String CURRENT_PHASE = "current_phase"; diff --git a/mateclaw-server/src/main/java/vip/mate/agent/model/TemplateDTO.java b/mateclaw-server/src/main/java/vip/mate/agent/model/TemplateDTO.java index 04a3ce85..ac097c99 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/model/TemplateDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/model/TemplateDTO.java @@ -30,6 +30,25 @@ public class TemplateDTO { private String systemPrompt; private List workspaceFiles; + /** + * Skill slugs (matching {@code mate_skill.name}) to pre-bind to the newly + * hired agent. Resolved against the target workspace at apply time; any + * slug whose row is missing in that workspace is logged and skipped so a + * partially-installed environment can still hire the agent. Templates ship + * with classpath-stable slugs, not numeric IDs, because skill ids vary per + * install. + */ + private List defaultSkillSlugs; + + /** + * Tool names to pre-bind directly (bypassing the skill layer). Filtered + * against {@code AvailableToolService.listAvailable()} at apply time — + * names the picker can't resolve are dropped with a warning rather than + * aborting the hire. Use for capabilities that aren't owned by any skill, + * not for system-level tools that are already universally available. + */ + private List defaultToolNames; + @Data public static class WorkspaceFileTemplate { private String filename; diff --git a/mateclaw-server/src/main/java/vip/mate/agent/service/TemplateService.java b/mateclaw-server/src/main/java/vip/mate/agent/service/TemplateService.java index ae416515..a05b37c8 100644 --- a/mateclaw-server/src/main/java/vip/mate/agent/service/TemplateService.java +++ b/mateclaw-server/src/main/java/vip/mate/agent/service/TemplateService.java @@ -1,5 +1,6 @@ package vip.mate.agent.service; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -8,9 +9,14 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import vip.mate.agent.AgentService; +import vip.mate.agent.binding.service.AgentBindingService; import vip.mate.agent.model.AgentEntity; import vip.mate.agent.model.TemplateDTO; import vip.mate.exception.MateClawException; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.repository.SkillMapper; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.service.AvailableToolService; import vip.mate.workspace.document.WorkspaceFileService; import java.io.IOException; @@ -18,6 +24,7 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.Comparator; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -36,6 +43,9 @@ public class TemplateService { private final AgentService agentService; private final WorkspaceFileService workspaceFileService; private final ObjectMapper objectMapper; + private final AgentBindingService agentBindingService; + private final SkillMapper skillMapper; + private final AvailableToolService availableToolService; /** * 列出所有可用模板 @@ -135,9 +145,123 @@ public class TemplateService { } } + // 4. Pre-bind skills the template declares so a hired agent is + // usable out of the box ("数据分析师" already knows SQL, "代码审查员" + // already has the test-driven-development playbook). Resolution + // failures (slug missing in this workspace) are skipped with a + // warning; bind-service exceptions still propagate and roll back + // the @Transactional hire — see helper Javadoc. + applyDefaultSkillBindings(template, created); + + // 5. Pre-bind any standalone tools the template wants. Picker + // outage and unknown names are dropped with a warning; a + // setToolBindings exception still propagates (same contract as + // skills) — see helper Javadoc. + applyDefaultToolBindings(template, created); + return created; } + /** + * Resolve {@link TemplateDTO#getDefaultSkillSlugs()} to skill ids inside + * the agent's own workspace and pre-bind them. + * + *

Failure contract — read carefully. + *

    + *
  • Resolution failures (slug not present in the agent's + * workspace, blank entries) → logged and dropped. A template MUST + * stay applyable on an offline upgrade or partial-seed install + * where some bundled skills haven't landed yet.
  • + *
  • Service-layer failures + * ({@link AgentBindingService#setSkillBindings} throws — e.g. a + * race deletes the skill row between resolve and bind, or the + * workspace check rejects it) → propagate. Because + * {@link #applyTemplate} runs under {@code @Transactional}, this + * rolls back the whole hire. That's deliberate: such a throw is + * a real wiring/race problem, and pretending the hire succeeded + * would leave the user with a half-configured agent.
  • + *
+ * + *

Reads the workspace off the just-persisted {@link AgentEntity} + * rather than a separate parameter so the lookup and the validator + * inside {@code AgentBindingService.requireSameWorkspace} can never + * disagree on which workspace they're talking about. + */ + private void applyDefaultSkillBindings(TemplateDTO template, AgentEntity created) { + List slugs = template.getDefaultSkillSlugs(); + if (slugs == null || slugs.isEmpty()) return; + + // Mirror the fallback inside AgentBindingService.requireSameWorkspace: + // a null workspace_id on a row is treated as workspace 1, so the + // lookup needs to agree or we'd silently turn `eq(workspaceId, null)` + // into `IS NULL` and match nothing. + Long workspaceId = created.getWorkspaceId() == null ? 1L : created.getWorkspaceId(); + + List resolvedIds = new ArrayList<>(); + for (String slug : slugs) { + if (slug == null || slug.isBlank()) continue; + SkillEntity skill = skillMapper.selectOne(new LambdaQueryWrapper() + .eq(SkillEntity::getName, slug.trim()) + .eq(SkillEntity::getWorkspaceId, workspaceId)); + if (skill == null) { + log.warn("[Template] template {} requested skill slug '{}' not found in workspace {}; skipping", + template.getId(), slug, workspaceId); + continue; + } + resolvedIds.add(skill.getId()); + } + if (resolvedIds.isEmpty()) return; + + agentBindingService.setSkillBindings(created.getId(), resolvedIds); + log.info("[Template] template {} pre-bound {} skill(s) on agent {}", + template.getId(), resolvedIds.size(), created.getId()); + } + + /** + * Pre-filter the template's tool names through the picker so + * {@link AgentBindingService#setToolBindings} sees only resolvable names + * — its own validation would otherwise abort the call on the first + * unknown name and leave the agent with no tool bindings at all. + * + *

Failure contract mirrors {@link #applyDefaultSkillBindings}: + * picker outage and unknown names are dropped with a warning; an + * exception from {@code setToolBindings} itself still propagates and + * rolls back the hire. + */ + private void applyDefaultToolBindings(TemplateDTO template, AgentEntity created) { + List names = template.getDefaultToolNames(); + if (names == null || names.isEmpty()) return; + + Set bindable; + try { + bindable = availableToolService.listAvailable().stream() + .filter(AvailableToolDTO::isAvailable) + .map(AvailableToolDTO::getName) + .collect(Collectors.toSet()); + } catch (Exception e) { + log.warn("[Template] picker unavailable during template {} apply; skipping tool pre-bind: {}", + template.getId(), e.getMessage()); + return; + } + + List filtered = new ArrayList<>(); + for (String name : names) { + if (name == null || name.isBlank()) continue; + String trimmed = name.trim(); + if (bindable.contains(trimmed)) { + filtered.add(trimmed); + } else { + log.warn("[Template] template {} requested tool '{}' not currently bindable; skipping", + template.getId(), trimmed); + } + } + if (filtered.isEmpty()) return; + + agentBindingService.setToolBindings(created.getId(), filtered); + log.info("[Template] template {} pre-bound {} tool(s) on agent {}", + template.getId(), filtered.size(), created.getId()); + } + /** * True when the raw Accept-Language header best-matches a Chinese locale. * Implementation is intentionally simple — we only need to disambiguate diff --git a/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java b/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java new file mode 100644 index 00000000..5bae044e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/agent/vo/AgentCapabilitiesVO.java @@ -0,0 +1,43 @@ +package vip.mate.agent.vo; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; + +import java.util.List; + +/** + * Lightweight capability snapshot for an agent — answers two questions + * the chat console needs synchronously while the user is composing a message: + * + *

    + *
  • What modalities does the agent's primary model support? (drives the + * attachment routing hint above the input box.)
  • + *
  • Are sidecar models configured at the system level? (drives the + * "configure a vision model" CTA when a user attaches an image to an + * agent whose primary model can't process it.)
  • + *
+ * + *

Returned by {@code GET /api/v1/agents/{id}/capabilities}. Computed on + * each request — cheap because everything is cached service-side and we only + * read at most three rows. Not persisted on {@code mate_agent}; sidecar + * configuration is system-wide and {@code modelCapabilities} is derived from + * {@code mate_model_config.modalities}. + */ +@Data +@Builder +@AllArgsConstructor +public class AgentCapabilitiesVO { + private Long agentId; + private String modelName; + private String providerId; + /** Resolved modality set: any of {@code TEXT / VISION / VIDEO / AUDIO}. */ + private List modalities; + /** System-level vision sidecar model id, null when not configured. */ + private Long defaultVisionModelId; + /** Display name of the configured vision sidecar (provider/modelName), null when not configured. */ + private String defaultVisionModelLabel; + /** System-level video sidecar model id (reserved in v1; never wired). */ + private Long defaultVideoModelId; + private String defaultVideoModelLabel; +} diff --git a/mateclaw-server/src/main/java/vip/mate/approval/ApprovalService.java b/mateclaw-server/src/main/java/vip/mate/approval/ApprovalService.java index 89f68968..d0a7e4f9 100644 --- a/mateclaw-server/src/main/java/vip/mate/approval/ApprovalService.java +++ b/mateclaw-server/src/main/java/vip/mate/approval/ApprovalService.java @@ -92,6 +92,34 @@ public class ApprovalService { pendingMap.remove(pendingId); } + /** + * INTERNAL — drop every map entry tied to the given conversation in one pass. + * Used by {@link ApprovalWorkflowService}'s {@code ConversationDeletedEvent} + * listener to clear residue once the {@code mate_tool_approval} rows for the + * conversation have already been deleted by the cascade. Without this, a + * still-PENDING entry (or any not yet GC'd resolved entry) would survive in + * the map until TTL eviction, and {@code findPendingByConversation} would + * keep handing out a ghost approval that points at a non-existent + * conversation row. + *

+ * Only {@code ApprovalWorkflowService} should call this. + * + * @return number of entries removed + */ + int removeAllByConversation(String conversationId) { + if (conversationId == null) return 0; + int removed = 0; + var iter = pendingMap.entrySet().iterator(); + while (iter.hasNext()) { + var entry = iter.next(); + if (conversationId.equals(entry.getValue().getConversationId())) { + iter.remove(); + removed++; + } + } + return removed; + } + /** * INTERNAL — register a {@link PendingApproval} reconstructed from DB during JVM startup. * Bypasses id generation and pre-existing-entry checks; the snapshot's {@code pendingId} diff --git a/mateclaw-server/src/main/java/vip/mate/approval/ApprovalWorkflowService.java b/mateclaw-server/src/main/java/vip/mate/approval/ApprovalWorkflowService.java index 8a664032..343b5d62 100644 --- a/mateclaw-server/src/main/java/vip/mate/approval/ApprovalWorkflowService.java +++ b/mateclaw-server/src/main/java/vip/mate/approval/ApprovalWorkflowService.java @@ -8,8 +8,11 @@ import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.event.EventListener; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -17,11 +20,13 @@ import org.springframework.transaction.support.TransactionSynchronization; import org.springframework.transaction.support.TransactionSynchronizationManager; import vip.mate.agent.context.ChatOrigin; import vip.mate.agent.context.ChatOriginHolder; +import vip.mate.approval.event.WorkflowApprovalResolvedEvent; import vip.mate.approval.model.ToolApprovalEntity; import vip.mate.approval.repository.ToolApprovalMapper; import vip.mate.tool.guard.model.GuardEvaluation; import vip.mate.tool.guard.model.GuardFinding; import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.event.ConversationDeletedEvent; import java.time.Instant; import java.time.LocalDateTime; @@ -50,6 +55,12 @@ public class ApprovalWorkflowService implements ApplicationRunner { private final ToolApprovalMapper approvalMapper; private final ObjectMapper objectMapper; private final ConversationService conversationService; + /** Optional — injected only in full Spring context. The workflow + * module listens for {@link WorkflowApprovalResolvedEvent}; in tests + * that don't wire the workflow runtime this stays null and the + * publish is a no-op. */ + @Autowired(required = false) + private ApplicationEventPublisher events; /** * GC scheduler — owns the 5-minute clock for the entire approval state machine @@ -82,6 +93,26 @@ public class ApprovalWorkflowService implements ApplicationRunner { } } + /** + * Drop in-memory approval state for a deleted conversation. The cascade in + * {@link ConversationService#deleteConversation} already removed the + * {@code mate_tool_approval} rows; this listener clears the parallel + * {@code pendingMap} entries so {@code findPendingByConversation} cannot + * keep returning a ghost approval that points at a non-existent + * conversation row. + *

+ * Runs after the DB cascade commits — see + * {@link ConversationDeletedEvent}. + */ + @EventListener + public void onConversationDeleted(ConversationDeletedEvent event) { + int removed = approvalService.removeAllByConversation(event.conversationId()); + if (removed > 0) { + log.info("[ApprovalWorkflow] Dropped {} in-memory pending entries for deleted conversation {}", + removed, event.conversationId()); + } + } + /** * Reconstruct in-memory pending approvals from DB at startup, preserving the * original {@code pendingId} and {@code createdAt} so subsequent resolve / GC @@ -238,6 +269,78 @@ public class ApprovalWorkflowService implements ApplicationRunner { toolCallPayload, siblingToolCalls, agentId, null); } + /** + * Workflow-scoped approval request — creates a {@code mate_tool_approval} + * row keyed to a workflow run + step instead of a conversation, so an + * {@code await_approval} step is visible in the same approval inbox the + * tool-approval flow uses. Returns the row's auto-generated long id; the + * caller (typically {@code AwaitApprovalStepAdapter}) writes that id back + * onto {@code mate_workflow_run_pause.external_approval_id} so a future + * approval-resolve callback can map "approval X resolved → resume run Y". + * + *

The approval row's {@code conversationId} is set to + * {@code "workflow:run:{runId}"} as a synthetic key — that lets the + * existing {@link ApprovalService#findPendingByConversation} surface the + * workflow approval to operator UIs without needing a parallel query + * surface. {@code toolName} is set to {@code "workflow:{kind}"} so the + * inbox can group / filter workflow approvals from tool approvals. + * + *

v0 keeps the resume path through {@code WorkflowResumeController} + * with the pauseToken; this method does not yet wire a resolve→resume + * callback. The approval row's purpose for v0 is operator visibility + * and a stable foreign key for the pause record. + */ + public Long requestWorkflowApproval(long workspaceId, + long runId, + Long stepId, + String approvalKind, + String approvalMessage, + java.util.List approverChannels, + Integer timeoutSecs) { + try { + ToolApprovalEntity entity = new ToolApprovalEntity(); + // pendingId is the string handle the existing approval pipeline + // uses for resolve / get; "wf-" prefix lets future code branch + // on workflow-scoped vs tool-scoped approvals at a glance. The + // pending_id column is VARCHAR(32) so we trim a no-dashes UUID + // down to fit ("wf-" + 24 hex chars = 27 chars; collisions of + // 24 hex chars per workflow are astronomically rare and we + // also fall back to UNIQUE-key violation handling). + String shortId = java.util.UUID.randomUUID().toString() + .replace("-", "").substring(0, 24); + entity.setPendingId("wf-" + shortId); + entity.setConversationId("workflow:run:" + runId); + String kind = approvalKind == null || approvalKind.isBlank() ? "manual" : approvalKind.trim(); + entity.setToolName("workflow:" + kind); + entity.setSummary(approvalMessage == null ? "" : approvalMessage); + // Encode approver channels in tool_arguments so the inbox UI can + // render which channels were asked. Plain JSON to keep parsing + // trivial on the read path. + try { + if (approverChannels != null && !approverChannels.isEmpty()) { + entity.setToolArguments(objectMapper.writeValueAsString( + java.util.Map.of( + "runId", runId, + "stepId", stepId, + "approverChannels", approverChannels))); + } + } catch (Exception e) { + log.warn("[ApprovalWorkflow] failed to encode approverChannels: {}", e.getMessage()); + } + entity.setStatus("PENDING"); + entity.setCreatedAt(LocalDateTime.now()); + entity.setExpireAt(LocalDateTime.now().plusSeconds( + timeoutSecs != null && timeoutSecs > 0 ? timeoutSecs : 30 * 60)); + approvalMapper.insert(entity); + log.info("[ApprovalWorkflow] requested workflow approval row id={}, runId={}, workspace={}, kind={}", + entity.getId(), runId, workspaceId, kind); + return entity.getId(); + } catch (Exception e) { + log.warn("[ApprovalWorkflow] requestWorkflowApproval failed: {}", e.getMessage()); + return null; + } + } + /** * Resolve a pending approval (approve / deny) following the RFC-067 §4.2 two-phase * contract: snapshot → DB UPDATE conditional on {@code status='PENDING'} → @@ -484,6 +587,42 @@ public class ApprovalWorkflowService implements ApplicationRunner { if (removeFromMap) approvalService.removeFromMap(snapshot.getPendingId()); }); + // Phase 4 — workflow bridge. Workflow-scoped approval rows + // (pendingId starting with "wf-") are linked to a paused workflow + // run via {@code mate_workflow_run_pause.external_approval_id}. + // Publishing the resolve here lets the workflow module's listener + // call WorkflowResumer with the matching outcome, so an operator + // approving in the inbox actually advances the workflow instead + // of leaving it paused forever. We publish AFTER commit so a tx + // rollback can't fire a stale resume; the row id is stable + // because the row already lived in DB. + if (events != null && snapshot.getPendingId() != null + && snapshot.getPendingId().startsWith("wf-")) { + // Look up the row id since the snapshot only carries the string + // pendingId, not the long primary key. One quick equality query. + try { + ToolApprovalEntity row = approvalMapper.selectOne( + new LambdaQueryWrapper() + .eq(ToolApprovalEntity::getPendingId, snapshot.getPendingId())); + if (row != null && row.getId() != null) { + final long rowId = row.getId(); + final String pendingId = snapshot.getPendingId(); + afterCommit(() -> { + try { + events.publishEvent(new WorkflowApprovalResolvedEvent( + rowId, pendingId, snapshotStatus, /* workspaceId */ null)); + } catch (Exception e) { + log.warn("[ApprovalWorkflow] failed to publish workflow-resolved event for {}: {}", + pendingId, e.getMessage()); + } + }); + } + } catch (Exception e) { + log.warn("[ApprovalWorkflow] approval row lookup for resolve event failed for {}: {}", + snapshot.getPendingId(), e.getMessage()); + } + } + boolean consumed = "consumed".equals(snapshotStatus); ResolveOutcome outcome = consumed ? ResolveOutcome.consumed(snapshot, true, rewritten) diff --git a/mateclaw-server/src/main/java/vip/mate/approval/event/WorkflowApprovalResolvedEvent.java b/mateclaw-server/src/main/java/vip/mate/approval/event/WorkflowApprovalResolvedEvent.java new file mode 100644 index 00000000..6c4d03b1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/approval/event/WorkflowApprovalResolvedEvent.java @@ -0,0 +1,36 @@ +package vip.mate.approval.event; + +/** + * Spring application event fired when a workflow-scoped approval row is + * resolved (approved / denied / timed out / superseded). The workflow + * module subscribes via {@code @EventListener}, looks up the pause row + * by {@code mate_workflow_run_pause.external_approval_id == approvalRowId}, + * and idempotently calls {@code WorkflowResumer.resume} so the pause + * actually advances. + * + *

Without this bridge, an operator who clicks "approve" in the + * approval inbox would only flip the {@code mate_tool_approval} row to + * APPROVED — the workflow run would stay paused forever until someone + * separately POSTed the pause token to the resume endpoint. That's the + * "approval is just a visibility surface, not an actual approval" + * trap RFC §3.4 calls out. + * + *

{@code approvalRowId} is the {@code mate_tool_approval.id} long key, + * NOT the {@code pendingId} string. Pause rows store the long id in + * {@code external_approval_id}, so the listener can find the right + * pause with a single equality query. + * + *

{@code decision} mirrors the resolve vocabulary so the listener + * can route to the right {@code WorkflowResumer.ResumeOutcome}: + *

    + *
  • {@code approved} / {@code consumed} → APPROVED
  • + *
  • {@code denied} / {@code superseded} → REJECTED
  • + *
  • {@code timeout} → TIMEOUT
  • + *
+ */ +public record WorkflowApprovalResolvedEvent( + long approvalRowId, + String pendingId, + String decision, + Long workspaceId +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/AbstractChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/AbstractChannelAdapter.java index fbbb3e78..52ea9cd4 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/AbstractChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/AbstractChannelAdapter.java @@ -319,6 +319,27 @@ public abstract class AbstractChannelAdapter implements ChannelAdapter { } } + /** + * Approval notice rendering — primary implementation position. + * + *

Subclasses that support a native card surface (WeCom + * {@code button_interaction}, DingTalk {@code ActionCard}, etc.) + * override this method and may call + * {@code super.sendApprovalNotice(...)} to fall back to the text + * path on render failure / payload-too-large / etc. + * + *

Lives on the abstract class rather than as an interface + * default method so the {@code super.x(...)} call from subclasses + * resolves cleanly via Java's normal class inheritance — see + * RFC-32 §2.0.4 (C-4 fix). + */ + @Override + public void sendApprovalNotice(String targetId, + vip.mate.channel.notification.ApprovalNotice notice) { + sendMessage(targetId, + vip.mate.channel.notification.ApprovalNotificationService.staticBuildText(notice)); + } + // ==================== 模板方法(子类实现) ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/channel/AsyncTaskMediaDispatcher.java b/mateclaw-server/src/main/java/vip/mate/channel/AsyncTaskMediaDispatcher.java new file mode 100644 index 00000000..d0ea75f3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/AsyncTaskMediaDispatcher.java @@ -0,0 +1,117 @@ +package vip.mate.channel; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.channel.model.ChannelSessionEntity; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.util.List; +import java.util.Set; + +/** + * Forward async-task results (image / video / music / 3D) generated by tool + * pipelines to the IM channel that originated the conversation. + *

+ * Without this dispatcher, completion bytes only land in + * {@code mate_message} + a Web SSE broadcast — IM users (WeCom / DingTalk / + * Feishu / Telegram / etc.) see nothing arrive in their chat client because + * the tool pipeline doesn't know about channel adapters. This dispatcher + * closes that loop: look up the conversation's bound channel session, get + * the live adapter from {@link ChannelManager}, and call + * {@link ChannelAdapter#sendContentParts} so the same bytes ride the + * channel-native attachment protocol. + *

+ * Web / webchat conversations are intentionally skipped because their SSE + * stream already carries the result; double-dispatching would render the + * image twice. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class AsyncTaskMediaDispatcher { + + private final ChannelSessionStore channelSessionStore; + private final ChannelManager channelManager; + + /** + * Channel types that handle their own UX via SSE (no IM forward needed). + * Everything not in this set is treated as an IM channel and gets the + * generated parts pushed via the adapter. + */ + private static final Set WEB_CHANNEL_TYPES = Set.of("web", "webchat"); + + /** + * Forward generated content parts to the IM channel bound to this + * conversation, if any. + *

+ * Best-effort: missing session, missing adapter, or adapter exception + * are all logged at debug/warn and never propagate. The caller has + * already persisted the message to {@code mate_message} and broadcast + * to Web SSE before invoking this — IM forwarding is additive. + * + * @param conversationId the conversation id used by the agent (e.g. + * {@code wecom:XuZhanFu}, {@code dingtalk:cid_xxx}, + * {@code conv_xxx} for Web) + * @param parts assistant content parts to dispatch (typically a + * single image / video / audio / file part) + */ + public void forwardToImIfBound(String conversationId, List parts) { + if (conversationId == null || conversationId.isBlank() || parts == null || parts.isEmpty()) { + return; + } + + ChannelSessionEntity session = channelSessionStore.getSession(conversationId); + if (session == null) { + // Common case for Web-only conversations — the session is never + // populated because Web doesn't write to ChannelSessionStore. + log.debug("[async-forward] No channel session for conv={}, skipping IM forward", + conversationId); + return; + } + + String channelType = session.getChannelType(); + if (channelType == null || WEB_CHANNEL_TYPES.contains(channelType)) { + log.debug("[async-forward] conv={} is web-class ({}), skipping IM forward", + conversationId, channelType); + return; + } + + Long channelId = session.getChannelId(); + if (channelId == null) { + log.debug("[async-forward] conv={} session has no channelId, skipping", + conversationId); + return; + } + + ChannelAdapter adapter = channelManager.getAdapter(channelId).orElse(null); + if (adapter == null) { + log.warn("[async-forward] conv={} channelId={} has no live adapter (channel disabled?), skipping", + conversationId, channelId); + return; + } + + String targetId = session.getTargetId(); + if (targetId == null || targetId.isBlank()) { + log.warn("[async-forward] conv={} session has no targetId, skipping", + conversationId); + return; + } + + try { + adapter.sendContentParts(targetId, parts); + log.info("[async-forward] Dispatched {} part(s) to {} adapter (conv={}, target={})", + parts.size(), channelType, conversationId, targetId); + } catch (UnsupportedOperationException uoe) { + // Adapter doesn't override sendContentParts — fall through to + // text-only fallback. Most adapters that handle media (wecom / + // feishu / dingtalk) override; the rest will stay text-only + // until they implement the part dispatcher. + log.info("[async-forward] {} adapter does not implement sendContentParts, skipping (conv={})", + channelType, conversationId); + } catch (Exception e) { + log.warn("[async-forward] Failed to dispatch to {} adapter for conv={}: {}", + channelType, conversationId, e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelAdapter.java index 73f62d1b..5e43885b 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelAdapter.java @@ -95,6 +95,53 @@ public interface ChannelAdapter { sendMessage(targetId, content); } + /** + * Extended render-and-send overload that carries an optional + * {@link SendContext} side-channel (e.g. WeCom AI Bot + * {@code feedback.id} for like/dislike collection). + * + *

Default implementation ignores {@code ctx} and falls back to + * {@link #renderAndSend(String, String)}, so existing channel + * adapters and callers see no behavior change. Channels that want + * to consume {@code SendContext} fields override this overload. + * + *

This was introduced as part of PR-0 (RFC-32 §2.0.3) to give + * {@code ChannelMessageRouter} a way to thread the pre-allocated + * feedback id (registered against the persisted + * {@code mate_message.id}) down to the WeCom adapter without + * widening the legacy two-arg signature. + */ + default void renderAndSend(String targetId, String content, SendContext ctx) { + renderAndSend(targetId, content); + } + + /** + * Render and deliver an approval notice. Channels that support a + * native interactive surface (WeCom {@code button_interaction}, + * DingTalk {@code ActionCard}, etc.) override this to skip the + * text path entirely. + * + *

Primary implementation lives on + * {@link AbstractChannelAdapter}, which keeps the bytewise + * fallback (markdown text → {@link #sendMessage}). Adapters that + * inherit from {@code AbstractChannelAdapter} can call + * {@code super.sendApprovalNotice(...)} to fall back; the default + * here is just a safety net for adapters that, for some reason, + * implement {@link ChannelAdapter} directly. + * + *

Introduced in PR-0 (RFC-32 §2.0.3) so the router does not + * need to know which channel renders cards vs text: + *

+     *     ApprovalNotice notice = approvalNotificationService.buildNotice(pending);
+     *     adapter.sendApprovalNotice(replyTarget, notice);
+     * 
+ */ + default void sendApprovalNotice(String targetId, + vip.mate.channel.notification.ApprovalNotice notice) { + sendMessage(targetId, + vip.mate.channel.notification.ApprovalNotificationService.staticBuildText(notice)); + } + // ==================== 主动推送 ==================== /** @@ -152,6 +199,34 @@ public interface ChannelAdapter { return getChannelType(); } + /** + * Whether this adapter must run on exactly one node in a multi-instance + * deployment. + * + *

Return {@code true} when the underlying transport rejects multiple + * concurrent connections from the same credentials — e.g. a bot WebSocket + * gateway that enforces a per-app connection cap, or a long-polling + * endpoint where multiple consumers would steal updates from each other. + * The channel manager will gate {@link #start()} on a distributed lease + * so only one node connects at a time, and failover to another node when + * the lease holder dies. + * + *

Webhook-based channels (DingTalk, WeCom, Slack, …) should leave this + * at the default {@code false}: inbound HTTP traffic is fanned out by the + * load balancer, so every node may safely subscribe. + * + *

Scope: this hook is honored by the framework for DB-backed + * channels registered via {@code ChannelManager.startChannel}. For + * plugin-registered channels the framework can only gate the initial + * register attempt — there is no follower retry, no hot-swap, and no + * disable-detection (plugins have a register/unregister lifecycle, not + * a DB-driven one). Plugin authors needing full single-leader semantics + * should depend on {@code ChannelLeaderElection} directly. + */ + default boolean requiresSingleLeader() { + return false; + } + /** * RFC-024 Change 2:本 adapter 认为"多久没活动就视作 stale 需要重启"的阈值。 * diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java index b7805453..ca3bfffa 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java @@ -10,6 +10,8 @@ import org.springframework.stereotype.Component; import vip.mate.channel.dingtalk.DingTalkChannelAdapter; import vip.mate.channel.discord.DiscordChannelAdapter; import vip.mate.channel.feishu.FeishuChannelAdapter; +import vip.mate.channel.leader.ChannelLeaderElection; +import vip.mate.channel.leader.LeaderLease; import vip.mate.channel.model.ChannelEntity; import vip.mate.channel.qq.QQChannelAdapter; import vip.mate.channel.service.ChannelService; @@ -17,7 +19,9 @@ import vip.mate.channel.telegram.TelegramChannelAdapter; import vip.mate.channel.web.WebChannelAdapter; import vip.mate.channel.wecom.WeComChannelAdapter; import vip.mate.channel.weixin.WeixinChannelAdapter; +import vip.mate.exception.MateClawException; +import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.locks.ReadWriteLock; @@ -46,12 +50,91 @@ public class ChannelManager { private final ObjectMapper objectMapper; private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; + /** + * Approval notification renderer — used by WeCom adapter (PR-0 + * threading; PR-1 wired the WeCom override to render a + * {@code button_interaction} card via this service's card builder). + * Other adapters keep using the text path on + * {@link AbstractChannelAdapter}, which calls + * {@code ApprovalNotificationService.staticBuildText} so this + * field is currently consumed only by WeCom. + */ + private final vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService; + + /** + * WeCom interactive card dispatcher (PR-1). Drives the + * {@code button_interaction} approval card render + the inbound + * {@code template_card_event} routing. + */ + private final vip.mate.channel.wecom.cards.WeComCardDispatcher weComCardDispatcher; + + /** + * WeCom keepalive scheduler (PR-1). Refreshes the "🤔 思考中..." + * placeholder every 20s and force-finishes after 180s so long- + * running agent tasks don't lose their stream slot. + */ + private final vip.mate.channel.wecom.WeComKeepaliveScheduler weComKeepaliveScheduler; + + /** + * Distributed leader election. Channels whose adapter reports + * {@link ChannelAdapter#requiresSingleLeader()} are gated on a lease so + * only one node opens the upstream WebSocket / long-poll at a time. + */ + private final ChannelLeaderElection leaderElection; + /** 运行中的渠道适配器:channelId -> adapter */ private final Map activeAdapters = new HashMap<>(); /** 插件注册的渠道适配器:pluginName -> adapter */ private final Map pluginChannels = new ConcurrentHashMap<>(); + /** Held leadership leases for plugin channels: pluginName -> lease */ + private final Map pluginLeases = new ConcurrentHashMap<>(); + + /** Lease-extension futures for plugin channels: pluginName -> heartbeat */ + private final Map> pluginHeartbeatFutures = new ConcurrentHashMap<>(); + + /** + * Serializes register / unregister / shutdown / heartbeat-loss cleanup + * for plugin channels. The three plugin maps each use + * {@code ConcurrentHashMap}, which makes individual put/remove atomic + * — but not the multi-map sequences these paths run (snapshot, clear, + * stop adapter, release lease). Without this lock, a concurrent + * {@code registerPluginChannel} could insert into {@code pluginChannels} + * after {@code stopAll} snapshot-copies it but before + * {@code pluginChannels.clear()}, leaking an unstopped adapter and a + * never-released lease. + */ + private final Object pluginLifecycleLock = new Object(); + + /** Held leadership leases for leader-required channels: channelId -> lease */ + private final Map activeLeases = new HashMap<>(); + + /** Lease-extension futures: channelId -> heartbeat */ + private final Map> heartbeatFutures = new HashMap<>(); + + /** Follower retry futures: channelId -> retry */ + private final Map> followerRetryFutures = new HashMap<>(); + + /** + * Reconcile futures for non-leader-required active adapters + * (e.g. webhook-mode Feishu, polling-disabled Telegram). Without these, + * cross-node admin actions on a follower would never reach the running + * adapter on another node — only leaders run reconciliation through + * their heartbeat, so a webhook-running node would otherwise be deaf + * to disable / delete / config / mode-flip changes processed elsewhere. + */ + private final Map> reconcileFutures = new HashMap<>(); + + /** + * Last {@code update_time} the leader observed for each owned channel. + * Compared against the DB row on every heartbeat — a newer value means + * another node has applied a config change, and we (the connection + * holder) need to apply it locally too. Otherwise the leader keeps + * running with stale credentials/mode after admin edits. + */ + private final Map lastSeenChannelUpdateTime = new HashMap<>(); + /** 读写锁:读操作(getAdapter 等)用读锁,写操作(start/stop/replace)用写锁 */ private final ReadWriteLock adapterLock = new ReentrantReadWriteLock(); @@ -62,9 +145,34 @@ public class ChannelManager { return t; }); + /** + * Heartbeat + follower-retry scheduler. A small pool is enough — both + * tasks are short-lived (lock extend or a single startChannel attempt). + */ + private final ScheduledExecutorService leaderScheduler = Executors.newScheduledThreadPool(2, r -> { + Thread t = new Thread(r, "channel-leader"); + t.setDaemon(true); + return t; + }); + /** 旧 Adapter stop() 超时时间(秒) */ private static final int STOP_TIMEOUT_SECONDS = 5; + /** + * Lease heartbeat cadence. Must be well under + * {@link ChannelLeaderElection#LOCK_AT_MOST_FOR} (60s) so a single + * missed tick doesn't lose leadership; 20s gives us three chances per + * window. + */ + private static final long HEARTBEAT_INTERVAL_SECONDS = 20L; + + /** + * How often a follower retries to acquire leadership. Picked so a + * leader dying gets failed-over within (lease window + this interval) + * = ~90s in the worst case, without hammering the DB. + */ + private static final long FOLLOWER_RETRY_INTERVAL_SECONDS = 30L; + /** 支持的渠道类型 */ private static final Set SUPPORTED_TYPES = Set.of( "web", "dingtalk", "feishu", "telegram", "discord", "wecom", "qq", "weixin", "slack", "webchat" @@ -97,6 +205,7 @@ public class ChannelManager { public void destroy() { log.info("Shutting down ChannelManager, stopping {} active channels...", activeAdapters.size()); stopAll(); + leaderScheduler.shutdownNow(); stopExecutor.shutdownNow(); messageRouter.shutdown(); } @@ -115,9 +224,25 @@ public class ChannelManager { } ChannelAdapter adapter = createAdapter(channel); - adapter.start(); - activeAdapters.put(channel.getId(), adapter); - log.info("Channel started: {} (type={}, id={})", channel.getName(), channel.getChannelType(), channel.getId()); + if (adapter.requiresSingleLeader()) { + attemptLeaderStart(channel, adapter); + } else { + adapter.start(); + activeAdapters.put(channel.getId(), adapter); + lastSeenChannelUpdateTime.put(channel.getId(), channel.getUpdateTime()); + // A follower retry may still be scheduled if this channel was + // previously in leader-required mode and just flipped to a + // non-leader transport (e.g. Feishu websocket → webhook). + // Cancel it so we don't tick forever on a no-op startChannel. + cancelFollowerRetryLocked(channel.getId()); + // Schedule cross-node reconciliation for this non-leader adapter: + // followers driven by their retry tick re-read the channel, but + // a running non-leader has neither a heartbeat nor a retry, so + // without this ticker it would never notice admin actions + // processed on a different node. + scheduleReconcileLocked(channel.getId(), channel.getName()); + log.info("Channel started: {} (type={}, id={})", channel.getName(), channel.getChannelType(), channel.getId()); + } } finally { adapterLock.writeLock().unlock(); } @@ -128,9 +253,15 @@ public class ChannelManager { */ public void stopChannel(Long channelId) { ChannelAdapter oldAdapter; + LeaderLease lease; adapterLock.writeLock().lock(); try { oldAdapter = activeAdapters.remove(channelId); + lease = activeLeases.remove(channelId); + lastSeenChannelUpdateTime.remove(channelId); + cancelHeartbeatLocked(channelId); + cancelFollowerRetryLocked(channelId); + cancelReconcileLocked(channelId); } finally { adapterLock.writeLock().unlock(); } @@ -138,6 +269,371 @@ public class ChannelManager { if (oldAdapter != null) { stopAdapterSafely(oldAdapter, "stopChannel"); } + if (lease != null) { + try { + lease.release(); + log.info("[leader] Released lease for channel id={}", channelId); + } catch (Exception e) { + log.warn("[leader] Failed to release lease for channel id={}: {}", channelId, e.getMessage()); + } + } + } + + // ==================== Leader election ==================== + + /** + * Try to become leader for {@code channel} and start its adapter on + * success. On failure (another node holds the lease) schedule a + * follower retry so we'll take over when the current leader dies. + * + *

Caller must hold the adapter write lock. + */ + private void attemptLeaderStart(ChannelEntity channel, ChannelAdapter adapter) { + String key = channel.getChannelType() + ":" + channel.getId(); + Optional maybeLease = leaderElection.tryAcquire(key); + if (maybeLease.isEmpty()) { + log.info("[leader] Channel {} (id={}, type={}) is owned by another node — entering follower mode", + channel.getName(), channel.getId(), channel.getChannelType()); + scheduleFollowerRetryLocked(channel.getId()); + return; + } + + LeaderLease lease = maybeLease.get(); + try { + adapter.start(); + activeAdapters.put(channel.getId(), adapter); + activeLeases.put(channel.getId(), lease); + lastSeenChannelUpdateTime.put(channel.getId(), channel.getUpdateTime()); + scheduleHeartbeatLocked(channel.getId(), channel.getName()); + cancelFollowerRetryLocked(channel.getId()); + log.info("[leader] Channel started as leader: {} (type={}, id={})", + channel.getName(), channel.getChannelType(), channel.getId()); + } catch (Exception e) { + log.error("[leader] Adapter start failed after acquiring lease for channel {}: {} — releasing lease", + channel.getName(), e.getMessage(), e); + lease.release(); + throw e instanceof RuntimeException re ? re + : new RuntimeException("Channel start failed: " + e.getMessage(), e); + } + } + + /** + * Schedule periodic heartbeat to extend the lease. Caller must hold + * the adapter write lock. + */ + private void scheduleHeartbeatLocked(Long channelId, String channelName) { + cancelHeartbeatLocked(channelId); + ScheduledFuture f = leaderScheduler.scheduleAtFixedRate( + () -> heartbeat(channelId, channelName), + HEARTBEAT_INTERVAL_SECONDS, HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); + heartbeatFutures.put(channelId, f); + } + + /** + * One heartbeat tick. Extends the lease, then reconciles the local + * adapter against the current DB row — the leader is the only node + * holding the upstream connection, so it's the only one that can + * apply admin actions (disable / config change / delete) issued on + * a different node. Without this, e.g. a {@code /toggle?enabled=false} + * processed by a follower would never reach the actual connection. + */ + private void heartbeat(Long channelId, String channelName) { + LeaderLease lease; + adapterLock.readLock().lock(); + try { + lease = activeLeases.get(channelId); + } finally { + adapterLock.readLock().unlock(); + } + if (lease == null) { + return; + } + boolean stillOurs = lease.extend(ChannelLeaderElection.LOCK_AT_MOST_FOR); + if (!stillOurs) { + log.warn("[leader] Lost leadership for channel {} (id={}) — stopping local adapter and re-entering election", + channelName, channelId); + handleLeadershipLoss(channelId); + return; + } + reconcileChannel(channelId, channelName); + } + + /** + * Re-read the channel from the DB and apply any admin-side changes + * (disable, delete, config update) the leader hasn't seen yet because + * the API call was processed by a different node. + * + *

Package-private for unit testing — callers should rely on the + * heartbeat scheduler invoking this on its tick. + */ + void reconcileChannel(Long channelId, String channelName) { + ChannelEntity current; + try { + current = channelService.getChannel(channelId); + } catch (MateClawException e) { + if (e.getMsgKey() != null && e.getMsgKey().startsWith("err.channel.not_found")) { + log.info("[reconcile] Channel id={} no longer exists — stopping local adapter and releasing lease", + channelId); + stopChannel(channelId); + } else { + log.debug("[reconcile] Channel lookup failed for id={}: {}", channelId, e.getMessage()); + } + return; + } catch (Exception e) { + // Transient DB issue; skip this tick and try again on the next heartbeat. + log.debug("[reconcile] Channel lookup failed for id={}: {}", channelId, e.getMessage()); + return; + } + + if (!Boolean.TRUE.equals(current.getEnabled())) { + log.info("[reconcile] Channel {} (id={}) is now disabled — stopping local adapter", + channelName, channelId); + stopChannel(channelId); + return; + } + + LocalDateTime previousSeen; + adapterLock.readLock().lock(); + try { + previousSeen = lastSeenChannelUpdateTime.get(channelId); + } finally { + adapterLock.readLock().unlock(); + } + LocalDateTime currentUpdateTime = current.getUpdateTime(); + if (currentUpdateTime != null && previousSeen != null + && currentUpdateTime.isAfter(previousSeen)) { + log.info("[reconcile] Channel {} (id={}) config changed ({} → {})", + channelName, channelId, previousSeen, currentUpdateTime); + applyConfigChange(channelId, current); + } + } + + /** + * Apply a detected config change to a locally-running channel. + * + *

The fast path is the in-place swap that preserves the lease — + * but it is only valid when we are the current leader (we already + * hold {@code activeLeases[channelId]}). Without that gate, a + * non-leader node observing a {@code webhook → websocket} flip + * would call {@code newAdapter.start()} directly inside the swap + * and open a duplicate upstream connection, defeating the leader + * election. Every other transition — including + * {@code non-leader → leader-required}, {@code leader-required → + * non-leader}, and plain non-leader config updates — must go + * through {@code stopChannel} + {@code startChannel} so the lease + * is correctly released or acquired and follower retry is + * scheduled when election is lost. + * + *

Package-private for unit testing — see {@link #reconcileChannel}. + */ + void applyConfigChange(Long channelId, ChannelEntity newChannel) { + ChannelAdapter probe = createAdapter(newChannel); + boolean newRequiresLeader = probe.requiresSingleLeader(); + boolean weHaveLease; + adapterLock.readLock().lock(); + try { + weHaveLease = activeLeases.containsKey(channelId); + } finally { + adapterLock.readLock().unlock(); + } + + if (newRequiresLeader && weHaveLease) { + // Case A: same leader-required mode and we are the current leader + // — preserve the lease across the adapter swap. + swapAdapterPreservingLease(channelId, newChannel); + return; + } + + // All other cases: tear down local state and route through + // startChannel so leader election runs, lease is released, or both. + log.info("[reconcile] Channel {} (id={}) config change (newRequiresLeader={}, weHaveLease={}) — stop+start", + newChannel.getName(), channelId, newRequiresLeader, weHaveLease); + stopChannel(channelId); + try { + startChannel(newChannel); + } catch (Exception e) { + log.error("[reconcile] Restart after config change failed for channel {} (id={}): {}", + newChannel.getName(), channelId, e.getMessage(), e); + } + } + + /** + * Swap to a freshly-built adapter using the new config, while keeping + * the leadership lease and heartbeat in place. The lease is only + * released if the new adapter fails to start, in which case we fall + * back to follower mode so another node can try. + */ + private void swapAdapterPreservingLease(Long channelId, ChannelEntity newChannel) { + ChannelAdapter oldAdapter; + adapterLock.writeLock().lock(); + try { + oldAdapter = activeAdapters.remove(channelId); + } finally { + adapterLock.writeLock().unlock(); + } + if (oldAdapter != null) { + stopAdapterSafely(oldAdapter, "reconcile-swap"); + } + + ChannelAdapter newAdapter = createAdapter(newChannel); + boolean started = false; + Exception startError = null; + try { + newAdapter.start(); + started = true; + } catch (Exception e) { + startError = e; + } + + LeaderLease leaseToRelease = null; + adapterLock.writeLock().lock(); + try { + if (started) { + activeAdapters.put(channelId, newAdapter); + lastSeenChannelUpdateTime.put(channelId, newChannel.getUpdateTime()); + } else { + leaseToRelease = activeLeases.remove(channelId); + lastSeenChannelUpdateTime.remove(channelId); + cancelHeartbeatLocked(channelId); + scheduleFollowerRetryLocked(channelId); + } + } finally { + adapterLock.writeLock().unlock(); + } + + if (!started) { + log.error("[reconcile] New adapter start failed for channel {} (id={}): {} — released lease, entering follower mode", + newChannel.getName(), channelId, + startError != null ? startError.getMessage() : "unknown"); + if (leaseToRelease != null) { + leaseToRelease.release(); + } + } + } + + /** + * Drop the local adapter (without releasing the already-lost lease) + * and start follower retry so we'll attempt to reclaim leadership + * once the current owner stops renewing. + */ + private void handleLeadershipLoss(Long channelId) { + ChannelAdapter local; + adapterLock.writeLock().lock(); + try { + local = activeAdapters.remove(channelId); + activeLeases.remove(channelId); // already lost; do not call release() + cancelHeartbeatLocked(channelId); + scheduleFollowerRetryLocked(channelId); + } finally { + adapterLock.writeLock().unlock(); + } + if (local != null) { + stopAdapterSafely(local, "leadership-loss"); + } + } + + /** + * Schedule periodic follower retry. Caller must hold the adapter + * write lock. + */ + private void scheduleFollowerRetryLocked(Long channelId) { + if (followerRetryFutures.containsKey(channelId)) { + return; + } + ScheduledFuture f = leaderScheduler.scheduleAtFixedRate( + () -> followerRetry(channelId), + FOLLOWER_RETRY_INTERVAL_SECONDS, FOLLOWER_RETRY_INTERVAL_SECONDS, TimeUnit.SECONDS); + followerRetryFutures.put(channelId, f); + } + + /** + * One follower-retry tick. Re-reads the channel from the DB (it may + * have been disabled or deleted) and attempts to start it. The retry + * cancels itself once we successfully become leader. + */ + /** Package-private for unit testing — see {@link #reconcileChannel}. */ + void followerRetry(Long channelId) { + ChannelEntity current; + try { + current = channelService.getChannel(channelId); + } catch (MateClawException e) { + // Channel was deleted on another node — cancel the retry so we + // don't leak a scheduled task forever. Other exception codes + // (e.g. transient DB errors) fall through to the generic catch + // and let the retry continue. + if (e.getMsgKey() != null && e.getMsgKey().startsWith("err.channel.not_found")) { + log.info("[leader] Follower retry: channel id={} no longer exists — cancelling retry", channelId); + adapterLock.writeLock().lock(); + try { + cancelFollowerRetryLocked(channelId); + } finally { + adapterLock.writeLock().unlock(); + } + return; + } + log.debug("[leader] Follower retry: lookup failed for channel id={}: {}", channelId, e.getMessage()); + return; + } catch (Exception e) { + log.debug("[leader] Follower retry: lookup failed for channel id={}: {}", channelId, e.getMessage()); + return; + } + if (!Boolean.TRUE.equals(current.getEnabled())) { + adapterLock.writeLock().lock(); + try { + cancelFollowerRetryLocked(channelId); + } finally { + adapterLock.writeLock().unlock(); + } + return; + } + try { + startChannel(current); + } catch (Exception e) { + log.debug("[leader] Follower retry: startChannel failed for id={}: {}", channelId, e.getMessage()); + } + } + + /** Package-private for unit testing. */ + boolean hasFollowerRetry(Long channelId) { + adapterLock.readLock().lock(); + try { + return followerRetryFutures.containsKey(channelId); + } finally { + adapterLock.readLock().unlock(); + } + } + + private void cancelHeartbeatLocked(Long channelId) { + ScheduledFuture f = heartbeatFutures.remove(channelId); + if (f != null) { + f.cancel(false); + } + } + + private void cancelFollowerRetryLocked(Long channelId) { + ScheduledFuture f = followerRetryFutures.remove(channelId); + if (f != null) { + f.cancel(false); + } + } + + /** + * Schedule the cross-node reconcile ticker for a non-leader active + * adapter. Caller must hold the adapter write lock. + */ + private void scheduleReconcileLocked(Long channelId, String channelName) { + cancelReconcileLocked(channelId); + ScheduledFuture f = leaderScheduler.scheduleAtFixedRate( + () -> reconcileChannel(channelId, channelName), + FOLLOWER_RETRY_INTERVAL_SECONDS, FOLLOWER_RETRY_INTERVAL_SECONDS, TimeUnit.SECONDS); + reconcileFutures.put(channelId, f); + } + + private void cancelReconcileLocked(Long channelId) { + ScheduledFuture f = reconcileFutures.remove(channelId); + if (f != null) { + f.cancel(false); + } } /** @@ -161,6 +657,50 @@ public class ChannelManager { return; } + // Leader-required channels: if we don't already own the local adapter + // (i.e. we are a follower, or this is a brand-new channel), there is + // nothing to hot-swap. Fall through to startChannel which handles + // lease acquisition and follower retry. Hot-swap (which briefly opens + // a second upstream connection) is also avoided here so we don't + // double-occupy the bot's connection quota during a restart. + boolean weHaveAdapter; + boolean weHaveLease; + adapterLock.readLock().lock(); + try { + weHaveAdapter = activeAdapters.containsKey(channelId); + weHaveLease = activeLeases.containsKey(channelId); + } finally { + adapterLock.readLock().unlock(); + } + ChannelAdapter probe = createAdapter(channel); + if (probe.requiresSingleLeader()) { + log.info("[hot-swap] Channel {} requires single-leader; stop+start instead of hot-swap (weHaveAdapter={}, weHaveLease={})", + channel.getName(), weHaveAdapter, weHaveLease); + stopChannel(channelId); + startChannel(channel); + return; + } + // Mode flip: we currently hold a lease but the new config is no + // longer leader-required (e.g. Feishu WS → webhook). The lease, + // heartbeat, and lastSeenUpdateTime must all be torn down before + // the new non-leader adapter starts — the in-place hot-swap path + // below would leave them behind until the next heartbeat tick + // noticed and re-restarted, causing a redundant restart and a + // window where this node is silently holding a lease nobody else + // can grab. Stop+start handles all the cleanup in one shot. + if (weHaveLease) { + log.info("[hot-swap] Channel {} flipping leader-required → non-leader; stop+start to release lease", + channel.getName()); + stopChannel(channelId); + startChannel(channel); + return; + } + if (!weHaveAdapter) { + log.info("[hot-swap] No local adapter for channel {}, delegating to startChannel", channel.getName()); + startChannel(channel); + return; + } + log.info("[hot-swap] Starting hot-swap for channel: {} (type={}, id={})", channel.getName(), channel.getChannelType(), channelId); @@ -182,6 +722,10 @@ public class ChannelManager { adapterLock.writeLock().lock(); try { oldAdapter = activeAdapters.put(channelId, newAdapter); + // Mark the version we've now applied so the reconcile ticker + // (running for non-leader adapters) doesn't immediately fire a + // redundant swap on its next tick. + lastSeenChannelUpdateTime.put(channelId, channel.getUpdateTime()); log.info("[hot-swap] Adapter reference swapped for channel: {} (old={})", channel.getName(), oldAdapter != null ? "present" : "none"); } finally { @@ -203,17 +747,65 @@ public class ChannelManager { */ public void stopAll() { List adaptersToStop; + List leasesToRelease; + List pluginAdaptersToStop; + List pluginLeasesToRelease; adapterLock.writeLock().lock(); try { adaptersToStop = new ArrayList<>(activeAdapters.values()); + leasesToRelease = new ArrayList<>(activeLeases.values()); activeAdapters.clear(); + activeLeases.clear(); + lastSeenChannelUpdateTime.clear(); + heartbeatFutures.values().forEach(f -> f.cancel(false)); + heartbeatFutures.clear(); + followerRetryFutures.values().forEach(f -> f.cancel(false)); + followerRetryFutures.clear(); + reconcileFutures.values().forEach(f -> f.cancel(false)); + reconcileFutures.clear(); } finally { adapterLock.writeLock().unlock(); } + // Plugin channels live on a different map (keyed by pluginName), but + // shutdown must release their leases + cancel their heartbeats just + // like DB-backed ones. Without this, a plugin-supplied single-leader + // adapter would skip graceful release on @PreDestroy and its lease + // would stay locked until the lockAtMostFor window expired. + // + // Holding pluginLifecycleLock makes the snapshot+clear atomic + // against concurrent register / unregister / heartbeat-loss + // cleanup, so we don't leak an adapter that was registered after + // the snapshot but before the clear. + synchronized (pluginLifecycleLock) { + pluginAdaptersToStop = new ArrayList<>(pluginChannels.values()); + pluginChannels.clear(); + pluginLeasesToRelease = new ArrayList<>(pluginLeases.values()); + pluginLeases.clear(); + pluginHeartbeatFutures.values().forEach(f -> f.cancel(false)); + pluginHeartbeatFutures.clear(); + } + for (ChannelAdapter adapter : adaptersToStop) { stopAdapterSafely(adapter, "stopAll"); } + for (ChannelAdapter adapter : pluginAdaptersToStop) { + stopAdapterSafely(adapter, "stopAll-plugin"); + } + for (LeaderLease lease : leasesToRelease) { + try { + lease.release(); + } catch (Exception e) { + log.warn("[leader] stopAll: failed to release lease '{}': {}", lease.getName(), e.getMessage()); + } + } + for (LeaderLease lease : pluginLeasesToRelease) { + try { + lease.release(); + } catch (Exception e) { + log.warn("[leader] stopAll: failed to release plugin lease '{}': {}", lease.getName(), e.getMessage()); + } + } } // ==================== 查询(读锁保护) ==================== @@ -373,16 +965,42 @@ public class ChannelManager { /** * Register a channel adapter from a plugin. * + *

If the adapter reports {@link ChannelAdapter#requiresSingleLeader()}, + * the framework gates the local register on a distributed lease keyed by + * {@code plugin:{pluginName}}. When another node already owns the lease + * this node skips registration (its plugin instance is loaded but inert + * locally) — see the scope note on {@link ChannelAdapter#requiresSingleLeader()}. + * * @param pluginName the plugin name (used as key for unregistration) * @param adapter the channel adapter */ public void registerPluginChannel(String pluginName, ChannelAdapter adapter) { - try { - adapter.start(); - pluginChannels.put(pluginName, adapter); - log.info("Plugin channel registered: {} (type={})", pluginName, adapter.getChannelType()); - } catch (Exception e) { - log.error("Failed to start plugin channel {}: {}", pluginName, e.getMessage(), e); + synchronized (pluginLifecycleLock) { + LeaderLease lease = null; + if (adapter.requiresSingleLeader()) { + Optional maybeLease = leaderElection.tryAcquire("plugin:" + pluginName); + if (maybeLease.isEmpty()) { + log.info("[leader] Plugin channel {} (type={}) is owned by another node — skipping local registration", + pluginName, adapter.getChannelType()); + return; + } + lease = maybeLease.get(); + } + + try { + adapter.start(); + pluginChannels.put(pluginName, adapter); + if (lease != null) { + pluginLeases.put(pluginName, lease); + schedulePluginHeartbeatLocked(pluginName); + } + log.info("Plugin channel registered: {} (type={})", pluginName, adapter.getChannelType()); + } catch (Exception e) { + log.error("Failed to start plugin channel {}: {}", pluginName, e.getMessage(), e); + if (lease != null) { + lease.release(); + } + } } } @@ -390,11 +1008,78 @@ public class ChannelManager { * Unregister a plugin channel. */ public void unregisterPluginChannel(String pluginName) { - ChannelAdapter adapter = pluginChannels.remove(pluginName); + ChannelAdapter adapter; + ScheduledFuture heartbeat; + LeaderLease lease; + synchronized (pluginLifecycleLock) { + adapter = pluginChannels.remove(pluginName); + heartbeat = pluginHeartbeatFutures.remove(pluginName); + lease = pluginLeases.remove(pluginName); + } + if (heartbeat != null) { + heartbeat.cancel(false); + } if (adapter != null) { stopAdapterSafely(adapter, "unregisterPluginChannel"); log.info("Plugin channel unregistered: {}", pluginName); } + if (lease != null) { + try { + lease.release(); + } catch (Exception e) { + log.warn("[leader] Failed to release plugin lease '{}': {}", pluginName, e.getMessage()); + } + } + } + + /** Caller must hold {@link #pluginLifecycleLock}. */ + private void schedulePluginHeartbeatLocked(String pluginName) { + ScheduledFuture existing = pluginHeartbeatFutures.remove(pluginName); + if (existing != null) { + existing.cancel(false); + } + ScheduledFuture f = leaderScheduler.scheduleAtFixedRate( + () -> pluginHeartbeatTick(pluginName), + HEARTBEAT_INTERVAL_SECONDS, HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); + pluginHeartbeatFutures.put(pluginName, f); + } + + /** + * One plugin heartbeat tick. The tick runs on the scheduler thread, so + * it can race with {@code registerPluginChannel} / {@code unregisterPluginChannel} + * / {@code stopAll}. Serializing the loss-handler on + * {@link #pluginLifecycleLock} keeps the three maps consistent (no + * "stop ran twice" or "lease released after register re-acquired"). + */ + private void pluginHeartbeatTick(String pluginName) { + LeaderLease lease = pluginLeases.get(pluginName); + if (lease == null) { + return; + } + boolean stillOurs = lease.extend(ChannelLeaderElection.LOCK_AT_MOST_FOR); + if (stillOurs) { + return; + } + ChannelAdapter local; + ScheduledFuture self; + synchronized (pluginLifecycleLock) { + // Re-check under the lock — a concurrent unregister may have + // already torn everything down between the failed extend and + // our entering the locked region. + LeaderLease currentLease = pluginLeases.remove(pluginName); + if (currentLease == null) { + return; + } + local = pluginChannels.remove(pluginName); + self = pluginHeartbeatFutures.remove(pluginName); + } + log.warn("[leader] Lost plugin lease '{}' — unregistering local adapter", pluginName); + if (self != null) { + self.cancel(false); + } + if (local != null) { + stopAdapterSafely(local, "plugin-leadership-loss"); + } } // ==================== 内部方法 ==================== @@ -446,8 +1131,11 @@ public class ChannelManager { /** * 根据渠道实体创建对应的适配器实例 * 采用渠道注册表模式,根据类型创建对应适配器 + * + *

Package-private + non-final so unit tests can substitute a stub + * adapter without spinning up real WebSocket / HTTP clients. */ - private ChannelAdapter createAdapter(ChannelEntity channel) { + ChannelAdapter createAdapter(ChannelEntity channel) { String type = channel.getChannelType(); return switch (type) { case "web" -> new WebChannelAdapter(channel, messageRouter, objectMapper); @@ -455,7 +1143,9 @@ public class ChannelManager { case "feishu" -> new FeishuChannelAdapter(channel, messageRouter, objectMapper); case "telegram" -> new TelegramChannelAdapter(channel, messageRouter, objectMapper); case "discord" -> new DiscordChannelAdapter(channel, messageRouter, objectMapper); - case "wecom" -> new WeComChannelAdapter(channel, messageRouter, objectMapper); + case "wecom" -> new WeComChannelAdapter(channel, messageRouter, objectMapper, + approvalNotificationService, weComCardDispatcher, weComKeepaliveScheduler, + generatedFileCache); case "qq" -> new QQChannelAdapter(channel, messageRouter, objectMapper); case "weixin" -> new WeixinChannelAdapter(channel, messageRouter, objectMapper); case "slack" -> new vip.mate.channel.slack.SlackChannelAdapter(channel, messageRouter, objectMapper); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java index f1bc8772..549c4d70 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelMessageRouter.java @@ -1,6 +1,8 @@ package vip.mate.channel; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Component; import reactor.core.publisher.Flux; import vip.mate.agent.AgentService; @@ -8,6 +10,7 @@ import vip.mate.agent.context.ChatOrigin; import vip.mate.approval.ApprovalWorkflowService; import vip.mate.approval.ResolveOutcome; import vip.mate.approval.PendingApproval; +import vip.mate.channel.event.ChannelMessageReceivedEvent; import vip.mate.channel.model.ChannelEntity; import vip.mate.channel.notification.ApprovalNotificationService; import vip.mate.channel.service.ChannelService; @@ -59,6 +62,11 @@ public class ChannelMessageRouter { private final ChatStreamTracker streamTracker; private final ChannelChatOriginFactory chatOriginFactory; private final ChannelErrorClassifier errorClassifier; + /** Field-injected (rather than constructor) to avoid a signature + * change that would ripple through every test that constructs the + * router directly. Spring's stock publisher is always available. */ + @Autowired(required = false) + private ApplicationEventPublisher events; /** 队列条目:封装消息及其路由上下文 */ private record QueueEntry(ChannelMessage message, ChannelAdapter adapter, ChannelEntity channelEntity) {} @@ -88,8 +96,47 @@ public class ChannelMessageRouter { /** 每个渠道的队列容量 */ private static final int QUEUE_CAPACITY = 1000; - /** 防抖等待时间(毫秒) */ - private static final long DEBOUNCE_MS = 500; + /** 防抖等待时间(毫秒)。Package-private for unit-test access. */ + static final long DEBOUNCE_MS = 500; + + /** + * Extended debounce window for suspected paste-split scenarios. WeCom + * (and other IM clients) silently split a single pasted long prompt + * into 2-4 separate messages when it exceeds the per-frame limit + * (~2000 chars). The fragments arrive 0.5-2s apart, which means the + * default {@link #DEBOUNCE_MS} flushes the first fragment before the + * second one arrives — the agent then sees a torn context, calls the + * LLM on a partial prompt, and gets re-triggered when the next + * fragment lands. When merged content exceeds + * {@link #LONG_TEXT_THRESHOLD} we extend the window so the merger has + * time to absorb the rest. + *

+ * Package-private for unit-test access. + */ + static final long LONG_DEBOUNCE_MS = 2500; + + /** + * Content length (chars) above which we treat the message as a likely + * paste-split fragment. 1500 sits below the typical ~2000-char IM + * client split point while staying well above any normally-typed + * message, so the long-debounce path doesn't penalize ordinary + * chatting. A short typed "hello" still flushes in 500ms. + *

+ * Package-private for unit-test access. + */ + static final int LONG_TEXT_THRESHOLD = 1500; + + /** + * Pick the debounce window: extend to {@link #LONG_DEBOUNCE_MS} when + * either the new arrival or the accumulated merged buffer looks like + * a paste-split fragment, otherwise stay at {@link #DEBOUNCE_MS}. + *

+ * Package-private + static so tests can pin the threshold without + * spinning up the whole router (which has 12+ injected dependencies). + */ + static long pickDebounceMs(int currentMergedLength) { + return currentMergedLength > LONG_TEXT_THRESHOLD ? LONG_DEBOUNCE_MS : DEBOUNCE_MS; + } /** * Plan-Execute SSE events that the Web Console mirror needs to see when @@ -189,6 +236,14 @@ public class ChannelMessageRouter { * @param channelEntity 渠道配置(含关联 agentId) */ public void enqueue(ChannelMessage message, ChannelAdapter adapter, ChannelEntity channelEntity) { + // Fan out to the trigger pipeline FIRST — channel_message and + // content_match triggers fire on every received message regardless + // of whether the channel has an agent attached. If we returned + // early on a missing agent below without publishing, the workflow + // side would silently lose every channel-event that doesn't also + // route to a chat agent. + publishChannelEvent(message, adapter, channelEntity); + Long agentId = channelEntity.getAgentId(); if (agentId == null) { log.warn("Channel {} has no associated agent, ignoring message from {}", @@ -207,27 +262,101 @@ public class ChannelMessageRouter { log.info("[{}] Enqueuing message: sender={}, conversationId={}, agentId={}", channelType, message.getSenderId(), conversationId, agentId); - // 防抖:同一会话 500ms 内的连续消息合并 + // Debounce + adaptive merge: same conversation messages within the + // (500ms / 2.5s) window get concatenated into one. Adaptive: when + // the merged buffer crosses the LONG_TEXT_THRESHOLD we extend to + // LONG_DEBOUNCE_MS so paste-split fragments arrive together + // instead of triggering one agent call per piece. synchronized (pendingMessages) { PendingMessage existing = pendingMessages.get(conversationId); if (existing != null) { - // 合并到已有的 pending 消息 - if (existing.timer != null) { - existing.timer.cancel(false); + // Sender boundary in groups: when a different user sends to the + // same group within the debounce window, merging would attribute + // both fragments to whoever sent first — the LLM then loses the + // ability to tell who asked what. Flush the existing buffer + // immediately so each user's text rides its own pending window. + // Reentrant on `pendingMessages`, so the inner flushPending's + // synchronized block re-acquires safely on the same thread. + String existingSender = existing.firstMessage.getSenderId(); + String incomingSender = message.getSenderId(); + boolean sameSender = isSameSender(existingSender, incomingSender); + if (!sameSender) { + log.info("[{}] Sender boundary in conversation {}: flushing pending from sender={}, accepting new sender={}", + channelType, conversationId, existingSender, incomingSender); + if (existing.timer != null) { + existing.timer.cancel(false); + } + flushPending(conversationId); + // Fall through to create a fresh pending for the new sender. + } else { + // Same sender — original paste-split / rapid-follow merge path. + if (existing.timer != null) { + existing.timer.cancel(false); + } + existing.appendContent(message.getContent()); + int mergedLen = existing.getMergedContent().length(); + long debounceMs = pickDebounceMs(mergedLen); + existing.timer = debounceScheduler.schedule( + () -> flushPending(conversationId), debounceMs, TimeUnit.MILLISECONDS); + if (debounceMs > DEBOUNCE_MS) { + log.info("[{}] Long-text merger active: conversationId={}, mergedLen={}, debounce={}ms (paste-split suspected)", + channelType, conversationId, mergedLen, debounceMs); + } else { + log.debug("[{}] Message merged with pending (debounce {}ms): conversationId={}", + channelType, debounceMs, conversationId); + } + return; } - existing.appendContent(message.getContent()); - existing.timer = debounceScheduler.schedule( - () -> flushPending(conversationId), DEBOUNCE_MS, TimeUnit.MILLISECONDS); - log.debug("[{}] Message merged with pending (debounce): conversationId={}", - channelType, conversationId); - return; } - // 首条消息,创建 PendingMessage 并设定防抖定时器 + // 首条消息(或 sender boundary 之后的新 sender),创建 PendingMessage 并设定防抖定时器 PendingMessage pending = new PendingMessage(message, adapter, channelEntity); pendingMessages.put(conversationId, pending); + int firstLen = message.getContent() != null ? message.getContent().length() : 0; + long debounceMs = pickDebounceMs(firstLen); pending.timer = debounceScheduler.schedule( - () -> flushPending(conversationId), DEBOUNCE_MS, TimeUnit.MILLISECONDS); + () -> flushPending(conversationId), debounceMs, TimeUnit.MILLISECONDS); + if (debounceMs > DEBOUNCE_MS) { + log.info("[{}] Long-text merger armed on first message: conversationId={}, len={}, debounce={}ms", + channelType, conversationId, firstLen, debounceMs); + } + } + } + + /** + * Publish a {@link ChannelMessageReceivedEvent} so the trigger module's + * bridge can fan the message out to channel_message + content_match + * triggers. Best-effort — a publish failure must never block the + * primary chat-routing path. {@code messageId} is used as the dedup + * key downstream so repeated webhook deliveries can't double-fire + * the same trigger. + */ + private void publishChannelEvent(ChannelMessage message, ChannelAdapter adapter, + ChannelEntity channelEntity) { + if (events == null || message == null || adapter == null || channelEntity == null) return; + try { + long ws = channelEntity.getWorkspaceId() == null ? 0L : channelEntity.getWorkspaceId(); + String channelType = adapter.getChannelType(); + // messageId may be null for adapters that don't surface one; + // fall back to a sender+timestamp composite so the dedup key + // is at least deterministic-ish per webhook delivery. + String messageId = message.getMessageId(); + if (messageId == null || messageId.isBlank()) { + messageId = channelType + ":" + message.getSenderId() + ":" + + (message.getTimestamp() == null ? System.currentTimeMillis() + : message.getTimestamp()); + } + events.publishEvent(new ChannelMessageReceivedEvent( + ws, + channelType, + messageId, + message.getSenderId(), + message.getSenderName(), + message.getChatId(), + message.getContent())); + } catch (Exception e) { + log.warn("[ChannelMessageRouter] event publish failed for sender {}: {}", + message.getSenderId(), e.getMessage()); } } @@ -416,7 +545,10 @@ public class ChannelMessageRouter { ResolveOutcome denyOutcome = approvalService.resolve( pending.getPendingId(), message.getSenderId(), "denied"); conversationService.removeApprovalPlaceholders(conversationId); - adapter.sendMessage(replyTarget, "⛔ 已拒绝执行工具: " + pending.getToolName()); + String denyHint = "⛔ 已拒绝执行工具: " + pending.getToolName(); + persistAndBroadcastApprovalHint(conversationId, denyHint, + "denied", pending.getPendingId(), pending.getToolName()); + adapter.sendMessage(replyTarget, denyHint); log.info("[{}] Approval DENIED via IM command: pendingId={}, tool={}, msgRewritten={}", adapter.getChannelType(), pending.getPendingId(), pending.getToolName(), denyOutcome.messagesRewritten()); @@ -426,7 +558,10 @@ public class ChannelMessageRouter { // Non-approval message while a pending exists → treat as implicit deny. approvalService.resolve(pending.getPendingId(), message.getSenderId(), "denied"); conversationService.removeApprovalPlaceholders(conversationId); - adapter.sendMessage(replyTarget, "⛔ 审批已取消。将继续处理您的新消息。"); + String cancelHint = "⛔ 审批已取消。将继续处理您的新消息。"; + persistAndBroadcastApprovalHint(conversationId, cancelHint, + "cancelled", pending.getPendingId(), pending.getToolName()); + adapter.sendMessage(replyTarget, cancelHint); log.info("[{}] Approval auto-cancelled (non-approval message): pendingId={}", adapter.getChannelType(), pending.getPendingId()); // Fall through to process the new message normally. @@ -454,11 +589,20 @@ public class ChannelMessageRouter { } // 保存用户消息(带 contentParts) + // Group sender attribution: tag the persisted content + the + // prompt with [@sender] in groups so the LLM can disambiguate + // multiple users sharing one conversation. Single chats pass + // through unchanged (chatId is null). List parts = message.getContentParts(); - conversationService.saveMessage(conversationId, "user", message.getContent(), parts); + String attributedContent = applyGroupTag(message, message.getContent()); + conversationService.saveMessage(conversationId, "user", attributedContent, parts); // 构建 prompt(语音输入时注入场景提示词) String promptText = buildPromptFromParts(message.getContent(), parts, message.getInputMode()); + // Re-apply the tag in case the prompt was assembled from + // non-text parts (image/file) where buildPromptFromParts + // ignored `content`. Idempotent: skips when already prefixed. + promptText = applyGroupTag(message, promptText); // 注册到 ChatStreamTracker:让 graph 节点广播的事件(phase / content_delta / tool_call_* 等) // 能被 ChatConsole observer 订阅到。不注册 → broadcast() 会因 state==null 短路丢弃。 @@ -511,9 +655,13 @@ public class ChannelMessageRouter { // 检查 chat 过程中是否产生了审批 pending PendingApproval newPending = approvalService.findPendingByConversation(conversationId); if (newPending != null) { - // 有审批需求:不保存 LLM 的审批占位回复到 DB,直接从 pending 元数据构建通知 - String approvalNotice = buildApprovalNotice(newPending); - adapter.renderAndSend(replyTarget, approvalNotice); + // Channel-specific approval rendering: WeCom overrides + // sendApprovalNotice to post a button_interaction card; + // every other adapter falls back to the markdown-text path + // on AbstractChannelAdapter (preserves PR-0 behavior for + // non-WeCom channels). + var notice = approvalNotificationService.buildNotice(newPending); + adapter.sendApprovalNotice(replyTarget, notice); log.info("[{}] Approval triggered during chat, sent notice (NOT saved to DB): tool={}", adapter.getChannelType(), newPending.getToolName()); } else { @@ -641,7 +789,11 @@ public class ChannelMessageRouter { PendingApproval newPending = approvalService.findPendingByConversation(conversationId); if (newPending != null) { String replyTarget = resolveReplyTarget(message); - streamingAdapter.sendMessage(replyTarget, buildApprovalNotice(newPending)); + // Same polymorphic dispatch as the non-streaming path — WeCom + // renders a card, others render text. See the buildNotice + + // sendApprovalNotice pair at the non-streaming call site above. + var notice = approvalNotificationService.buildNotice(newPending); + streamingAdapter.sendApprovalNotice(replyTarget, notice); log.info("[{}] Approval triggered during streaming (NOT saved to DB): tool={}", channelType, newPending.getToolName()); } else if (finalContent != null && !finalContent.isBlank()) { @@ -702,8 +854,14 @@ public class ChannelMessageRouter { String replyTarget = resolveReplyTarget(triggerMessage); Long agentId = channelEntity.getAgentId(); - // 通知用户审批已通过 - adapter.sendMessage(replyTarget, "✅ 已批准执行工具: " + consumed.getToolName()); + // Notify the user that the approval went through. Persist + broadcast so a + // Web mirror of the same conversationId sees the resolution; otherwise this + // hint would only land in the IM channel and the Web admin console would + // show the replay reply with no preceding "approved" marker. + String approveHint = "✅ 已批准执行工具: " + consumed.getToolName(); + persistAndBroadcastApprovalHint(conversationId, approveHint, + "approved", consumed.getPendingId(), consumed.getToolName()); + adapter.sendMessage(replyTarget, approveHint); // 清理 DB 中残留的审批占位消息 conversationService.removeApprovalPlaceholders(conversationId); @@ -739,7 +897,62 @@ public class ChannelMessageRouter { adapter.getChannelType(), consumed.getToolName(), reply.length()); } catch (Exception e) { log.error("[approval-replay] Replay failed: {}", e.getMessage(), e); - adapter.sendMessage(replyTarget, "❌ 工具执行失败: " + e.getMessage()); + String errHint = "❌ 工具执行失败: " + e.getMessage(); + persistAndBroadcastApprovalHint(conversationId, errHint, null, null, null); + adapter.sendMessage(replyTarget, errHint); + } + } + + /** + * Persist an approval-related hint as an assistant message and best-effort + * broadcast it to any live SSE viewer of the conversation. + *

+ * Without this, IM-driven approve/deny only reaches the originating IM + * channel via {@code adapter.sendMessage(...)} — a Web mirror of the same + * conversationId has no record of the resolution because nothing lands in + * {@code mate_message} and no SSE event is emitted. The hint then "vanishes" + * from the Web admin console even though it shows up on the user's phone. + *

+ * Persistence is the load-bearing fix (Web reload picks it up). Broadcast + * is best-effort: if no SSE stream is currently registered for the + * conversation, the broadcast no-ops silently — that's the common case + * since IM-driven clicks rarely race with an active web subscriber. + * + * @param conversationId conversation owning the hint + * @param hint text to render as an assistant bubble + * @param decision "approved" / "denied" / "cancelled" / null (skips the + * structured resolved event when null, e.g. on replay error) + * @param pendingId pending approval id; null when not applicable + * @param toolName tool name for the structured event; null when not applicable + */ + private void persistAndBroadcastApprovalHint(String conversationId, String hint, + String decision, String pendingId, + String toolName) { + try { + conversationService.saveMessage(conversationId, "assistant", hint, null, "completed"); + } catch (Exception e) { + log.warn("[approval-hint] saveMessage failed for conv={}: {}", + conversationId, e.getMessage()); + } + try { + if (decision != null) { + streamTracker.broadcastObject(conversationId, "tool_approval_resolved", Map.of( + "pendingId", pendingId == null ? "" : pendingId, + "decision", decision, + "toolName", toolName == null ? "" : toolName, + "timestamp", System.currentTimeMillis() + )); + } + streamTracker.broadcastObject(conversationId, "message_start", + Map.of("role", "assistant")); + streamTracker.broadcastObject(conversationId, "content_delta", + Map.of("delta", hint)); + streamTracker.broadcastObject(conversationId, "message_complete", + Map.of("status", "completed")); + } catch (Exception e) { + // Broadcast is best-effort; a missing run state is the common case. + log.debug("[approval-hint] broadcast skipped/failed for conv={}: {}", + conversationId, e.getMessage()); } } @@ -776,9 +989,13 @@ public class ChannelMessageRouter { conversationService.getOrCreateConversation(conversationId, agentId, username, channelEntity.getWorkspaceId()); List parts = message.getContentParts(); - conversationService.saveMessage(conversationId, "user", message.getContent(), parts); + // Mirror processMessage's group attribution for the streaming path + // (Web channel today; future streaming IM channels inherit it). + String attributedContent = applyGroupTag(message, message.getContent()); + conversationService.saveMessage(conversationId, "user", attributedContent, parts); String promptText = buildPromptFromParts(message.getContent(), parts, message.getInputMode()); + promptText = applyGroupTag(message, promptText); // RFC-063r §2.5: forward ChatOrigin so tools created during this // streaming conversation inherit channel binding. ChatOrigin origin = chatOriginFactory.from( @@ -846,6 +1063,70 @@ public class ChannelMessageRouter { return message.getChannelType() + ":" + identifier; } + /** + * Build a sender-attribution tag for group messages. Returns + * {@code [@senderName]} when the message is from a multi-user channel + * context (chatId is set), else {@code null} for 1:1 chats. + * + *

Without this tag, three users asking three different questions in + * the same group conversation collapse into an unattributed wall of + * "user:" turns and the LLM can no longer tell who is asking what — + * it answers based on the most-recent text and ignores the rest. + * Single chats are unaffected because chatId is null there. + * + *

Prefer {@code senderName} when populated; otherwise fall back to + * {@code senderId}. WeCom currently sets both to the same opaque + * openid which is still useful for disambiguation; future channels + * (DingTalk, Slack) carry friendlier display names that flow through + * unchanged. + * + * @return sender tag like {@code [@Alice]}, or {@code null} if the + * message is not from a group context. + */ + static String buildGroupTag(ChannelMessage message) { + if (message == null) return null; + String chatId = message.getChatId(); + if (chatId == null || chatId.isBlank()) return null; + String name = (message.getSenderName() != null && !message.getSenderName().isBlank()) + ? message.getSenderName() : message.getSenderId(); + if (name == null || name.isBlank()) return null; + return "[@" + name + "]"; + } + + /** + * Apply {@link #buildGroupTag} to {@code content}. Idempotent: if + * {@code content} already starts with the tag (e.g. an upstream + * adapter has pre-attributed it), returns it unchanged so we don't + * double-stamp. No-op for single chats. + */ + static String applyGroupTag(ChannelMessage message, String content) { + String tag = buildGroupTag(message); + if (tag == null) return content; + // Empty content: leave empty rather than persist or prompt with a + // bare "[@Alice]" — the message had no payload to attribute. + if (content == null || content.isEmpty()) return content; + if (content.startsWith(tag)) return content; + return tag + " " + content; + } + + /** + * Decision helper for the debounce merger: should an incoming message + * from {@code incomingSender} merge into a pending buffer started by + * {@code existingSender}? True only when the senders match — different + * senders in the same conversation (a group context) must NOT merge, + * else the second user's text gets attributed to the first. + * + *

Null-handling: a null {@code existingSender} means "no buffer to + * merge into" so the answer is always false; a null + * {@code incomingSender} (rare, but seen in test fixtures) is also + * not allowed to silently merge — returning false routes to the + * "create new pending" branch which is safe. + */ + static boolean isSameSender(String existingSender, String incomingSender) { + if (existingSender == null || incomingSender == null) return false; + return existingSender.equals(incomingSender); + } + /** * 确定回复目标 * 优先使用 replyToken(渠道特有的回复标识),其次 chatId,最后 senderId diff --git a/mateclaw-server/src/main/java/vip/mate/channel/SendContext.java b/mateclaw-server/src/main/java/vip/mate/channel/SendContext.java new file mode 100644 index 00000000..d72b9ad8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/SendContext.java @@ -0,0 +1,63 @@ +package vip.mate.channel; + +import java.util.Map; + +/** + * Per-send context used to thread optional metadata from + * {@link ChannelMessageRouter} down into channel-specific renderers + * without having to expand {@code renderAndSend} signatures every time + * a new channel needs a side-channel value. + * + *

Currently used by: + *

    + *
  • {@code feedbackId} — pre-allocated by Router after the + * assistant message is persisted, so the WeCom AI Bot adapter + * can attach {@code feedback.id} to its final stream chunk for + * like/dislike collection. The id maps back to + * {@code (conversationId, savedMessageId, senderId)} in the + * channel's feedback registry.
  • + *
  • {@code savedMessageId} — the persisted assistant + * {@code mate_message.id}; used by feedback registry to bridge + * a feedback event back to the originating message.
  • + *
  • {@code extra} — open-ended map for future side-channel hints + * so we don't need yet another record field migration.
  • + *
+ * + *

Adapters that don't need any of this can ignore the parameter: + * the default {@code renderAndSend(targetId, content, ctx)} on + * {@link ChannelAdapter} delegates to the legacy two-arg version and + * drops {@code ctx} entirely. + * + * @param feedbackId optional like/dislike correlation id; null if + * the channel does not collect feedback or the + * assistant message could not be persisted + * @param savedMessageId persisted {@code mate_message.id} for the + * assistant reply; null in error / streaming + * passthrough paths where no row was created + * @param extra open-ended map; must never be null — use + * {@link #empty()} if no extras + */ +public record SendContext( + String feedbackId, + Long savedMessageId, + Map extra +) { + + public SendContext { + // Defensive: a null extra map breaks downstream get-or-default lookups. + if (extra == null) { + extra = Map.of(); + } + } + + private static final SendContext EMPTY = new SendContext(null, null, Map.of()); + + /** + * Reusable empty context. Adapters and callers that have no + * side-channel hints to thread should use this rather than + * allocating a fresh empty record per message. + */ + public static SendContext empty() { + return EMPTY; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/discord/DiscordChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/discord/DiscordChannelAdapter.java index 8fb17b6a..86658eea 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/discord/DiscordChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/discord/DiscordChannelAdapter.java @@ -373,6 +373,18 @@ public class DiscordChannelAdapter extends AbstractChannelAdapter { return CHANNEL_TYPE; } + /** + * Discord enforces a single Gateway session per bot token (any duplicate + * {@code IDENTIFY} closes the previous shard) and there is no active + * webhook fallback in this adapter — the legacy webhook endpoint is + * a no-op. The leader gate ensures only one node holds the Gateway + * connection at a time. + */ + @Override + public boolean requiresSingleLeader() { + return true; + } + // ==================== Webhook 兼容(保留接口,不再使用) ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/channel/event/ChannelMessageReceivedEvent.java b/mateclaw-server/src/main/java/vip/mate/channel/event/ChannelMessageReceivedEvent.java new file mode 100644 index 00000000..926855ec --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/event/ChannelMessageReceivedEvent.java @@ -0,0 +1,26 @@ +package vip.mate.channel.event; + +/** + * Spring application event fired when a channel adapter accepts an + * inbound message and hands it off to {@code ChannelMessageRouter}. + * The trigger module subscribes via {@code @EventListener} and forwards + * the payload through {@code TriggerEventIngestService} so triggers of + * pattern type {@code channel_message} or {@code content_match} can fan + * out to workflows. Going through the event bus instead of injecting + * the trigger service directly into the channel module keeps the two + * worlds decoupled and dodges the construction cycle. + * + *

{@code messageId} doubles as the dedup key — repeated webhook + * deliveries of the same message can't double-fire downstream triggers + * because the {@code mate_trigger_event} unique constraint catches the + * second insert. + */ +public record ChannelMessageReceivedEvent( + long workspaceId, + String channelType, + String messageId, + String senderId, + String senderName, + String chatId, + String content +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java index 602251bf..992937ed 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java @@ -1435,4 +1435,19 @@ public class FeishuChannelAdapter extends AbstractChannelAdapter { public String getChannelType() { return CHANNEL_TYPE; } + + /** + * WebSocket mode opens a long-lived connection to Lark's gateway, which + * caps concurrent connections per bot app (~2). In a multi-instance + * deployment every node would race for that quota and reconnect-loop on + * {@code 1000040350: the number of connections exceeded the limit}. + * The leader gate ensures only one node holds the connection at a time. + * + *

Webhook mode is exempt: callbacks are HTTP-fanned by the load + * balancer, so all nodes can safely subscribe. + */ + @Override + public boolean requiresSingleLeader() { + return "websocket".equals(getConfigString("connection_mode", "websocket")); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/channel/leader/ChannelLeaderElection.java b/mateclaw-server/src/main/java/vip/mate/channel/leader/ChannelLeaderElection.java new file mode 100644 index 00000000..8951cb08 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/leader/ChannelLeaderElection.java @@ -0,0 +1,84 @@ +package vip.mate.channel.leader; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockProvider; +import net.javacrumbs.shedlock.core.SimpleLock; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +/** + * Distributed leader election for channel adapters whose upstream + * service rejects multiple concurrent connections from the same bot + * credentials. + * + *

Typical examples are WebSocket-mode IM channels: Feishu's Lark + * SDK enforces a per-app connection cap (~2) and QQ's bot gateway + * rejects duplicate {@code IDENTIFY} sessions. Without coordination, + * every node of a multi-instance deployment trips that cap on startup + * and reconnects in a tight loop. + * + *

Backed by ShedLock's {@link LockProvider} (already wired for cron + * coordination), so single-node deployments incur no extra + * infrastructure — the lock is acquired trivially on the only node. + * + *

Semantics: + *

    + *
  • {@link #tryAcquire(String)} returns the lease, or empty if + * another node already holds it.
  • + *
  • The owning node must call {@link LeaderLease#extend(Duration)} + * on a fixed cadence (heartbeat) shorter than the + * {@code lockAtMostFor} window, or the lease expires and another + * node may claim it.
  • + *
  • If a node dies without releasing, the lease auto-expires after + * {@code lockAtMostFor}, after which any waiting follower can + * become leader on its next retry tick.
  • + *
+ */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ChannelLeaderElection { + + /** + * How long the lock is held without a renewal. A failed node's lease + * stays locked for this long before another node can take over — + * so longer values increase failover latency, shorter values increase + * the risk of false handover during a GC pause or DB hiccup. + */ + public static final Duration LOCK_AT_MOST_FOR = Duration.ofSeconds(60); + + private final LockProvider lockProvider; + + /** + * Attempt to acquire leadership for the given key. + * + * @param key a stable identifier for the resource (e.g. + * {@code "feishu:42"}). Used verbatim as the underlying + * lock name (prefixed by this class to avoid collisions + * with other lock users). + * @return an empty optional if another node already holds the lease, + * otherwise a {@link LeaderLease} that the caller is + * responsible for periodically extending and finally + * releasing. + */ + public Optional tryAcquire(String key) { + String lockName = "channel-leader:" + key; + LockConfiguration config = new LockConfiguration( + Instant.now(), + lockName, + LOCK_AT_MOST_FOR, + Duration.ZERO); + Optional lock = lockProvider.lock(config); + if (lock.isEmpty()) { + log.debug("[leader] Lock '{}' is held by another node", lockName); + return Optional.empty(); + } + log.info("[leader] Acquired lease '{}'", lockName); + return Optional.of(new LeaderLease(lockName, lock.get())); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/leader/LeaderLease.java b/mateclaw-server/src/main/java/vip/mate/channel/leader/LeaderLease.java new file mode 100644 index 00000000..d222fc1b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/leader/LeaderLease.java @@ -0,0 +1,77 @@ +package vip.mate.channel.leader; + +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.SimpleLock; + +import java.time.Duration; +import java.util.Optional; + +/** + * A held leadership lease on a single resource (typically a channel id). + * + *

Wraps a ShedLock {@link SimpleLock} so callers don't depend on the + * underlying lock provider. The lease must be periodically extended via + * {@link #extend(Duration)} or it expires automatically, at which point + * another node can claim leadership. + * + *

Threading: a single lease instance is not safe for concurrent + * {@link #extend(Duration)} / {@link #release()} calls. The owning + * scheduler is expected to serialize them. + */ +@Slf4j +public class LeaderLease { + + private final String name; + private volatile SimpleLock current; + private volatile boolean released; + + LeaderLease(String name, SimpleLock initial) { + this.name = name; + this.current = initial; + } + + public String getName() { + return name; + } + + /** + * Try to extend this lease for another {@code lockAtMostFor} window. + * + * @return true if the lease is still ours; false if it has been lost + * (e.g. the previous window expired before extend ran and + * another node acquired the lock — the caller should treat + * this as a leadership loss and stop the protected resource). + */ + public boolean extend(Duration lockAtMostFor) { + if (released) { + return false; + } + try { + Optional next = current.extend(lockAtMostFor, Duration.ZERO); + if (next.isPresent()) { + current = next.get(); + return true; + } + log.warn("[leader] Lease '{}' extend returned empty — lock lost", name); + return false; + } catch (Exception e) { + log.warn("[leader] Lease '{}' extend threw: {}", name, e.getMessage()); + return false; + } + } + + /** + * Release this lease. Idempotent — calling release twice is safe. + */ + public void release() { + if (released) { + return; + } + released = true; + try { + current.unlock(); + } catch (Exception e) { + log.warn("[leader] Lease '{}' unlock threw: {}", name, e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/notification/ApprovalNotificationService.java b/mateclaw-server/src/main/java/vip/mate/channel/notification/ApprovalNotificationService.java index 085a46e0..36e83d77 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/notification/ApprovalNotificationService.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/notification/ApprovalNotificationService.java @@ -57,29 +57,42 @@ public class ApprovalNotificationService { } /** - * 从 ApprovalNotice 构建文本 + * 从 ApprovalNotice 构建文本(实例方法,保留供历史调用方使用) */ public String buildApprovalText(ApprovalNotice notice) { + return staticBuildText(notice); + } + + /** + * Static text renderer — used by the {@code AbstractChannelAdapter} + * default {@code sendApprovalNotice} implementation, which has no + * Spring-managed reference to the service instance. + * + *

Logic is identical to {@link #buildApprovalText(ApprovalNotice)}; + * the instance method delegates here so the two paths can never + * drift. + */ + public static String staticBuildText(ApprovalNotice notice) { StringBuilder sb = new StringBuilder(); sb.append("🔐 **工具需要审批**\n\n"); sb.append("**工具名称**: ").append(notice.toolName()).append("\n"); - // 风险等级 + // Risk severity if (notice.maxSeverity() != null) { - sb.append("**风险等级**: ").append(severityLabel(notice.maxSeverity())).append("\n"); + sb.append("**风险等级**: ").append(staticSeverityLabel(notice.maxSeverity())).append("\n"); } - // 摘要 + // Summary if (notice.summary() != null && !notice.summary().isEmpty()) { sb.append("**摘要**: ").append(notice.summary()).append("\n"); } - // 参数预览 + // Args preview if (notice.argumentsPreview() != null && !notice.argumentsPreview().isEmpty()) { sb.append("**参数**: `").append(notice.argumentsPreview()).append("`\n"); } - // Findings 摘要(最多显示 3 条) + // Findings (top 3) if (notice.findings() != null && !notice.findings().isEmpty()) { sb.append("\n**发现的问题**:\n"); int shown = 0; @@ -100,6 +113,18 @@ public class ApprovalNotificationService { return sb.toString(); } + private static String staticSeverityLabel(String severity) { + if (severity == null) return ""; + return switch (severity) { + case "CRITICAL" -> "🔴 CRITICAL"; + case "HIGH" -> "🟠 HIGH"; + case "MEDIUM" -> "🟡 MEDIUM"; + case "LOW" -> "🔵 LOW"; + case "INFO" -> "⚪ INFO"; + default -> severity; + }; + } + /** * 构建 Web SSE 事件数据 */ diff --git a/mateclaw-server/src/main/java/vip/mate/channel/qq/QQChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/qq/QQChannelAdapter.java index bddde1ce..72bdc0b9 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/qq/QQChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/qq/QQChannelAdapter.java @@ -199,6 +199,17 @@ public class QQChannelAdapter extends AbstractChannelAdapter { return CHANNEL_TYPE; } + /** + * The QQ bot gateway rejects duplicate {@code IDENTIFY} sessions for + * the same app credentials. Multiple nodes connecting simultaneously + * trip the connection cap and reconnect-loop forever. The leader gate + * ensures only one node holds the WebSocket at a time. + */ + @Override + public boolean requiresSingleLeader() { + return true; + } + // ==================== Access Token 管理 ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/channel/slack/SlackChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/slack/SlackChannelAdapter.java index eb1b2737..7634cb5b 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/slack/SlackChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/slack/SlackChannelAdapter.java @@ -7,6 +7,7 @@ import com.slack.api.bolt.AppConfig; import com.slack.api.bolt.socket_mode.SocketModeApp; import com.slack.api.methods.SlackApiException; import com.slack.api.methods.response.chat.ChatPostMessageResponse; +import com.slack.api.methods.response.files.FilesUploadV2Response; import com.slack.api.model.event.MessageEvent; import lombok.extern.slf4j.Slf4j; import vip.mate.channel.AbstractChannelAdapter; @@ -14,8 +15,16 @@ import vip.mate.channel.ChannelMessage; import vip.mate.channel.ChannelMessageRouter; import vip.mate.channel.ExponentialBackoff; import vip.mate.channel.model.ChannelEntity; +import vip.mate.workspace.conversation.model.MessageContentPart; import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; import java.time.LocalDateTime; import java.util.List; import java.util.Map; @@ -295,4 +304,199 @@ public class SlackChannelAdapter extends AbstractChannelAdapter { result = result.replaceAll("(?m)^#{1,6}\\s+(.+)$", "*$1*"); return result; } + + /** + * Lazily-initialised JDK HTTP client for fetching media bytes from + * fully-qualified {@code fileUrl} fields. Only used when + * {@link MessageContentPart#getPath()} isn't set. + */ + private volatile HttpClient httpClient; + + private HttpClient getHttpClient() { + HttpClient hc = httpClient; + if (hc == null) { + synchronized (this) { + hc = httpClient; + if (hc == null) { + hc = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + httpClient = hc; + } + } + } + return hc; + } + + /** + * Dispatch a list of {@link MessageContentPart}s to Slack. Text parts + * fall through to the existing {@link #sendMessage} chat-post path; + * media parts (image / audio / video / file / model3d) ride + * {@code filesUploadV2} so users see a native file card with thumbnail + * preview rather than an unopenable markdown link. + *

+ * Wired by {@link vip.mate.channel.AsyncTaskMediaDispatcher} so async + * tool results (image generation / video generation / etc.) reach + * Slack channels the same way they reach WeCom / DingTalk / Feishu. + */ + @Override + public void sendContentParts(String targetId, List parts) { + if (parts == null || parts.isEmpty()) return; + + for (MessageContentPart part : parts) { + if (part == null) continue; + String type = part.getType(); + try { + switch (type == null ? "" : type) { + case "text", "thinking" -> { + String text = part.getText(); + if (text != null && !text.isBlank()) { + sendMessage(targetId, text); + } + } + case "refusal" -> { + String text = part.getText(); + if (text != null && !text.isBlank()) { + sendMessage(targetId, "⚠️ " + text); + } + } + case "image", "audio", "video", "file", "model3d" -> uploadFilePart(targetId, part); + default -> { + // Unknown part type — fall back to its text body if any. + if (part.getText() != null && !part.getText().isBlank()) { + sendMessage(targetId, part.getText()); + } + } + } + } catch (Exception e) { + log.warn("[slack] Failed to send {} part to {}: {}", type, targetId, e.getMessage()); + sendFallbackText(targetId, part); + } + } + } + + /** + * Upload a media part to Slack via {@code files.uploadV2}. Resolves + * bytes from the part's local {@code path} first (set by image / video + * / music / 3D generation services), falling back to an HTTP fetch of + * a fully-qualified {@code fileUrl}. Returns silently after sending a + * markdown fallback if no bytes are recoverable. + */ + private void uploadFilePart(String targetId, MessageContentPart part) { + byte[] bytes = resolveBytes(part); + if (bytes == null || bytes.length == 0) { + log.warn("[slack] No bytes resolvable for {} part (path={}, fileUrl={}), falling back to text", + part.getType(), part.getPath(), part.getFileUrl()); + sendFallbackText(targetId, part); + return; + } + + String botToken = getConfigString("bot_token"); + if (botToken == null || botToken.isBlank()) { + log.warn("[slack] Missing bot_token, cannot upload {} part", part.getType()); + return; + } + + String filename = part.getFileName(); + if (filename == null || filename.isBlank()) { + filename = defaultFilenameFor(part.getType()); + } + String threadTs = lookupThreadTs(targetId); + // Slack derives the MIME from the filename's extension; passing + // contentType here is unnecessary and not part of the V2 API. + final String finalFilename = filename; + final byte[] finalBytes = bytes; + try { + FilesUploadV2Response response = slack.methods(botToken).filesUploadV2(req -> { + var b = req + .channel(targetId) + .fileData(finalBytes) + .filename(finalFilename); + if (threadTs != null && !threadTs.isBlank()) { + b.threadTs(threadTs); + } + return b; + }); + if (!response.isOk()) { + log.warn("[slack] filesUploadV2 failed for {} ({}): {}", + finalFilename, finalBytes.length, response.getError()); + sendFallbackText(targetId, part); + return; + } + log.info("[slack] Uploaded {} part ({} bytes) to {}", part.getType(), finalBytes.length, targetId); + } catch (IOException | SlackApiException e) { + log.warn("[slack] filesUploadV2 error for {}: {}", finalFilename, e.getMessage()); + sendFallbackText(targetId, part); + } + } + + /** + * Read the part's bytes from disk first (paths set by the generation + * services + the WeCom inbound pipeline both populate this), or fetch + * via HTTP when only an external {@code fileUrl} is available. Returns + * null when neither path nor URL yields bytes. + */ + private byte[] resolveBytes(MessageContentPart part) { + String path = part.getPath(); + if (path != null && !path.isBlank()) { + try { + Path p = Path.of(path); + if (Files.exists(p)) { + return Files.readAllBytes(p); + } + } catch (Exception e) { + log.debug("[slack] Reading local path failed ({}): {}", path, e.getMessage()); + } + } + String url = part.getFileUrl(); + if (url != null && (url.startsWith("http://") || url.startsWith("https://"))) { + try { + HttpRequest req = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build(); + HttpResponse resp = getHttpClient().send(req, HttpResponse.BodyHandlers.ofByteArray()); + if (resp.statusCode() == 200) { + return resp.body(); + } + log.debug("[slack] HTTP fetch returned status {} for {}", resp.statusCode(), url); + } catch (Exception e) { + log.debug("[slack] HTTP fetch failed for {}: {}", url, e.getMessage()); + } + } + return null; + } + + private static String defaultFilenameFor(String type) { + return switch (type == null ? "" : type) { + case "image" -> "image.png"; + case "audio" -> "audio.mp3"; + case "video" -> "video.mp4"; + case "model3d" -> "model.glb"; + default -> "file.bin"; + }; + } + + /** Thread-aware reply: same lookup pattern as {@link #sendMessage}. */ + private String lookupThreadTs(String channelId) { + for (var entry : threadTsCache.entrySet()) { + if (entry.getKey().contains(channelId)) { + return entry.getValue(); + } + } + return null; + } + + /** Final fallback when both upload and resolve fail — keep the user + * informed instead of dropping the message silently. */ + private void sendFallbackText(String targetId, MessageContentPart part) { + String url = part.getFileUrl(); + String fileName = part.getFileName() != null ? part.getFileName() : part.getType(); + if (url != null && !url.isBlank()) { + sendMessage(targetId, "📎 " + fileName + ": " + url); + } else { + sendMessage(targetId, "[" + fileName + "]"); + } + } } diff --git a/mateclaw-server/src/main/java/vip/mate/channel/telegram/TelegramChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/telegram/TelegramChannelAdapter.java index 97cf420c..cce33ac4 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/telegram/TelegramChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/telegram/TelegramChannelAdapter.java @@ -165,8 +165,13 @@ public class TelegramChannelAdapter extends AbstractChannelAdapter { * - connection_mode=polling → Polling * - connection_mode 缺失 + webhook_url 非空 → Webhook(兼容旧配置) * - 其余 → Polling + * + *

Package-private so {@link #requiresSingleLeader()} can mirror the + * same predicate — the two answers must stay in lockstep, otherwise a + * single change to mode detection here would silently mis-classify the + * channel for multi-node coordination. */ - private boolean resolveWebhookMode() { + boolean resolveWebhookMode() { String connectionMode = getConfigString("connection_mode"); String webhookUrl = getConfigString("webhook_url"); boolean hasWebhookUrl = webhookUrl != null && !webhookUrl.isBlank(); @@ -794,6 +799,21 @@ public class TelegramChannelAdapter extends AbstractChannelAdapter { return CHANNEL_TYPE; } + /** + * Long-polling mode runs a {@code getUpdates(offset=…)} loop that + * acknowledges each delivered update; multiple nodes polling the same + * bot token would steal updates from each other (whichever node calls + * {@code getUpdates} next consumes the queue, and the others get + * nothing). The leader gate ensures only one node polls at a time. + * + *

Webhook mode is exempt: Telegram POSTs to a public URL fanned + * by the load balancer, so every node may safely receive callbacks. + */ + @Override + public boolean requiresSingleLeader() { + return !resolveWebhookMode(); + } + /** RFC-025 Change 4 入站文本净化上限(防止 caption 含超长二进制撑爆 prompt)。 */ private static final int INBOUND_TEXT_MAX = 4096; diff --git a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java index f31411f4..19c79358 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/web/ChatController.java @@ -1416,6 +1416,14 @@ public class ChatController { payload.put("status", status); if (savedAssistant != null && savedAssistant.getId() != null) { payload.put("assistantMessageId", savedAssistant.getId()); + // Surface runtime model attribution so the chat bubble can show + // which model produced this reply without waiting for a history reload. + if (savedAssistant.getRuntimeModel() != null && !savedAssistant.getRuntimeModel().isBlank()) { + payload.put("runtimeModel", savedAssistant.getRuntimeModel()); + } + if (savedAssistant.getRuntimeProvider() != null && !savedAssistant.getRuntimeProvider().isBlank()) { + payload.put("runtimeProvider", savedAssistant.getRuntimeProvider()); + } } if (promptTokens > 0) payload.put("promptTokens", promptTokens); if (completionTokens > 0) payload.put("completionTokens", completionTokens); @@ -1638,10 +1646,26 @@ public class ChatController { * having to guess from text. Empty string until the event arrives. */ private String finishReason = ""; + /** + * Recovery affordance payload from {@link + * vip.mate.agent.GraphEventPublisher#feedback}. Persisted into + * {@code metadata.feedbackEvent} so a page reload still surfaces + * the retry/regenerate/report card on the failed assistant + * bubble. Null when the turn ended cleanly. + */ + private Map feedbackEvent = null; private Long planId = null; private List planSteps = List.of(); private Integer currentPlanStep = null; private Map pendingApproval = null; + /** + * Multimodal sidecar routing decision for this turn (null when no + * routing happened). Captured from the {@code _routing_decision} + * event emitted before the graph stream and folded into + * {@code metadata.routing} on persistence so the chat UI can show + * which sidecar (if any) was invoked. + */ + private Map routingDecision = null; synchronized void accept(AgentService.StreamDelta delta, String conversationId) { if (delta == null) return; @@ -1675,6 +1699,22 @@ public class ChatController { finishReason = String.valueOf(reason); } } + if (vip.mate.agent.GraphEventPublisher.EVENT_FEEDBACK + .equals(delta.eventType())) { + // Snapshot the affordance payload so it persists into + // message metadata. The same event is also rebroadcast + // live (via the broadcastEvent fall-through below) so + // an already-mounted UI sees it instantly without + // waiting for the message-save round trip. + feedbackEvent = delta.eventData(); + } + if (vip.mate.agent.GraphEventPublisher.EVENT_ROUTING_DECISION.equals(delta.eventType())) { + // Captured at turn start; persisted under metadata.routing so the + // chat UI can render which sidecar (if any) was invoked. Internal + // event — return early to skip rebroadcast on IM channels. + routingDecision = delta.eventData(); + return; + } accumulateToolEvent(delta.eventType(), delta.eventData(), conversationId); try { broadcastEvent(conversationId, delta.eventType(), delta.eventData()); @@ -1959,6 +1999,18 @@ public class ChatController { // brittle text matching on the assistant content. metadata.put("finishReason", finishReason); } + if (feedbackEvent != null && !feedbackEvent.isEmpty()) { + // Persist the recovery-affordance payload so the + // retry/regenerate/report card survives page reload. + // Stored as-is (errorType, errorMessage, actions, + // timestamp) — frontend MessageBubble reads + // metadata.feedbackEvent and renders one button per + // entry in `actions`. + metadata.put("feedbackEvent", feedbackEvent); + } + if (routingDecision != null && !routingDecision.isEmpty()) { + metadata.put("routing", routingDecision); + } return objectMapper.writeValueAsString(metadata); } catch (Exception e) { log.warn("Failed to serialize metadata: {}", e.getMessage()); diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java index 2aefe81f..aa27f7f8 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComChannelAdapter.java @@ -12,6 +12,7 @@ import vip.mate.workspace.conversation.model.MessageContentPart; import javax.crypto.Cipher; import javax.crypto.spec.IvParameterSpec; import javax.crypto.spec.SecretKeySpec; +import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; @@ -26,6 +27,8 @@ import java.time.Duration; import java.time.LocalDateTime; import java.util.*; import java.util.concurrent.*; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -83,6 +86,13 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { private static final String CMD_HEARTBEAT = "ping"; private static final String CMD_RESPONSE = "aibot_respond_msg"; private static final String CMD_RESPONSE_WELCOME = "aibot_respond_welcome_msg"; + /** + * Update an interactive template card. Source-verified against the + * aibot SDK at {@code aibot/types.py:81} (RESPONSE_UPDATE constant) + * — used by {@link #updateTemplateCard} to replace a posted card + * within the 5-second window WeCom enforces after a button click. + */ + private static final String CMD_RESPONSE_UPDATE = "aibot_respond_update_msg"; private static final String CMD_SEND_MSG = "aibot_send_msg"; private static final String CMD_CALLBACK = "aibot_msg_callback"; private static final String CMD_EVENT_CALLBACK = "aibot_event_callback"; @@ -114,18 +124,67 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { /** 消息去重集合 */ private final Set processedMessageIds = ConcurrentHashMap.newKeySet(); - /** 回复 ACK 等待:reqId -> CompletableFuture */ + /** 回复 ACK 等待:reqId -> CompletableFuture(在 reqIdWorker 串行内 put,避免同 reqId 多次发送时撞 key) */ private final ConcurrentHashMap>> pendingAcks = new ConcurrentHashMap<>(); - /** 回复队列:reqId -> 串行队列(保证同一 reqId 的回复按序发送) */ - private final ConcurrentHashMap> replyQueues = new ConcurrentHashMap<>(); + /** + * Per-reqId 串行回复队列状态。Key=reqId,value 是 {@link ReplyQueueState} + * 包装的 (queue, closed) 二元组,用来在 idle-close 与 late-offer 之间提供 + * compute-bin-lock 级原子化(RFC-32 §2.4.1 a-2 / R-5 修正)。 + * + *

{@code closed} 是防御性标志位:worker 在 idle compute 退出时会把 entry + * 从 map 删掉,所以正常路径上 enqueue 看不到一个"closed=true 还在 map 里"的 + * state;保留这个标志为未来重构兜底,避免任何破坏"close = remove entry" + * 耦合的改动让 silent drop 复活。 + */ + private final ConcurrentHashMap replyQueues = new ConcurrentHashMap<>(); - /** 回复队列处理线程池 */ - private final ExecutorService replyExecutor = Executors.newCachedThreadPool(r -> { - Thread t = new Thread(r, "wecom-reply"); - t.setDaemon(true); - return t; - }); + /** + * 回复队列 worker 池。volatile + 非 final 是 RFC-32 §2.4.1 a-1 / R-2 修正 + * 的一部分:stop/重连时需要 {@link ExecutorService#shutdownNow()} 中断 + * worker 阻塞中的 {@code queue.poll(60s)},但 cached pool 一旦 shutdown + * 就不能重用,所以必须能在 {@link #ensureReplyExecutor()} 里重建。 + */ + private volatile ExecutorService replyExecutor; + + /** + * Lifecycle gate:控制 {@link #sendFrameWithAck} 是否接受新任务。 + * + *

必须由 transport-ready 信号 触发置 true(即认证成功后的 + * {@link #markReady()}),而不是 executor-ready({@link #ensureReplyExecutor()})。 + * 否则会出现"executor 活的、accepting=true、但 webSocket=null"的窗口—— + * worker 调 {@link #sendFrame} 看到 {@code webSocket==null} 就 warn 后默默 return, + * 让 caller 等 5s 假超时(RFC-32 §2.4.1 a-1 / R-7 修正)。 + */ + private final AtomicBoolean replyQueueAccepting = new AtomicBoolean(false); + + /** + * Idle-timeout (ms) for the per-reqId worker's {@code queue.poll}. + * Default 60s in production; tests in the same package may lower + * this to the millisecond range to surface idle-close vs late-offer + * races without waiting a real minute (RFC-32 §3.0 S-3 stress). + * + *

Package-private on purpose — not exposed via getter or + * setter; tests assign it directly. Production code never writes + * to this field. + */ + @SuppressWarnings("PackageVisibleField") + volatile long workerIdleTimeoutMs = 60_000L; + + /** + * Per-reqId 回复队列状态。 + * + * @param queue 串行回复任务队列 + * @param closed 防御性 closed 标志(详见 {@link #replyQueues} 注释) + */ + private record ReplyQueueState( + LinkedBlockingQueue queue, + AtomicBoolean closed + ) { + static ReplyQueueState fresh() { + return new ReplyQueueState(new LinkedBlockingQueue<>(), new AtomicBoolean(false)); + } + } /** WebSocket 消息碎片缓冲区 */ private final StringBuilder wsBuffer = new StringBuilder(); @@ -142,6 +201,33 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { /** 回复上下文:replyToken -> (frameReqId, processingStreamId),用于 sendContentParts 回写 */ private final ConcurrentHashMap replyContexts = new ConcurrentHashMap<>(); + /** + * Group-chat reply-slot fallback cache: {@code chatId → most recent + * inbound frameReqId from that group}. + *

+ * The WeCom AI Bot platform blocks {@code aibot_send_msg} in + * group chats — proactive pushes (cron summaries, + * image-generation completions, TTS audio, async-task forwards) must + * ride {@code aibot_respond_msg} bound to some prior frame + * id. Without this cache every group push silently failed: + * {@code sendMessageToChat} fell through to {@code aibot_send_msg}, + * the platform rejected it, the user saw nothing. + *

+ * Populated for group inbound frames only — single-chat + * {@code aibot_send_msg} still works, so we don't need a cached + * reqId there. {@link #pickGroupReplyReqId(String)} returns the + * cached id (or null when there's never been a group inbound), and + * the proactive send paths fall through to {@code aibot_send_msg} + * when null. + *

+ * Bounded LRU at {@link #LAST_CHAT_REQ_IDS_MAX_SIZE} via insertion- + * order eviction — long-lived bots in many groups don't unbounded-grow. + */ + private final ConcurrentHashMap lastChatReqIds = new ConcurrentHashMap<>(); + + /** Max chat-id entries to keep in {@link #lastChatReqIds} before evicting. */ + private static final int LAST_CHAT_REQ_IDS_MAX_SIZE = 1000; + private record WeComReplyContext(String frameReqId, String processingStreamId) {} /** @@ -153,10 +239,71 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { */ private final AtomicBoolean disconnectInflight = new AtomicBoolean(false); + /** + * Approval-notification renderer. Held for symmetry with other channel + * adapters; the WeCom override of {@link #sendApprovalNotice} delegates + * card rendering to {@link #cardDispatcher} but still uses this service + * to build the {@link vip.mate.channel.notification.ApprovalNotice} + * data carrier. Null-tolerant: if Spring DI fails (test contexts), the + * default text-approval fallback still works. + */ + @SuppressWarnings("unused") // consumed via card dispatcher's tool_guard kind in PR-1 + private final vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService; + + /** + * WeCom interactive-card dispatcher (PR-1). + * + *

Routes outbound approval notices to a {@code button_interaction} + * card via tool_guard renderer, and inbound {@code template_card_event} + * frames to the matching handler by task_id prefix. Null-tolerant for + * test contexts (the {@link #sendApprovalNotice} override falls back + * to the abstract-class text path when the dispatcher is missing). + */ + private final vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher; + + /** + * Refreshes the "🤔 思考中..." processing-stream chunk every 20s and + * force-finishes after 180s, so WeCom's server-side stream slot + * doesn't drop while a long-running agent task is still computing + * (RFC-32 §2.1.2 / R-7 / B-5). Null-tolerant: if missing (test DI + * gap), placeholder still appears once but is not refreshed. + */ + private final WeComKeepaliveScheduler keepaliveScheduler; + + /** + * In-memory cache of bytes generated by tools like + * {@code DocxRenderTool} / {@code PptxRenderTool}. The agent emits a + * {@code /api/v1/files/generated/{id}} URL referencing this cache; the + * channel layer resolves that URL back to bytes and uploads them as a + * native WeCom file message so the user actually receives a tappable + * document instead of an unopenable link. Null-tolerant: if missing + * (older constructor / test DI gap), URL stays inline as plain markdown + * which renders as a non-interactive link in the bubble. + */ + private final vip.mate.tool.document.GeneratedFileCache generatedFileCache; + public WeComChannelAdapter(ChannelEntity channelEntity, ChannelMessageRouter messageRouter, - ObjectMapper objectMapper) { + ObjectMapper objectMapper, + vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService, + vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher, + WeComKeepaliveScheduler keepaliveScheduler) { + this(channelEntity, messageRouter, objectMapper, approvalNotificationService, + cardDispatcher, keepaliveScheduler, null); + } + + public WeComChannelAdapter(ChannelEntity channelEntity, + ChannelMessageRouter messageRouter, + ObjectMapper objectMapper, + vip.mate.channel.notification.ApprovalNotificationService approvalNotificationService, + vip.mate.channel.wecom.cards.WeComCardDispatcher cardDispatcher, + WeComKeepaliveScheduler keepaliveScheduler, + vip.mate.tool.document.GeneratedFileCache generatedFileCache) { super(channelEntity, messageRouter, objectMapper); + this.approvalNotificationService = approvalNotificationService; + this.cardDispatcher = cardDispatcher; + this.keepaliveScheduler = keepaliveScheduler; + this.generatedFileCache = generatedFileCache; // Default to 8 bounded attempts (~4 minutes total at 2s..30s exponential) // so the UI eventually settles in ERROR instead of getting stuck in // RECONNECTING forever. User config still overrides (-1 = infinite). @@ -185,6 +332,11 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { .connectTimeout(Duration.ofSeconds(10)) .build(); + // Build the reply-queue worker pool BEFORE the WS handshake kicks off, so + // any inbound auth_succeed → markReady → openReplyQueue path finds a live + // executor to schedule against. The gate stays closed until markReady runs. + ensureReplyExecutor(); + connectWebSocket(botId, secret); log.info("[wecom] WeCom bot channel initialized: botId={}, maxReconnectAttempts={}", @@ -211,6 +363,11 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { .connectTimeout(Duration.ofSeconds(10)) .build(); + // Re-arm the reply-queue worker pool BEFORE attempting the new + // handshake. accepting flag stays false until the new connection's + // auth_succeed fires markReady → openReplyQueue. + ensureReplyExecutor(); + String botId = getConfigString("bot_id"); String secret = getConfigString("secret"); connectWebSocket(botId, secret); @@ -229,6 +386,16 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { * "Disable + Enable" did to recover. */ private void releaseConnectionResources(String reason) { + // ============================================================================ + // RFC-32 §2.4.1 a-3 / R-6 + R-8 修正:必须按 step 0~4 顺序,不是尾部追加。 + // step 0 (replyQueueAccepting=false) 必须在 ws.close()/wsThread.join() 之前; + // 否则在 ws teardown 期间还会有 keepalive / 最终回复 / proactiveSend 漏进 enqueue。 + // ============================================================================ + + // ---- Step 0:先关 lifecycle gate,让任何后续 sendFrameWithAck 立刻 fast-fail ---- + replyQueueAccepting.set(false); + + // ---- 现有的 ws/heartbeat teardown(功能未变;插在 step 0 之后、step 1 之前) ---- if (heartbeatFuture != null) { heartbeatFuture.cancel(false); heartbeatFuture = null; @@ -253,17 +420,173 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } wsThread = null; } - pendingAcks.forEach((k, f) -> - f.completeExceptionally(new RuntimeException("Channel " + reason))); - pendingAcks.clear(); + + // ---- Step 1:第一次 drain replyQueues ---- + // forEach 是 weakly-consistent 迭代器,可能错过 step 0 之前刚提交但还没出 compute + // 的 enqueue —— step 3 会再 drain 一次兜底。 + replyQueues.forEach((rid, state) -> { + state.closed().set(true); + ReplyTask t; + while ((t = state.queue().poll()) != null) { + if (!t.future().isDone()) { + t.future().completeExceptionally(new IllegalStateException("Channel " + reason)); + } + } + }); + + // ---- Step 2:shutdownNow 中断 worker 阻塞中的 poll(60s) + 拒绝后续 submit ---- + ExecutorService oldExecutor = this.replyExecutor; + if (oldExecutor != null) { + oldExecutor.shutdownNow(); + this.replyExecutor = null; + } + + // ---- Step 3:second drain,捕获 step 1 与 step 2 之间的窗口期残留 ---- + // 此刻 shutdownNow 已经把任何新 fresh state 的 worker 拒掉,drain 是它们唯一退路。 + replyQueues.forEach((rid, state) -> { + state.closed().set(true); + ReplyTask t; + while ((t = state.queue().poll()) != null) { + if (!t.future().isDone()) { + t.future().completeExceptionally(new IllegalStateException("Channel " + reason)); + } + } + }); replyQueues.clear(); + + // ---- Step 4:pendingAcks 残留 ---- + pendingAcks.forEach((k, f) -> { + if (!f.isDone()) { + f.completeExceptionally(new IllegalStateException("Channel " + reason)); + } + }); + pendingAcks.clear(); + + // ---- 其他 per-connection 状态 ---- pendingFrames.clear(); replyContexts.clear(); + streamLastContent.clear(); + if (keepaliveScheduler != null) { + keepaliveScheduler.shutdownAll(); + } missedPongCount.set(0); this.httpClient = null; } + // ==================================================================== + // RFC-32 §2.0.5 / §2.4.1 a-1: lifecycle gate plumbing + // ==================================================================== + + /** + * (Re)build the worker pool. Called from {@link #doStart()} and + * {@link #doReconnect()}. Does not touch the {@link #replyQueueAccepting} + * gate — that flag is controlled by the transport-ready signal + * ({@link #markReady()}). See §2.4.1 a-1 / R-7. + */ + private void ensureReplyExecutor() { + if (replyExecutor == null || replyExecutor.isShutdown()) { + replyExecutor = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "wecom-reply"); + t.setDaemon(true); + return t; + }); + } + } + + /** + * Open the {@link #replyQueueAccepting} lifecycle gate. Only + * called from {@link #markReady()} after auth_succeed. Until this + * runs, every {@link #sendFrameWithAck} call fast-fails the caller's + * future with {@link IllegalStateException}. + */ + private void openReplyQueue() { + replyQueueAccepting.set(true); + } + + /** + * Per-reqId serial worker. Started lazily by + * {@link #sendFrameWithAck} when a fresh {@link ReplyQueueState} is + * created. Exits when: + *

    + *
  • queue is idle for 60s and atomically closes via compute + * (so any concurrent late-offer is observed and we stay alive)
  • + *
  • {@code running} flips to false
  • + *
  • worker thread is interrupted (e.g. by + * {@link ExecutorService#shutdownNow()})
  • + *
+ * + *

The compute-based idle-close fixes the TOCTOU race called out + * in RFC-32 §2.4.1 a-2 / R-5: enqueue's {@code compute} and + * worker's idle-close {@code compute} share the same bin lock, + * so offer and remove never interleave on the same key. + */ + private void reqIdWorker(String reqId, ReplyQueueState state) { + while (running.get() && !Thread.currentThread().isInterrupted()) { + ReplyTask task; + try { + task = state.queue().poll(workerIdleTimeoutMs, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; // fall through to drainStateExceptionally + return + } + + if (task == null) { + // Atomic close — serialized against sendFrameWithAck.compute on + // the same reqId by ConcurrentHashMap's bin lock. + ReplyQueueState afterClose = replyQueues.compute(reqId, (k, current) -> { + if (current != state) return current; // (c) replaced — defensive exit + if (!current.queue().isEmpty()) return current; // (b) late offer — stay alive + current.closed().set(true); // (a) truly idle — close + return null; // (a) remove entry + }); + if (afterClose != state) return; // (a) or (c) — exit + continue; // (b) — keep going + } + + try { + pendingAcks.put(reqId, task.future()); + // orTimeout 5s 兜底,whenComplete 在完成时清 pendingAcks。 + // 用 (key, value) 双参 remove 避免误删后续 task 的注册。 + task.future().orTimeout(REPLY_ACK_TIMEOUT_MS, TimeUnit.MILLISECONDS) + .whenComplete((r, ex) -> pendingAcks.remove(reqId, task.future())); + sendFrame(task.frame()); + task.future().join(); // serialize: don't dequeue next until this is done + } catch (CompletionException ce) { + // join() 抛的是 orTimeout 注入的异常(典型:TimeoutException)—— + // task.future 已经 complete,无需手动 fail + log.debug("[wecom] reply task ACK failed for reqId={}: {}", reqId, ce.getCause()); + } catch (Exception e) { + // sendFrame 同步抛 → ACK 永远不会到 → 必须显式 fail,否则 caller future 永久 pending + if (!task.future().isDone()) { + task.future().completeExceptionally(e); + } + pendingAcks.remove(reqId, task.future()); + log.debug("[wecom] reply task send failed for reqId={}: {}", reqId, e.getMessage()); + } + } + + // running=false / interrupted: mark closed + drain leftover + state.closed().set(true); + drainStateExceptionally(reqId, state, "channel stopped"); + } + + /** + * Drain remaining tasks in a {@link ReplyQueueState} and best-effort + * remove the entry from {@link #replyQueues}. Used by worker exit + * paths (running=false / interrupt). For {@link #releaseConnectionResources} + * the drain is inlined (step 1 / step 3) to keep the ordering proof local. + */ + private void drainStateExceptionally(String reqId, ReplyQueueState state, String reason) { + ReplyTask t; + while ((t = state.queue().poll()) != null) { + if (!t.future().isDone()) { + t.future().completeExceptionally(new IllegalStateException(reason)); + } + } + replyQueues.remove(reqId, state); + } + // ==================== WebSocket 连接 ==================== /** @@ -354,6 +677,10 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { reconnectFuture = null; } disconnectInflight.set(false); + // RFC-32 §2.4.1 a-1 / R-7: only NOW does sendFrameWithAck start + // accepting tasks — auth_succeed has just been observed and the + // WS is the canonical "transport ready" anchor. + openReplyQueue(); } /** @@ -622,15 +949,9 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map imgBody = (Map) body.getOrDefault("image", Map.of()); String url = (String) imgBody.getOrDefault("url", ""); String aesKey = (String) imgBody.getOrDefault("aeskey", ""); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "image.jpg"); - if (localPath != null) { - contentParts.add(MessageContentPart.image(localPath, url)); - } else { - contentParts.add(MessageContentPart.image(url, url)); - } - } else if (!url.isBlank()) { - contentParts.add(MessageContentPart.image(url, url)); + String inboundConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + contentParts.add(buildInboundImagePart(url, aesKey, msgId, "image.jpg", inboundConvId)); } textContent = "[图片]"; } @@ -649,11 +970,23 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map fileBody = (Map) body.getOrDefault("file", Map.of()); String url = (String) fileBody.getOrDefault("url", ""); String aesKey = (String) fileBody.getOrDefault("aeskey", ""); - String filename = (String) fileBody.getOrDefault("filename", "file.bin"); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, filename); - if (localPath != null) { - contentParts.add(MessageContentPart.file(localPath, filename, null)); + // WeCom sometimes omits filename for forwarded files. Try a + // few fallback keys before giving up to "file.bin"; the + // magic-byte sniffer in downloadInboundMedia will fix the + // extension either way, but having something user-readable + // here keeps the bubble title meaningful. + String filename = (String) fileBody.getOrDefault("filename", + fileBody.getOrDefault("file_name", + fileBody.getOrDefault("name", "file.bin"))); + String fileConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + MessageContentPart filePart = buildInboundFilePart( + url, aesKey, msgId, filename, fileConvId); + contentParts.add(filePart); + // Surface the corrected filename (with proper extension) + // back into the [文件: X] text marker the agent sees. + if (filePart.getFileName() != null && !filePart.getFileName().isBlank()) { + filename = filePart.getFileName(); } } textContent = "[文件: " + filename + "]"; @@ -675,15 +1008,10 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map img = (Map) item.getOrDefault("image", Map.of()); String url = (String) img.getOrDefault("url", ""); String aesKey = (String) img.getOrDefault("aeskey", ""); - if (getConfigBoolean("media_download_enabled", true) && !url.isBlank()) { - String localPath = downloadAndDecryptMedia(url, aesKey, msgId, "mixed_image.jpg"); - if (localPath != null) { - contentParts.add(MessageContentPart.image(localPath, url)); - } else { - contentParts.add(MessageContentPart.image(url, url)); - } - } else if (!url.isBlank()) { - contentParts.add(MessageContentPart.image(url, url)); + String mixedConvId = inboundConversationId(senderId, chatId, chatType); + if (!url.isBlank()) { + contentParts.add(buildInboundImagePart( + url, aesKey, msgId, "mixed_image.jpg", mixedConvId)); } } else if ("voice".equals(itemType)) { Map v = (Map) item.getOrDefault("voice", Map.of()); @@ -699,12 +1027,66 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { contentParts.add(0, MessageContentPart.text(textContent)); } } + case "appmsg" -> { + // Forwarded WeCom complex messages: PDF / Word / Excel + // transfers, public-account article links, miniprogram + // cards. Without this branch every forwarded PDF / + // article / miniprogram fell into default and got + // silently dropped — bot looked dumb to users. + AppmsgContent appmsg = extractAppmsgContent(body, msgId, senderId, chatId, chatType); + if (appmsg.text() != null && !appmsg.text().isBlank()) { + textContent = appmsg.text(); + contentParts.add(0, MessageContentPart.text(textContent)); + } + contentParts.addAll(appmsg.attachedParts()); + } default -> { log.debug("[wecom] Ignoring unsupported message type: {}", msgType); return; } } + // Apply any quoted-message context. The "quote" field appears at + // body level alongside the new message regardless of outer + // msgtype — without parsing it, the agent only sees the user's + // current text and silently loses the conversational reference + // ("user quoted the bot's previous image and asked '什么意思'" + // arrives as bare "什么意思", agent goes off-topic). + QuoteContext quote = extractQuoteContext(body, msgId, senderId, chatId, chatType); + if (quote != null && !quote.isEmpty()) { + String prefixedText = quote.prefix() + + (textContent != null && !textContent.isBlank() ? textContent : ""); + textContent = prefixedText; + + // Find an existing text part (text/voice cases produce one) + // and overwrite it with the prefixed text. Also pull it to + // the front so agent reading order starts with quote prefix. + boolean updated = false; + for (int i = 0; i < contentParts.size(); i++) { + if ("text".equals(contentParts.get(i).getType())) { + contentParts.set(i, MessageContentPart.text(prefixedText)); + if (i != 0) { + MessageContentPart promoted = contentParts.remove(i); + contentParts.add(0, promoted); + } + updated = true; + break; + } + } + if (!updated) { + // image/file/mixed cases without a text part — insert one. + contentParts.add(0, MessageContentPart.text(prefixedText)); + } + // Quoted media (image/file the user referenced) goes right + // after the text prefix so the agent reads: + // prefix → quoted media → user's own media (if any) + if (!quote.attachedParts().isEmpty()) { + contentParts.addAll(1, quote.attachedParts()); + } + log.debug("[wecom] Applied quote context: prefixLen={}, attachedParts={}", + quote.prefix().length(), quote.attachedParts().size()); + } + if (contentParts.isEmpty()) { if (textContent != null && !textContent.isBlank()) { contentParts.add(MessageContentPart.text(textContent)); @@ -727,6 +1109,16 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { boolean isGroup = "group".equals(chatType); String effectiveChatId = isGroup ? chatId : null; + // Cache the group's most recent inbound frameReqId so future + // proactive pushes into this group can ride aibot_respond_msg + // (the AI bot platform blocks aibot_send_msg in groups — + // without this cache async-task completions and cron summaries + // silently fail to deliver). + if (isGroup && chatId != null && !chatId.isBlank() + && frameReqId != null && !frameReqId.isBlank()) { + rememberGroupReplyReqId(chatId, frameReqId); + } + // conversationId 格式:wecom:{userid} 或 wecom:group:{chatid} // 由 ChannelMessageRouter.buildConversationId() 根据 channelType + chatId/senderId 构建 @@ -754,6 +1146,19 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { String replyToken = isGroup ? chatId : senderId; replyContexts.put(replyToken, new WeComReplyContext(frameReqId, processingStreamId)); + // PR-1: launch keepalive for the processing stream so long-running agent + // tasks (>60s) keep their stream slot alive — without this, WeCom's + // server-side TTL drops the slot and the eventual real reply gets + // silently rejected. RFC-32 §2.1.2 / R-7 / B-5. + if (keepaliveScheduler != null + && processingStreamId != null && !processingStreamId.isBlank()) { + try { + keepaliveScheduler.start(this, frameReqId, processingStreamId, replyToken); + } catch (Exception e) { + log.debug("[wecom] keepalive start failed: {}", e.getMessage()); + } + } + log.info("[wecom] Received message: sender={}, chatType={}, msgType={}, textLen={}", senderId.length() > 20 ? senderId.substring(0, 20) : senderId, chatType, msgType, @@ -791,14 +1196,122 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { return; } + if ("template_card_event".equals(eventType)) { + handleTemplateCardEvent(frame, body, event); + return; + } + log.debug("[wecom] Ignoring event type: {}", eventType); } catch (Exception e) { log.error("[wecom] Failed to handle event callback: {}", e.getMessage(), e); } } + /** + * Route an inbound {@code template_card_event} (a button click on a + * card we previously sent) to the correct + * {@link vip.mate.channel.wecom.cards.WeComCardKind} based on the + * task_id prefix. Each card kind owns its own validation + + * resolved-state render + command-injection logic. + * + *

5-second window: WeCom requires the + * {@code aibot_respond_update_msg} for this event to be sent inside + * 5s. Handlers therefore run synchronously here; the heavy work + * (e.g. agent re-execution) is deferred to the router's normal + * processMessage path via {@link #injectSyntheticMessage}. + */ + @SuppressWarnings("unchecked") + private void handleTemplateCardEvent(Map frame, + Map body, + Map event) { + if (cardDispatcher == null) { + log.debug("[wecom] template_card_event ignored: dispatcher not wired"); + return; + } + Map tce = event.get("template_card_event") instanceof Map m + ? (Map) m + : (Map) event; // some firmware nests directly under event + String taskId = (String) tce.getOrDefault("task_id", ""); + if (taskId.isBlank()) { + log.debug("[wecom] template_card_event missing task_id, ignoring"); + return; + } + + var kindOpt = cardDispatcher.lookupByTaskId(taskId); + if (kindOpt.isEmpty()) { + log.warn("[wecom] No registered card kind matches task_id={}, ignoring", taskId); + return; + } + + Map fromBlock = body.get("from") instanceof Map fm + ? (Map) fm + : Map.of(); + try { + kindOpt.get().handler().handle(this, frame, tce, fromBlock); + } catch (Exception e) { + log.error("[wecom] template_card_event handler ({}) threw: {}", + kindOpt.get().name(), e.getMessage(), e); + } + } + // ==================== 消息发送 ==================== + /** + * Render an approval notice as a WeCom {@code button_interaction} + * card and post it via the active reply context, instead of the + * abstract-class default text path. + * + *

Falls back to {@code super.sendApprovalNotice} (markdown text) + * in three failure modes: + *

    + *
  1. No card dispatcher available (DI did not wire it — usually + * a test / hot-swap context)
  2. + *
  3. No active {@link WeComReplyContext} for {@code targetId} — + * proactive paths (cron without a recent inbound message) + * cannot post a card because WeCom AI Bots reject + * {@code aibot_send_msg + template_card}; fall back to text + * so the user still sees the approval
  4. + *
  5. {@link CardOversizedException} thrown by the renderer + * (button.key payload > 1024 bytes)
  6. + *
+ * + *

The card is sent via {@link #replyTemplateCard} bound to the + * inbound frame's {@code req_id} that + * {@link #handleMessageCallback} stashed in {@link #replyContexts}. + */ + @Override + public void sendApprovalNotice(String targetId, + vip.mate.channel.notification.ApprovalNotice notice) { + if (cardDispatcher == null) { + super.sendApprovalNotice(targetId, notice); + return; + } + WeComReplyContext ctx = replyContexts.get(targetId); + if (ctx == null || ctx.frameReqId() == null || ctx.frameReqId().isBlank()) { + // No bound reply context — fall back to text. Most common in + // proactive paths (cron-triggered approvals) which WeCom AI + // Bot rejects for cards anyway. + super.sendApprovalNotice(targetId, notice); + return; + } + var kindOpt = cardDispatcher.lookupByMessageType( + vip.mate.channel.wecom.cards.tool_guard.ToolGuardCardKindFactory.MESSAGE_TYPE); + if (kindOpt.isEmpty()) { + super.sendApprovalNotice(targetId, notice); + return; + } + try { + Map card = kindOpt.get().renderer().render(notice); + replyTemplateCard(ctx.frameReqId(), card); + } catch (vip.mate.channel.wecom.cards.CardOversizedException oversized) { + log.warn("[wecom] approval card oversized, falling back to text: {}", oversized.getMessage()); + super.sendApprovalNotice(targetId, notice); + } catch (Exception e) { + log.warn("[wecom] approval card render/send failed, falling back to text: {}", e.getMessage()); + super.sendApprovalNotice(targetId, notice); + } + } + @Override public void sendMessage(String targetId, String content) { if (webSocket == null) { @@ -814,27 +1327,96 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } /** - * 通过 WebSocket send_message 命令主动推送消息 + * 通过 WebSocket send_message 命令主动推送消息。 + *

+ * Group fallback: WeCom AI Bot platform rejects {@code aibot_send_msg} + * in group chats. When {@code chatId} matches a known group (via + * {@link #pickGroupReplyReqId}), ride a cached inbound reqId via + * {@code aibot_respond_msg} instead. Single chats still use + * {@code aibot_send_msg} (which the platform allows). */ private void sendMessageToChat(String chatId, String content) { if (webSocket == null || content == null || content.isBlank()) return; + Map textBody = Map.of( + "msgtype", "markdown", + "markdown", Map.of("content", content) + ); + sendOutboundFrame(chatId, textBody); + } + + /** + * Send a body via either {@code aibot_respond_msg} (groups, using a + * cached inbound reqId) or {@code aibot_send_msg} (single chats). + *

+ * Centralised so {@link #sendMessageToChat} (text) and + * {@link #sendMediaMessage} (image/file/voice/video) share the same + * group-vs-single dispatch — without this, every new outbound path + * had to remember the group rule, and several didn't. + */ + private void sendOutboundFrame(String chatId, Map bodyWithMsgtype) { + if (webSocket == null || chatId == null || chatId.isBlank()) return; try { - String reqId = generateReqId(CMD_SEND_MSG); - Map frame = Map.of( - "cmd", CMD_SEND_MSG, - "headers", Map.of("req_id", reqId), - "body", Map.of( - "chatid", chatId, - "msgtype", "markdown", - "markdown", Map.of("content", content) - ) - ); - sendFrameWithAck(reqId, frame); + String groupReplyReqId = pickGroupReplyReqId(chatId); + if (groupReplyReqId != null) { + // Group chat — ride aibot_respond_msg with the cached reqId. + // The body for respond_msg does NOT include "chatid" — the + // server infers the target from the original frame's reqId. + Map frame = Map.of( + "cmd", CMD_RESPONSE, + "headers", Map.of("req_id", groupReplyReqId), + "body", bodyWithMsgtype + ); + sendFrameWithAck(groupReplyReqId, frame); + log.debug("[wecom] Group send via aibot_respond_msg: chatId={}, reqId={}", + chatId, groupReplyReqId); + } else { + // Single chat — aibot_send_msg accepts a chatid field. + Map withChatId = new LinkedHashMap<>(bodyWithMsgtype); + withChatId.put("chatid", chatId); + String reqId = generateReqId(CMD_SEND_MSG); + Map frame = Map.of( + "cmd", CMD_SEND_MSG, + "headers", Map.of("req_id", reqId), + "body", withChatId + ); + sendFrameWithAck(reqId, frame); + } } catch (Exception e) { - log.error("[wecom] Failed to send message to {}: {}", chatId, e.getMessage(), e); + log.error("[wecom] Failed to send outbound frame to {}: {}", chatId, e.getMessage(), e); } } + /** + * Record the most recent inbound frameReqId for a group chat. Bounded + * LRU: when over {@link #LAST_CHAT_REQ_IDS_MAX_SIZE} we evict the + * oldest insertion-order key (ConcurrentHashMap iteration is + * insertion-order-ish for small maps and good enough — the cache + * exists to bound memory, not to be perfectly LRU). + */ + private void rememberGroupReplyReqId(String chatId, String frameReqId) { + lastChatReqIds.put(chatId, frameReqId); + while (lastChatReqIds.size() > LAST_CHAT_REQ_IDS_MAX_SIZE) { + // Pop one entry — any will do for memory bounding. + var it = lastChatReqIds.keySet().iterator(); + if (it.hasNext()) { + String victim = it.next(); + lastChatReqIds.remove(victim); + } else break; + } + } + + /** + * Look up the cached reply reqId for a group chat. Returns null when + * the chatId isn't a known group (single chats, or no inbound seen + * yet) — callers fall back to {@code aibot_send_msg}. + *

+ * Package-private for test access. + */ + String pickGroupReplyReqId(String chatId) { + if (chatId == null || chatId.isBlank()) return null; + return lastChatReqIds.get(chatId); + } + /** * 覆写 renderAndSend:如果有 processing_stream_id 则用 reply_stream 覆盖"思考中..." */ @@ -842,6 +1424,21 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { public void renderAndSend(String targetId, String content) { // 消费回复上下文(如果有的话) WeComReplyContext ctx = replyContexts.remove(targetId); + // Stop keepalive before we send the real reply: avoids racing the next + // refresh tick against this finish=true chunk on the same stream. + // No-op if force-finish already evicted the entry. + if (keepaliveScheduler != null && ctx != null && ctx.processingStreamId() != null) { + keepaliveScheduler.stop(ctx.processingStreamId()); + } + + // Sniff `/api/v1/files/generated/{id}` URLs out of the agent's text + // BEFORE rendering. Each hit gets upgraded to a native WeCom file + // message via the chunked upload API; the URL in the text is replaced + // with a "📎 filename" marker so the bubble doesn't repeat itself. + // Without this, a generated docx/pptx would arrive as a markdown link + // the user can't open inside WeCom (no public access + JWT required). + List uploadJobs = new ArrayList<>(); + String rewrittenContent = sniffGeneratedFiles(content, uploadJobs); // 先进行正常的内容渲染(过滤 thinking、分割长文本) boolean filterThinking = getConfigBoolean("filter_thinking", true); @@ -850,7 +1447,7 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { int maxLen = vip.mate.channel.ChannelMessageRenderer.PLATFORM_LIMITS.getOrDefault(getChannelType(), 2048); List segments = vip.mate.channel.ChannelMessageRenderer.renderForChannel( - content, filterThinking, filterToolMessages, format, maxLen); + rewrittenContent, filterThinking, filterToolMessages, format, maxLen); boolean first = true; for (String rawSegment : segments) { @@ -865,27 +1462,122 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { sendMessage(targetId, segment); } } + + // Upload + dispatch any generated files we sniffed out. Done after the + // text bubble so the order in the IM client mirrors the markdown: + // explanatory text first, then the actual file card the user can tap. + // Use the original frameReqId once for the first attachment (so it + // rides the reply path) and active-push for the rest. + if (!uploadJobs.isEmpty()) { + String frameReqId = ctx != null ? ctx.frameReqId() : null; + for (int i = 0; i < uploadJobs.size(); i++) { + UploadJob job = uploadJobs.get(i); + String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType()); + if (mediaId == null) { + log.warn("[wecom] Generated-file upload failed: {} ({} bytes)", + job.fileName(), job.bytes().length); + continue; + } + // Only the first attachment can use the inbound frameReqId + // reply slot; subsequent attachments must go via active-push. + String replyReqId = (i == 0) ? frameReqId : null; + sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId); + } + } + } + + /** Carries one to-be-uploaded generated file from {@link #sniffGeneratedFiles}. */ + private record UploadJob(byte[] bytes, String fileName, String mediaType) {} + + /** + * URL pattern for the in-memory generated-file cache served by + * {@code GeneratedFileController}. Lives in the channel layer because + * each adapter rewrites the URL to a channel-native attachment. + */ + private static final java.util.regex.Pattern GENERATED_URL_PATTERN = + java.util.regex.Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)"); + + /** + * Scan the agent's text for {@code /api/v1/files/generated/{id}} URLs; + * for each hit, look up the cached bytes and queue an {@link UploadJob}. + * Replaces the URL in the returned text with a "📎 filename" marker so + * the bubble shows the file name without dangling an unopenable link. + * Cache misses (entry expired or never existed) leave the URL untouched + * — the user can still try clicking from the Web mirror's history view. + */ + private String sniffGeneratedFiles(String text, List jobs) { + if (text == null || text.isEmpty() || generatedFileCache == null) return text; + java.util.regex.Matcher m = GENERATED_URL_PATTERN.matcher(text); + StringBuilder out = new StringBuilder(); + while (m.find()) { + String id = m.group(1); + var entry = generatedFileCache.get(id).orElse(null); + if (entry != null) { + String mediaType = isImageMime(entry.mimeType()) ? "image" : "file"; + jobs.add(new UploadJob(entry.bytes(), entry.filename(), mediaType)); + m.appendReplacement(out, + java.util.regex.Matcher.quoteReplacement("📎 " + entry.filename())); + } else { + // Cache miss has two real-world causes, both surfaced with + // the same retry hint so the user just resubmits: + // 1) LLM hallucinated a UUID-shaped string instead of + // calling a render tool — IDs like + // "a1b2c3d4-e5f6-7890-abcd-ef1234567890" with sequential + // hex are textbook fakes. {@code GeneratedFileCache} + // logs every real {@code put}, so its absence here is + // proof the file was never generated this turn. + // 2) Cache entry expired (10-min TTL) before the IM + // client got around to clicking, or was wiped on + // JVM restart. + // Without this replacement, users tap a markdown link that + // returns 404 and the IM client saves the error body as a + // ".docx" — they then "open" what is actually an HTML 404 + // page and report "file is corrupted". + log.warn("[wecom] Generated-file cache miss for id={} — likely LLM skipped the render tool and wrote a fake URL (toolCallCount=0 in this turn). Bubble will show retry hint.", + id); + m.appendReplacement(out, java.util.regex.Matcher.quoteReplacement( + "⚠️ 文件未真正生成(模型未调用文档生成工具),请重新发送请求")); + } + } + m.appendTail(out); + return out.toString(); + } + + private static boolean isImageMime(String mimeType) { + return mimeType != null && mimeType.toLowerCase().startsWith("image/"); } @Override public void sendContentParts(String targetId, List parts) { WeComReplyContext ctx = replyContexts.remove(targetId); + if (keepaliveScheduler != null && ctx != null && ctx.processingStreamId() != null) { + keepaliveScheduler.stop(ctx.processingStreamId()); + } boolean sentText = false; boolean firstText = true; + // Mirror renderAndSend's sniff so contentParts-mode replies also + // upgrade /api/v1/files/generated/{id} URLs into native WeCom file + // attachments. Text parts get the URL replaced with "📎 filename"; + // the actual bytes are queued for native upload after all parts are + // dispatched (preserves the "text first, attachments after" ordering). + List uploadJobs = new ArrayList<>(); + for (MessageContentPart part : parts) { if (part == null) continue; try { switch (part.getType()) { case "text" -> { - if (part.getText() != null && !part.getText().isBlank()) { + String txt = part.getText(); + if (txt != null && !txt.isBlank()) { + String rewritten = sniffGeneratedFiles(txt, uploadJobs); // 第一条文本用 processingStreamId 覆盖"思考中..." if (firstText && ctx != null && ctx.processingStreamId() != null && !ctx.processingStreamId().isBlank()) { - replyStream(ctx.frameReqId(), ctx.processingStreamId(), part.getText(), true); + replyStream(ctx.frameReqId(), ctx.processingStreamId(), rewritten, true); firstText = false; } else { - sendMessage(targetId, part.getText()); + sendMessage(targetId, rewritten); } sentText = true; } @@ -923,10 +1615,30 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { log.debug("[wecom] Failed to clear processing indicator: {}", e.getMessage()); } } + + // After all parts are dispatched, upload any generated files we + // sniffed out of text parts. First attachment rides the inbound + // frameReqId reply slot; subsequent ones go via active-push. Mirror + // the ordering used in renderAndSend so the user sees text bubble + // first, then the actual file card. + if (!uploadJobs.isEmpty()) { + String frameReqId = ctx != null ? ctx.frameReqId() : null; + for (int i = 0; i < uploadJobs.size(); i++) { + UploadJob job = uploadJobs.get(i); + String mediaId = uploadMedia(job.bytes(), job.fileName(), job.mediaType()); + if (mediaId == null) { + log.warn("[wecom] Generated-file upload failed: {} ({} bytes)", + job.fileName(), job.bytes().length); + continue; + } + String replyReqId = (i == 0) ? frameReqId : null; + sendMediaMessage(targetId, mediaId, job.mediaType(), replyReqId); + } + } } /** - * 发送图片部分:压缩 → 上传 → 发送 media_id + * 发送图片部分:压缩 → 大小预校验 → 上传 → 发送 media_id */ private void sendImagePart(String targetId, MessageContentPart part, WeComReplyContext ctx) { byte[] imageBytes = resolveFileBytes(part); @@ -938,17 +1650,29 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { String fileName = part.getFileName() != null ? part.getFileName() : "image.jpg"; imageBytes = WeComImageCompressor.compressIfNeeded(imageBytes, fileName); - String mediaId = uploadMedia(imageBytes, fileName, "image"); + WeComUploadLimitDecision decision = applyWeComUploadLimits( + imageBytes.length, "image", part.getContentType()); + if (decision.rejected()) { + log.warn("[wecom] Image upload rejected: {} ({} bytes) — {}", + fileName, imageBytes.length, decision.rejectReason()); + sendMessageToChat(targetId, "⚠️ " + decision.rejectReason()); + return; + } + + String mediaId = uploadMedia(imageBytes, fileName, decision.finalMediaType()); if (mediaId != null) { String frameReqId = ctx != null ? ctx.frameReqId() : null; - sendMediaMessage(targetId, mediaId, "image", frameReqId); + sendMediaMessage(targetId, mediaId, decision.finalMediaType(), frameReqId); + if (decision.downgraded()) { + sendMessageToChat(targetId, "ℹ️ " + decision.downgradeNote()); + } } else { sendFallbackText(targetId, part); } } /** - * 发送文件部分:上传 → 发送 media_id + * 发送文件部分:大小预校验 → 上传 → 发送 media_id */ private void sendFilePart(String targetId, MessageContentPart part, WeComReplyContext ctx) { byte[] fileBytes = resolveFileBytes(part); @@ -958,21 +1682,30 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } String fileName = part.getFileName() != null ? part.getFileName() : "file.bin"; - String mediaId = uploadMedia(fileBytes, fileName, "file"); + WeComUploadLimitDecision decision = applyWeComUploadLimits( + fileBytes.length, "file", part.getContentType()); + if (decision.rejected()) { + log.warn("[wecom] File upload rejected: {} ({} bytes) — {}", + fileName, fileBytes.length, decision.rejectReason()); + sendMessageToChat(targetId, "⚠️ " + decision.rejectReason()); + return; + } + + String mediaId = uploadMedia(fileBytes, fileName, decision.finalMediaType()); if (mediaId != null) { String frameReqId = ctx != null ? ctx.frameReqId() : null; - sendMediaMessage(targetId, mediaId, "file", frameReqId); + sendMediaMessage(targetId, mediaId, decision.finalMediaType(), frameReqId); } else { sendFallbackText(targetId, part); } } /** - * 发送音频部分:读取字节 → 上传 → 发送 + * 发送音频部分:读取字节 → 大小+格式预校验 → 上传 → 发送。 *

- * WeCom 原生语音消息要求 AMR 格式。TTS 输出为 MP3, - * Phase 1 以 file 类型发送(用户可点击播放),避免引入 AMR 转码依赖。 - * 非 AMR 格式走 file 类型而非 voice 类型,避免企微语音播放兼容问题。 + * WeCom 原生语音消息要求 AMR 格式 + ≤ 2 MB。预校验里非 AMR 或超 2 MB + * 自动降级为 file(文件卡片,可点击下载播放)+ 给用户一行说明 + * 提示——避免用户期待"语音气泡"但收到一个 .mp3 文件却不知道为啥。 */ private void sendAudioPart(String targetId, MessageContentPart part, WeComReplyContext ctx) { byte[] audioBytes = resolveFileBytes(part); @@ -983,15 +1716,30 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { String fileName = part.getFileName() != null ? part.getFileName() : "voice_reply.mp3"; boolean isAmr = fileName.toLowerCase().endsWith(".amr"); + // Pre-decide native voice vs file based on extension; the limits + // checker can still downgrade voice→file if size exceeds 2MB. + String requestedType = isAmr ? "voice" : "file"; + String contentTypeHint = part.getContentType(); + if (contentTypeHint == null && isAmr) contentTypeHint = "audio/amr"; - // AMR 格式:以原生 voice 类型发送(语音气泡) - // 其他格式(MP3 等):以 file 类型发送(文件卡片,可点击播放) - String uploadType = isAmr ? "voice" : "file"; - String mediaId = uploadMedia(audioBytes, fileName, uploadType); + WeComUploadLimitDecision decision = applyWeComUploadLimits( + audioBytes.length, requestedType, contentTypeHint); + if (decision.rejected()) { + log.warn("[wecom] Audio upload rejected: {} ({} bytes) — {}", + fileName, audioBytes.length, decision.rejectReason()); + sendMessageToChat(targetId, "⚠️ " + decision.rejectReason()); + return; + } + + String mediaId = uploadMedia(audioBytes, fileName, decision.finalMediaType()); if (mediaId != null) { String frameReqId = ctx != null ? ctx.frameReqId() : null; - sendMediaMessage(targetId, mediaId, uploadType, frameReqId); - log.info("[wecom] Audio sent as {}: {} ({}KB)", uploadType, fileName, audioBytes.length / 1024); + sendMediaMessage(targetId, mediaId, decision.finalMediaType(), frameReqId); + log.info("[wecom] Audio sent as {}: {} ({}KB)", + decision.finalMediaType(), fileName, audioBytes.length / 1024); + if (decision.downgraded()) { + sendMessageToChat(targetId, "ℹ️ " + decision.downgradeNote()); + } } else { sendFallbackText(targetId, part); } @@ -1063,10 +1811,51 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { * @param finish 是否结束流式消息 */ private void replyStream(String originalReqId, String streamId, String content, boolean finish) { + replyStream(originalReqId, streamId, content, finish, null); + } + + /** + * Streaming reply with optional WeCom feedback id attached on the + * final chunk (PR-2 hook installed in PR-0 so the protocol surface + * is stable). + * + *

Per WeCom AI Bot protocol (verified against the langbot + * reference implementation), {@code feedback.id} is only meaningful + * on the chunk where {@code finish=true}. We accept the parameter + * on every chunk for ergonomics but only emit the JSON field on + * the finishing chunk to avoid surfacing it where the server + * would ignore it. + * + *

Callers that don't need feedback collection pass {@code null} + * for {@code feedbackId} (or use the legacy 4-arg overload). + */ + private void replyStream(String originalReqId, String streamId, String content, + boolean finish, String feedbackId) { + // PR-1 chunk dedup: skip the network round-trip when a non-final chunk + // has the exact same content as the previous one for the same streamId. + // Tool-call argument streaming in particular emits many redundant chunks + // (each token re-flushes the partial JSON args) that would otherwise + // flicker the IM client. The final chunk (finish=true) ALWAYS goes + // through so WeCom closes the slot cleanly. RFC-32 §2.1.3. + if (!finish) { + String content_safe = content == null ? "" : content; + String prev = streamLastContent.get(streamId); + if (content_safe.equals(prev)) { + return; + } + streamLastContent.put(streamId, content_safe); + } else { + // Final chunk consumes the dedup slot. + streamLastContent.remove(streamId); + } + Map streamBody = new LinkedHashMap<>(); streamBody.put("id", streamId); streamBody.put("finish", finish); streamBody.put("content", content); + if (finish && feedbackId != null && !feedbackId.isBlank()) { + streamBody.put("feedback", Map.of("id", feedbackId)); + } Map body = Map.of( "msgtype", "stream", @@ -1082,6 +1871,112 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { sendFrameWithAck(originalReqId, frame); } + /** + * Per-streamId last-content cache for chunk dedup. Bounded only by + * the number of in-flight streams (a small handful in practice); + * cleared on each finish=true chunk and on connection release. + */ + private final ConcurrentHashMap streamLastContent = new ConcurrentHashMap<>(); + + // ==================== 上传大小预校验(WeCom 平台限制) ==================== + + /** WeCom hard limits — verified empirically; sources differ slightly. */ + static final long IMAGE_MAX_BYTES = 10L * 1024 * 1024; // 10 MB + static final long VIDEO_MAX_BYTES = 10L * 1024 * 1024; // 10 MB + static final long VOICE_MAX_BYTES = 2L * 1024 * 1024; // 2 MB + static final long FILE_MAX_BYTES = 20L * 1024 * 1024; // 20 MB (absolute cap) + static final Set VOICE_SUPPORTED_MIMES = Set.of("audio/amr"); + + /** + * Decision result for {@link #applyWeComUploadLimits}. + * + * @param finalMediaType effective media type after auto-downgrade + * (image / video / voice could become "file" + * when oversized or unsupported) + * @param rejected true → don't upload at all; surface + * {@code rejectReason} to the user instead + * @param rejectReason human-readable reason when {@code rejected} + * @param downgraded true → uploaded as {@code finalMediaType} + * but caller should append a notice to the + * bubble explaining why it's not native + * @param downgradeNote user-friendly note when {@code downgraded} + */ + record WeComUploadLimitDecision(String finalMediaType, boolean rejected, + String rejectReason, boolean downgraded, + String downgradeNote) { + static WeComUploadLimitDecision pass(String mediaType) { + return new WeComUploadLimitDecision(mediaType, false, null, false, null); + } + } + + /** + * Pre-validate an outbound upload against WeCom's per-media-type limits + * and decide whether to upload as-is, downgrade to {@code file}, or + * reject outright. Mirrors the strategy proven in production by + * comparable Java/Python WeCom integrations: + *

    + *
  • Files over 20 MB → reject — even {@code msgtype=file} can't + * carry them.
  • + *
  • Image > 10 MB → downgrade to file (still delivers, just as + * a tappable card instead of an inline preview).
  • + *
  • Video > 10 MB → downgrade to file.
  • + *
  • Voice with non-AMR mime type → downgrade to file (WeCom + * voice msgtype only accepts AMR).
  • + *
  • Voice in AMR but > 2 MB → downgrade to file.
  • + *
+ * Without this layer, oversized uploads would chunk-upload for up to + * a minute before the platform server rejected them at the finish + * step, with the user seeing nothing arrive in their chat. + *

+ * Package-private for unit-test access. + */ + static WeComUploadLimitDecision applyWeComUploadLimits(long fileSize, String mediaType, + String contentType) { + String type = mediaType == null ? "file" : mediaType.toLowerCase(); + String mime = contentType == null ? "" : contentType.toLowerCase().trim(); + + if (fileSize > FILE_MAX_BYTES) { + double mb = fileSize / 1024.0 / 1024.0; + return new WeComUploadLimitDecision( + type, true, + String.format(java.util.Locale.ROOT, + "文件大小 %.2fMB 超过企业微信 20MB 上限,无法发送。请压缩或拆分后再发。", mb), + false, null); + } + if ("image".equals(type) && fileSize > IMAGE_MAX_BYTES) { + double mb = fileSize / 1024.0 / 1024.0; + return new WeComUploadLimitDecision( + "file", false, null, + true, + String.format(java.util.Locale.ROOT, + "图片 %.2fMB 超过 10MB 限制,已转为文件形式发送", mb)); + } + if ("video".equals(type) && fileSize > VIDEO_MAX_BYTES) { + double mb = fileSize / 1024.0 / 1024.0; + return new WeComUploadLimitDecision( + "file", false, null, + true, + String.format(java.util.Locale.ROOT, + "视频 %.2fMB 超过 10MB 限制,已转为文件形式发送", mb)); + } + if ("voice".equals(type)) { + if (!mime.isEmpty() && !VOICE_SUPPORTED_MIMES.contains(mime)) { + return new WeComUploadLimitDecision( + "file", false, null, true, + "语音格式 " + mime + " 不支持(企微仅支持 AMR),已转为文件形式发送"); + } + if (fileSize > VOICE_MAX_BYTES) { + double mb = fileSize / 1024.0 / 1024.0; + return new WeComUploadLimitDecision( + "file", false, null, + true, + String.format(java.util.Locale.ROOT, + "语音 %.2fMB 超过 2MB 限制,已转为文件形式发送", mb)); + } + } + return WeComUploadLimitDecision.pass(type); + } + /** * 发送欢迎消息 */ @@ -1098,6 +1993,139 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { sendFrameWithAck(reqId, frame); } + /** + * Send an interactive template card (e.g. button_interaction approval card). + * + *

Wraps the card payload in {@code msgtype=template_card} and routes via + * the existing reply channel ({@code aibot_respond_msg}, bound to the inbound + * frame's req_id). Source-verified against aibot SDK + * {@code client.py:188-207 reply_template_card}. + * + *

Caller must have an active reply context for {@code reqId} — i.e. the + * card is sent in response to a previously received message frame, not as a + * proactive group push (which WeCom rejects for AI Bots, see RFC-32 G-12). + * + * @param reqId the original inbound frame's {@code headers.req_id} + * @param templateCard the WeCom template_card payload (card_type / task_id / + * main_title / button_list / etc.) + */ + public void replyTemplateCard(String reqId, Map templateCard) { + Map body = Map.of( + "msgtype", "template_card", + "template_card", templateCard + ); + Map frame = Map.of( + "cmd", CMD_RESPONSE, + "headers", Map.of("req_id", reqId), + "body", body + ); + sendFrameWithAck(reqId, frame); + } + + /** + * Update a previously-posted template card. Used by inbound + * {@code template_card_event} handlers (e.g. tool-guard approval) to swap + * the {@code button_interaction} card for a {@code text_notice} resolved + * state once the user clicks a button. + * + *

5-second window: per the aibot protocol, the response must be + * sent within 5s of receiving the {@code template_card_event} frame — + * otherwise the update is silently dropped. The handler path therefore + * has to validate identity + render the new card synchronously (fast + * DB lookup + map construction, well under 1ms) and only enqueue the + * inject-command on the agent thread afterwards. + * + *

Source-verified against aibot SDK {@code client.py:260-284 update_template_card}. + * + * @param eventReqId the inbound {@code template_card_event} frame's req_id + * (DIFFERENT from the original card-posting req_id) + * @param templateCard the replacement card payload (same task_id as the + * original card) + */ + public void updateTemplateCard(String eventReqId, Map templateCard) { + Map body = Map.of( + "response_type", "update_template_card", + "template_card", templateCard + ); + Map frame = Map.of( + "cmd", CMD_RESPONSE_UPDATE, + "headers", Map.of("req_id", eventReqId), + "body", body + ); + sendFrameWithAck(eventReqId, frame); + } + + /** + * Keepalive refresh tick (called by {@link WeComKeepaliveScheduler} + * every 20s). Sends {@code finish=false} on the existing stream so + * WeCom's server-side TTL counter resets. + * + *

Public so the scheduler in this same package can invoke it; the + * scheduler is itself a singleton bean and outside callers should + * not be triggering refresh ticks. + */ + public void replyStreamRefreshForKeepalive(String reqId, String streamId, String text) { + replyStream(reqId, streamId, text, false); + } + + /** + * Force-finish the keepalive stream (180s ceiling reached). Sends + * {@code finish=true} so WeCom closes the slot cleanly. The + * scheduler immediately follows this with + * {@link #invalidateReplyContext} so the eventual real reply takes + * the fresh-stream path. + */ + public void replyStreamFinishForKeepalive(String reqId, String streamId, String text) { + replyStream(reqId, streamId, text, true); + } + + /** + * Drop the {@link WeComReplyContext} entry for a {@code targetId} + * if (and only if) its current {@code processingStreamId} matches + * the supplied {@code streamId}. Idempotent and safe to call from + * any thread. + * + *

Used by {@link WeComKeepaliveScheduler} after force-finishing + * a stuck stream — RFC-32 §2.1.2 invariant: the next + * {@link #renderAndSend} call must NOT reuse a finished + * {@code processingStreamId}. + * + *

The match-and-remove uses {@link + * java.util.concurrent.ConcurrentHashMap#computeIfPresent} so a + * concurrent {@code renderAndSend} that already swapped the + * context for a fresh stream is left untouched. + */ + public void invalidateReplyContext(String targetId, String streamId) { + if (targetId == null || streamId == null) return; + replyContexts.computeIfPresent(targetId, (k, ctx) -> { + if (streamId.equals(ctx.processingStreamId())) { + log.debug("[wecom] invalidateReplyContext: cleared {} (stream={})", targetId, streamId); + return null; // remove entry + } + return ctx; + }); + } + + /** + * Route a synthetic message into the standard + * {@link ChannelMessageRouter} pipeline as if the user had typed it. + * + *

Bypasses {@link AbstractChannelAdapter#onMessage} so the + * pre-flight bot-prefix filter and access-control check are SKIPPED + * — appropriate for events that already represent an explicit user + * intent (e.g. a button click on an approval card). The router still + * runs its own approval validation in + * {@link ChannelMessageRouter#processMessage}, so the identity check + * for "only original requester can approve" still fires. + * + *

Currently used by tool-guard card handler. Package-private (no + * modifier) so only sibling classes in the wecom package can inject; + * external code must go through {@link ChannelAdapter#onMessage}. + */ + public void injectSyntheticMessage(ChannelMessage message) { + messageRouter.enqueue(message, this, channelEntity); + } + // ==================== 媒体上传协议 ==================== /** @@ -1118,6 +2146,18 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { if (webSocket == null || fileBytes == null || fileBytes.length == 0) { return null; } + // Pre-flight chunk count guard. WeCom's chunked upload protocol caps + // out near 100 chunks (~50 MB at 512 KB / chunk) but we already + // reject anything over FILE_MAX_BYTES (20 MB ≈ 40 chunks) before + // reaching here, so this is a defence-in-depth log line rather + // than a routine path. + int totalChunks = (int) Math.ceil((double) fileBytes.length / UPLOAD_CHUNK_SIZE); + if (totalChunks > 100) { + log.warn("[wecom] Upload would require {} chunks (>100 cap), rejecting: {} ({} bytes)", + totalChunks, fileName, fileBytes.length); + return null; + } + boolean acquired = false; try { acquired = uploadLock.tryAcquire(60, TimeUnit.SECONDS); @@ -1127,7 +2167,6 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } String md5 = md5Hex(fileBytes); - int totalChunks = (int) Math.ceil((double) fileBytes.length / UPLOAD_CHUNK_SIZE); // Phase 1: Init String initReqId = generateReqId(CMD_UPLOAD_INIT); @@ -1163,7 +2202,13 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { Map chunkBody = new LinkedHashMap<>(); chunkBody.put("upload_id", uploadId); chunkBody.put("chunk_index", i); - chunkBody.put("data", base64Data); + // Field name MUST be "base64_data" — the WeCom AI bot upload + // server reads the chunk bytes from this exact key. A previous + // version sent "data" which the server silently dropped, so + // metadata (filename/size) committed but the bytes never made + // it to storage. Receivers then saw the file with the right + // name/size but couldn't open it ("文件已损坏"). + chunkBody.put("base64_data", base64Data); Map chunkFrame = Map.of( "cmd", CMD_UPLOAD_CHUNK, @@ -1244,7 +2289,8 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { mediaBody.put(mediaType, Map.of("media_id", mediaId)); if (frameReqId != null && !frameReqId.isBlank()) { - // Reply 路径:使用 aibot_respond_msg + // Caller has an explicit inbound frameReqId (the message we're + // replying to is "now") — use that reply slot directly. Map frame = Map.of( "cmd", CMD_RESPONSE, "headers", Map.of("req_id", frameReqId), @@ -1252,15 +2298,12 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { ); sendFrameWithAck(frameReqId, frame); } else { - // 主动推送路径:使用 aibot_send_msg - mediaBody.put("chatid", targetId); - String reqId = generateReqId(CMD_SEND_MSG); - Map frame = Map.of( - "cmd", CMD_SEND_MSG, - "headers", Map.of("req_id", reqId), - "body", mediaBody - ); - sendFrameWithAck(reqId, frame); + // Proactive push (cron summary, async-task forward, etc.) — + // delegate to sendOutboundFrame so groups ride the cached + // reqId via aibot_respond_msg (the AI bot platform rejects + // aibot_send_msg in groups). Single chats fall through to + // aibot_send_msg as before. + sendOutboundFrame(targetId, mediaBody); } } @@ -1392,7 +2435,12 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { /** * 发送 WebSocket 帧(fire and forget) */ - private void sendFrame(Map frame) { + /** + * Visible to package-level tests (RFC-32 §3.0 S-1/S-2 stress) so a + * test subclass can override frame dispatch without monkey-patching + * the private WS field. Production callers stay within this class. + */ + void sendFrame(Map frame) { WebSocket ws = this.webSocket; if (ws == null) { log.warn("[wecom] WebSocket not connected, cannot send frame"); @@ -1407,33 +2455,741 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { } /** - * 串行队列发送帧,等待 ACK(带超时) - *

- * 同一 reqId 的消息按顺序发送,每条等待 ACK 后再发下一条。 + * Serially send a frame on the WS and wait (in a per-reqId worker) + * for its ACK. Same {@code reqId} messages are guaranteed to be + * dispatched in arrival order: the worker reads from the queue, + * registers {@link #pendingAcks} only after the previous ACK + * settled, sends, then blocks on the future until the ACK arrives + * or {@link #REPLY_ACK_TIMEOUT_MS} elapses. + * + *

RFC-32 §2.4.1 a-2 / R-5/R-6/R-7 invariants this implements: + *

    + *
  • Lifecycle gate: outer + inner check on + * {@link #replyQueueAccepting}. If closed, the returned + * future is fast-failed with {@link IllegalStateException} + * — never registered, never enqueued.
  • + *
  • TOCTOU between idle-close and late-offer: the + * offer happens INSIDE the {@code compute} lambda, sharing + * the bin lock with the worker's own {@code compute}-based + * idle-close. They serialize cleanly.
  • + *
  • Executor null/shutdown defense: re-checked inside + * compute; submit wrapped in try/catch for + * {@link RejectedExecutionException}.
  • + *
  • offered[] flag: any path that doesn't successfully + * offer falls through to {@code completeExceptionally}; no + * caller future ever hangs forever.
  • + *
+ * + *

Returns the ACK future for callers that want to chain on + * success (e.g. extract {@code body} fields from the ACK frame). + * Existing fire-and-forget callers can ignore the return value; + * timeout/error handling lives inside the worker. */ - private void sendFrameWithAck(String reqId, Map frame) { + @SuppressWarnings("UnusedReturnValue") + private CompletableFuture> sendFrameWithAck(String reqId, Map frame) { CompletableFuture> ackFuture = new CompletableFuture<>(); - // 注册 ACK 等待 - pendingAcks.put(reqId, ackFuture); + // ---- Outer lifecycle check (fast-fail, no allocations beyond the future) ---- + if (!replyQueueAccepting.get()) { + ackFuture.completeExceptionally( + new IllegalStateException("WeCom channel not accepting reply tasks (lifecycle gate closed)")); + return ackFuture; + } - // 发送帧 - sendFrame(frame); + ReplyTask task = new ReplyTask(frame, ackFuture); + boolean[] offered = {false}; - // 等待 ACK(超时 5 秒,不阻塞当前线程 — fire and forget) - ackFuture.orTimeout(REPLY_ACK_TIMEOUT_MS, TimeUnit.MILLISECONDS) - .whenComplete((result, ex) -> { - pendingAcks.remove(reqId); - if (ex != null) { - log.debug("[wecom] Reply ACK timeout or error for reqId={}: {}", reqId, ex.getMessage()); - } - }); + try { + replyQueues.compute(reqId, (k, existing) -> { + // ---- Inner lifecycle check: gate may have flipped between outer check and bin lock ---- + if (!replyQueueAccepting.get()) { + return existing; // do NOT modify map; offered[0] stays false → fail below + } + + // ---- Reuse open state if present ---- + if (existing != null && !existing.closed().get()) { + existing.queue().offer(task); + offered[0] = true; + return existing; + } + + // ---- Need to start a fresh state. Defend against late-shutdown ---- + ExecutorService exec = this.replyExecutor; + if (exec == null || exec.isShutdown()) { + return existing; // executor torn down by release; fail below + } + + ReplyQueueState fresh = ReplyQueueState.fresh(); + fresh.queue().offer(task); + try { + exec.submit(() -> reqIdWorker(k, fresh)); + offered[0] = true; + return fresh; + } catch (RejectedExecutionException ree) { + // Race with shutdownNow between isShutdown check and submit + return existing; // do not write fresh; fail below + } + }); + } catch (Exception e) { + // compute lambda surfaced something we didn't expect — never let this leak as + // a hung future + ackFuture.completeExceptionally(e); + return ackFuture; + } + + // ---- Final guarantee: any path that didn't offer must fail-fast ---- + if (!offered[0] && !ackFuture.isDone()) { + ackFuture.completeExceptionally( + new IllegalStateException("WeCom channel transitioning, reply task rejected")); + } + return ackFuture; } // ==================== 媒体文件下载与 AES 解密 ==================== /** - * 下载并解密企业微信媒体文件 + * Conversation id for inbound media: matches the format + * {@code ChannelMessageRouter.buildConversationId} produces from the same + * (channelType, chatId, senderId) tuple. Pre-computing it here lets the + * media-download helper write into the right per-conversation directory + * before the {@link ChannelMessage} is built. + * + *

The router's identifier is {@code chatId} when present (group context) + * and {@code senderId} otherwise (1:1) — there is no {@code group:} infix + * because the router doesn't add one. An earlier divergence here ({@code + * "wecom:group:" + chatId}) persisted media at + * {@code data/chat-uploads/wecom:group:{chatId}/} and stamped that path + * onto {@link MessageContentPart#getFileUrl()}, but the message's actual + * conversationId in {@code mate_conversation} was {@code wecom:{chatId}} + * (no {@code group:} infix). The {@code /api/v1/chat/files/...} endpoint + * then ran {@code isConversationOwner("wecom:group:...")}, found no + * matching row, returned 403, and the IM client rendered every + * group-quoted image as a broken icon. + */ + private static String inboundConversationId(String senderId, String chatId, String chatType) { + boolean isGroup = "group".equals(chatType); + return isGroup ? "wecom:" + chatId : "wecom:" + senderId; + } + + // ==================== 引用消息(quote)解析 ==================== + + /** + * Quoted-message extraction result: a human-readable prefix string the + * agent prompt prepends, plus any media (image / file) that was quoted + * and needs to be available as a content part. Either field may be + * empty; {@link #isEmpty()} returns true only when both are. + */ + private record QuoteContext(String prefix, List attachedParts) { + boolean isEmpty() { + return (prefix == null || prefix.isBlank()) && attachedParts.isEmpty(); + } + } + + /** + * Parse the {@code body.quote} field of an inbound WeCom AI Bot frame. + * Quote payloads sit alongside the new message at body level — + * independent of the outer {@code msgtype} — and may themselves carry + * any of text / voice / image / file / mixed. We flatten everything + * into: + *

    + *
  • a single {@code prefix} string of the form + * {@code "[引用消息: ]\n"} that prepends to the + * agent's user prompt
  • + *
  • a list of {@link MessageContentPart}s for any quoted media so + * the vision / document tools can analyse the actually-quoted + * image or PDF, not just see "[图片]" in the prefix
  • + *
+ * Returns null when {@code body.quote} is absent / empty / malformed — + * the caller treats that the same as "no quote context". + */ + private QuoteContext extractQuoteContext(Map body, String msgId, + String senderId, String chatId, String chatType) { + Object raw = body.get("quote"); + if (!(raw instanceof Map map)) return null; + @SuppressWarnings("unchecked") + Map quote = (Map) map; + String quoteType = (String) quote.getOrDefault("msgtype", ""); + if (quoteType == null || quoteType.isBlank()) return null; + + // Flatten: a "mixed" quote nests its own msg_item array; single-type + // quotes act as a one-element list of themselves. + List> items; + if ("mixed".equals(quoteType)) { + @SuppressWarnings("unchecked") + Map mixed = (Map) quote.getOrDefault("mixed", Map.of()); + @SuppressWarnings("unchecked") + List> mi = (List>) mixed.getOrDefault("msg_item", List.of()); + items = mi; + } else { + items = List.of(quote); + } + + String inboundConvId = inboundConversationId(senderId, chatId, chatType); + StringBuilder summary = new StringBuilder(); + List attached = new ArrayList<>(); + + for (Map item : items) { + String itemType = (String) item.getOrDefault("msgtype", ""); + switch (itemType == null ? "" : itemType) { + case "text" -> { + @SuppressWarnings("unchecked") + Map t = (Map) item.getOrDefault("text", Map.of()); + String content = ((String) t.getOrDefault("content", "")).trim(); + if (!content.isBlank()) { + appendQuoteSummary(summary, content); + } + } + case "voice" -> { + @SuppressWarnings("unchecked") + Map v = (Map) item.getOrDefault("voice", Map.of()); + String asr = ((String) v.getOrDefault("content", "")).trim(); + if (!asr.isBlank()) { + appendQuoteSummary(summary, "[语音] " + asr); + } else { + appendQuoteSummary(summary, "[语音消息]"); + } + } + case "image" -> { + @SuppressWarnings("unchecked") + Map img = (Map) item.getOrDefault("image", Map.of()); + String url = (String) img.getOrDefault("url", ""); + String aesKey = (String) img.getOrDefault("aeskey", ""); + if (!url.isBlank()) { + attached.add(buildInboundImagePart(url, aesKey, msgId, + "quoted_image.jpg", inboundConvId)); + } + appendQuoteSummary(summary, "[图片]"); + } + case "file" -> { + @SuppressWarnings("unchecked") + Map f = (Map) item.getOrDefault("file", Map.of()); + String url = (String) f.getOrDefault("url", ""); + String aesKey = (String) f.getOrDefault("aeskey", ""); + String filename = (String) f.getOrDefault("filename", + f.getOrDefault("file_name", f.getOrDefault("name", "file.bin"))); + if (!url.isBlank()) { + MessageContentPart part = buildInboundFilePart(url, aesKey, msgId, + filename, inboundConvId); + attached.add(part); + if (part.getFileName() != null && !part.getFileName().isBlank()) { + filename = part.getFileName(); + } + } + appendQuoteSummary(summary, "[文件: " + filename + "]"); + } + default -> { + // Unknown quote sub-type — surface the type tag so the + // agent knows something was quoted even if we can't + // unpack it. + if (itemType != null && !itemType.isBlank()) { + appendQuoteSummary(summary, "[" + itemType + "]"); + } + } + } + } + + if (summary.length() == 0 && attached.isEmpty()) { + return null; + } + String prefix = "[引用消息: " + summary + "]\n"; + return new QuoteContext(prefix, attached); + } + + private static void appendQuoteSummary(StringBuilder summary, String fragment) { + if (summary.length() > 0) summary.append(' '); + summary.append(fragment); + } + + // ==================== appmsg 解析(PDF/Word/Excel/链接/小程序)==================== + + /** + * Decoded {@code msgtype=appmsg} payload — text marker the agent reads + * + any attached media parts. Returned by {@link #extractAppmsgContent} + * so the inbound switch can splice it into {@code textContent} and + * {@code contentParts} uniformly. + */ + record AppmsgContent(String text, List attachedParts) {} + + /** + * Parse a {@code msgtype=appmsg} body into a text marker + media parts. + *

+ * Supports four variants observed in the WeCom platform: + *

    + *
  • {@code appmsg.file} — forwarded PDF / Word / Excel; reuses + * {@link #buildInboundFilePart} so magic-byte sniff + ZIP + * refinement work identically to plain {@code msgtype=file}
  • + *
  • {@code appmsg.image} — forwarded image card
  • + *
  • {@code appmsg.miniprogram} — miniprogram card; surface the + * title so the agent at least knows what was shared
  • + *
  • {@code appmsg.url} — public-account article / external link; + * flatten title + description + URL into text
  • + *
+ * Unknown variants fall back to a {@code [appmsg]} marker so the + * agent isn't completely blind. + *

+ * Package-private for unit-test access. + */ + AppmsgContent extractAppmsgContent(Map body, String msgId, + String senderId, String chatId, String chatType) { + @SuppressWarnings("unchecked") + Map appmsg = (Map) body.getOrDefault("appmsg", Map.of()); + String title = ((String) appmsg.getOrDefault("title", "")).trim(); + String desc = ((String) appmsg.getOrDefault("description", "")).trim(); + String linkUrl = ((String) appmsg.getOrDefault("url", "")).trim(); + String inboundConvId = inboundConversationId(senderId, chatId, chatType); + + Object fileObj = appmsg.get("file"); + Object imageObj = appmsg.get("image"); + Object miniObj = appmsg.get("miniprogram"); + + List attached = new ArrayList<>(); + StringBuilder text = new StringBuilder(); + + if (fileObj instanceof Map fileMap) { + @SuppressWarnings("unchecked") + Map fileBody = (Map) fileMap; + String fileUrl = (String) fileBody.getOrDefault("url", ""); + String aesKey = (String) fileBody.getOrDefault("aeskey", ""); + String filename = (String) fileBody.getOrDefault("filename", + fileBody.getOrDefault("file_name", + fileBody.getOrDefault("name", + title.isBlank() ? "file.bin" : title))); + if (!fileUrl.isBlank()) { + MessageContentPart filePart = buildInboundFilePart( + fileUrl, aesKey, msgId, filename, inboundConvId); + attached.add(filePart); + if (filePart.getFileName() != null && !filePart.getFileName().isBlank()) { + filename = filePart.getFileName(); + } + } + text.append("[文件: ").append(filename).append("]"); + } else if (imageObj instanceof Map imgMap) { + @SuppressWarnings("unchecked") + Map imgBody = (Map) imgMap; + String imgUrl = (String) imgBody.getOrDefault("url", ""); + String aesKey = (String) imgBody.getOrDefault("aeskey", ""); + if (!imgUrl.isBlank()) { + attached.add(buildInboundImagePart(imgUrl, aesKey, msgId, + "appmsg_image.jpg", inboundConvId)); + } + text.append("[图片").append(title.isBlank() ? "" : ": " + title).append("]"); + } else if (miniObj instanceof Map miniMap) { + @SuppressWarnings("unchecked") + Map mini = (Map) miniMap; + String miniTitle = ((String) mini.getOrDefault("title", + title.isBlank() ? "未命名小程序" : title)).trim(); + text.append("[小程序: ").append(miniTitle).append("]"); + } else if (!linkUrl.isBlank()) { + text.append("[链接]"); + if (!title.isBlank()) text.append(' ').append(title); + if (!desc.isBlank()) text.append('\n').append(desc); + text.append('\n').append(linkUrl); + // WeChat public-account articles (mp.weixin.qq.com) ship the + // body behind a captcha-gated SSR page — the URL is opaque to + // any LLM tool. Without this hint the model invents plausible + // content from the title alone (observed: "本文讲了三个要点…" + // hallucinations). The hint nudges the agent to ask the user + // to paste the article text instead of guessing. + if (isPublicAccountArticle(linkUrl)) { + text.append('\n').append(PUBLIC_ACCOUNT_ARTICLE_HINT); + } + } else if (!title.isBlank()) { + text.append("[appmsg: ").append(title).append("]"); + } else { + text.append("[appmsg]"); + } + return new AppmsgContent(text.toString(), attached); + } + + /** + * Hint appended to forwarded WeChat public-account articles. Worded as + * a directive for the agent (not a user-visible message) — the agent's + * reasoning prompt picks it up alongside the link itself, so the model + * sees the directive in-band with the share. + */ + static final String PUBLIC_ACCOUNT_ARTICLE_HINT = + "(提示:该链接为公众号文章,正文需要用户在微信内打开后复制粘贴," + + "请优先请用户粘贴正文,不要凭标题猜测内容。)"; + + /** + * Public-account article links are hosted on {@code mp.weixin.qq.com}. + * Compared to a generic URL host check, this is intentionally narrow — + * other Tencent properties (e.g. video.qq.com) don't share the same + * "title-only, body needs paste" property and shouldn't get the hint. + */ + static boolean isPublicAccountArticle(String url) { + if (url == null) return false; + String lower = url.toLowerCase(); + return lower.contains("://mp.weixin.qq.com/") + || lower.startsWith("mp.weixin.qq.com/"); + } + + /** + * Build a fully-populated image content part for inbound WeCom media. + *

+ * When download is enabled and succeeds, the part carries + * {@code path}, {@code fileUrl} (browser-servable), {@code fileName}, + * {@code storedName}, {@code fileSize}, and a precise {@code contentType} + * — every field the chat bubble and the multimodal sidecar inspect. When + * download is disabled or fails, the part falls back to URL-only fields + * but still sets {@code fileName} so the bubble doesn't render + * "未命名 / unknown". + */ + private MessageContentPart buildInboundImagePart(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + if (getConfigBoolean("media_download_enabled", true)) { + InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId); + if (r != null) { + MessageContentPart part = new MessageContentPart(); + part.setType("image"); + part.setFileName(r.fileName()); + part.setStoredName(r.storedName()); + part.setPath(r.localPath()); + part.setFileUrl(r.fileUrl()); + part.setFileSize(r.fileSize()); + // Prefer the sniffed contentType (could be image/png) over a + // hardcoded image/jpeg. Falls back to image/jpeg only when the + // sniff was inconclusive. + String ct = r.contentType(); + part.setContentType((ct != null && ct.startsWith("image/")) ? ct : "image/jpeg"); + // mediaId mirrors path so callers that prefer it still resolve + // to the same on-disk file (matches Web upload's behaviour). + part.setMediaId(r.localPath()); + return part; + } + } + // Fallback: download disabled or failed. Browser preview will be broken + // because the WeCom CDN URL carries a short-lived signature, but at + // least the bubble shows "image.jpg" instead of "未命名 / unknown". + MessageContentPart part = new MessageContentPart(); + part.setType("image"); + part.setFileName(fileNameHint); + part.setFileUrl(url); + part.setMediaId(url); + part.setContentType("image/jpeg"); + return part; + } + + /** + * Build a fully-populated file content part for inbound WeCom media. + *

+ * Mirrors {@link #buildInboundImagePart} but for non-image attachments + * (PDF, DOCX, ZIP, etc.). The magic-byte sniffer inside + * {@link #downloadInboundMedia} fixes generic {@code file.bin} hints to + * the real extension so downstream tools (PDF text extractor, magika, + * etc.) key off the correct mime. + */ + private MessageContentPart buildInboundFilePart(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + if (getConfigBoolean("media_download_enabled", true)) { + InboundMediaResult r = downloadInboundMedia(url, aesKey, msgId, fileNameHint, conversationId); + if (r != null) { + MessageContentPart part = new MessageContentPart(); + part.setType("file"); + part.setFileName(r.fileName()); + part.setStoredName(r.storedName()); + part.setPath(r.localPath()); + part.setFileUrl(r.fileUrl()); + part.setFileSize(r.fileSize()); + part.setContentType(r.contentType()); + part.setMediaId(r.localPath()); + return part; + } + } + // Fallback when download is disabled or fails — at least keep the + // original hint so the bubble doesn't say "file.bin" for a PDF. + MessageContentPart part = new MessageContentPart(); + part.setType("file"); + part.setFileName(fileNameHint); + part.setFileUrl(url); + part.setMediaId(url); + return part; + } + + /** + * Inbound-media download result. Carries every field the bubble renderer + * and the multimodal sidecar need so callers don't have to re-derive + * storedName / fileUrl from scratch. + * + * @param localPath absolute filesystem path of the saved file + * @param storedName the on-disk filename (matches the last segment of localPath) + * @param fileUrl browser-servable URL: {@code /api/v1/chat/files/{convId}/{storedName}} + * @param fileSize byte length after decryption + * @param fileName human-readable display name (extension corrected by magic-byte sniff) + * @param contentType MIME type derived from magic bytes (or {@code application/octet-stream}) + */ + record InboundMediaResult(String localPath, String storedName, + String fileUrl, long fileSize, String fileName, + String contentType) {} + + /** Magic-byte sniff result. */ + private record MagicSniff(String extension, String contentType) { + static final MagicSniff UNKNOWN = new MagicSniff(".bin", "application/octet-stream"); + } + + /** + * Best-effort MIME sniff from the first 12 bytes of a file. Covers the + * formats users routinely forward to bots (PDF, Office, archives, common + * image / audio / video). When nothing matches, returns + * {@link MagicSniff#UNKNOWN} so the caller falls back to {@code .bin}. + *

+ * This exists because WeCom's {@code aibot_msg_callback} {@code file} + * body sometimes omits {@code filename} entirely (forwarded files in + * particular), and shipping the agent a part labelled {@code file.bin} + * makes downstream tools mis-route the content. Sniffing recovers a + * useful extension so PDF tools fire on PDFs. + */ + private static MagicSniff sniffMagic(byte[] head) { + if (head == null || head.length < 4) return MagicSniff.UNKNOWN; + // PDF: %PDF + if (head[0] == 0x25 && head[1] == 0x50 && head[2] == 0x44 && head[3] == 0x46) { + return new MagicSniff(".pdf", "application/pdf"); + } + // PNG: 89 50 4E 47 + if (head[0] == (byte) 0x89 && head[1] == 0x50 && head[2] == 0x4E && head[3] == 0x47) { + return new MagicSniff(".png", "image/png"); + } + // JPEG: FF D8 FF + if (head[0] == (byte) 0xFF && head[1] == (byte) 0xD8 && head[2] == (byte) 0xFF) { + return new MagicSniff(".jpg", "image/jpeg"); + } + // GIF: "GIF8" + if (head[0] == 0x47 && head[1] == 0x49 && head[2] == 0x46 && head[3] == 0x38) { + return new MagicSniff(".gif", "image/gif"); + } + // ZIP-based container: PK\x03\x04. Could be a plain ZIP, a JAR, + // an OOXML document (DOCX/XLSX/PPTX), an ODF document (ODT/ODS/ODP), + // or an EPUB. Magic-byte alone can't tell them apart — caller is + // expected to follow up with refineZipKind(fullBytes) to pick a + // specific type. + if (head[0] == 0x50 && head[1] == 0x4B && head[2] == 0x03 && head[3] == 0x04) { + return new MagicSniff(".zip", "application/zip"); + } + // Legacy Office (DOC/XLS/PPT): D0 CF 11 E0 A1 B1 1A E1 + if (head.length >= 8 + && head[0] == (byte) 0xD0 && head[1] == (byte) 0xCF + && head[2] == 0x11 && head[3] == (byte) 0xE0 + && head[4] == (byte) 0xA1 && head[5] == (byte) 0xB1 + && head[6] == 0x1A && head[7] == (byte) 0xE1) { + return new MagicSniff(".doc", "application/msword"); + } + // RTF: "{\rtf" + if (head.length >= 5 + && head[0] == 0x7B && head[1] == 0x5C + && head[2] == 0x72 && head[3] == 0x74 && head[4] == 0x66) { + return new MagicSniff(".rtf", "application/rtf"); + } + // 7z: 37 7A BC AF 27 1C + if (head.length >= 6 + && head[0] == 0x37 && head[1] == 0x7A && head[2] == (byte) 0xBC + && head[3] == (byte) 0xAF && head[4] == 0x27 && head[5] == 0x1C) { + return new MagicSniff(".7z", "application/x-7z-compressed"); + } + // RAR: "Rar!\x1A\x07" + if (head.length >= 6 + && head[0] == 0x52 && head[1] == 0x61 && head[2] == 0x72 + && head[3] == 0x21 && head[4] == 0x1A && head[5] == 0x07) { + return new MagicSniff(".rar", "application/x-rar-compressed"); + } + // MP3: ID3v2 ("ID3") or MPEG sync 0xFFFB / 0xFFF3 / 0xFFF2 + if (head[0] == 0x49 && head[1] == 0x44 && head[2] == 0x33) { + return new MagicSniff(".mp3", "audio/mpeg"); + } + // MP4: "....ftyp" — bytes 4..7 == "ftyp" + if (head.length >= 8 + && head[4] == 0x66 && head[5] == 0x74 && head[6] == 0x79 && head[7] == 0x70) { + return new MagicSniff(".mp4", "video/mp4"); + } + // OGG: "OggS" + if (head[0] == 0x4F && head[1] == 0x67 && head[2] == 0x67 && head[3] == 0x53) { + return new MagicSniff(".ogg", "audio/ogg"); + } + return MagicSniff.UNKNOWN; + } + + /** + * Peek inside a ZIP container to distinguish OOXML (DOCX/XLSX/PPTX), + * ODF (ODT/ODS/ODP), JAR, and EPUB from a plain ZIP. Reads the local + * file headers in order via {@link ZipInputStream}; the discriminator + * entry is almost always within the first few entries (OOXML places + * {@code [Content_Types].xml} first, ODF places {@code mimetype} first), + * so we cap iteration at 16 entries to bound CPU. + *

+ * Returns the original {@code zipDefault} sniff (plain + * {@code application/zip}) when no specific kind is detected — that's + * the right answer for actual ZIPs and unknown archive formats. + */ + private static MagicSniff refineZipKind(byte[] fileData, MagicSniff zipDefault) { + if (fileData == null || fileData.length < 30) return zipDefault; + String mimetypeContent = null; + try (ZipInputStream zis = new ZipInputStream(new ByteArrayInputStream(fileData))) { + ZipEntry entry; + int seen = 0; + while ((entry = zis.getNextEntry()) != null && seen < 16) { + String name = entry.getName(); + // OOXML — Office Open XML (Word/Excel/PowerPoint). Each format + // has a distinct top-level directory; we match on prefix + // because the entry order isn't guaranteed. + if (name.startsWith("word/")) { + return new MagicSniff(".docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document"); + } + if (name.startsWith("xl/")) { + return new MagicSniff(".xlsx", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"); + } + if (name.startsWith("ppt/")) { + return new MagicSniff(".pptx", + "application/vnd.openxmlformats-officedocument.presentationml.presentation"); + } + // Visio (rare but worth catching) + if (name.startsWith("visio/")) { + return new MagicSniff(".vsdx", + "application/vnd.ms-visio.drawing"); + } + // ODF marker: a {@code mimetype} entry that contains the full + // application/vnd.oasis.opendocument.* string — read its body + // and decide once we have it. + if ("mimetype".equals(name)) { + byte[] buf = zis.readAllBytes(); + mimetypeContent = new String(buf, java.nio.charset.StandardCharsets.UTF_8).trim(); + } + // JAR + if ("META-INF/MANIFEST.MF".equals(name)) { + return new MagicSniff(".jar", "application/java-archive"); + } + // EPUB always has META-INF/container.xml + if ("META-INF/container.xml".equals(name)) { + return new MagicSniff(".epub", "application/epub+zip"); + } + seen++; + } + } catch (Exception e) { + log.debug("[wecom] refineZipKind failed (treating as plain zip): {}", e.getMessage()); + return zipDefault; + } + if (mimetypeContent != null) { + if (mimetypeContent.contains("opendocument.text")) { + return new MagicSniff(".odt", "application/vnd.oasis.opendocument.text"); + } + if (mimetypeContent.contains("opendocument.spreadsheet")) { + return new MagicSniff(".ods", "application/vnd.oasis.opendocument.spreadsheet"); + } + if (mimetypeContent.contains("opendocument.presentation")) { + return new MagicSniff(".odp", "application/vnd.oasis.opendocument.presentation"); + } + if (mimetypeContent.contains("epub")) { + return new MagicSniff(".epub", "application/epub+zip"); + } + } + return zipDefault; + } + + /** + * Strip a trailing extension from a filename. {@code "image.jpg" → "image"}; + * {@code "no_ext" → "no_ext"}; {@code "" → ""}. + */ + private static String stripExtension(String name) { + if (name == null || name.isBlank()) return ""; + int dot = name.lastIndexOf('.'); + if (dot <= 0) return name; + return name.substring(0, dot); + } + + /** + * Download + decrypt an inbound media attachment and stash it under + * {@code data/chat-uploads/{conversationId}/} so the existing + * {@code /api/v1/chat/files/...} endpoint can serve it back to the chat + * bubble. Returns a fully-populated {@link InboundMediaResult} on success + * or null on download/decrypt failure (callers fall back to URL-only). + *

+ * Storing under chat-uploads rather than {@code data/media} means + * {@link MessageContentPart#getPath()} resolves to a real file for the + * vision sidecar AND {@code fileUrl} renders as a thumbnail in the Web + * mirror — instead of the WeCom-signed CDN URL whose 5-minute query-string + * signature expires before the browser can fetch it. + */ + private InboundMediaResult downloadInboundMedia(String url, String aesKey, String msgId, + String fileNameHint, String conversationId) { + try { + // Mirror ChatController.uploadRoot ("data/chat-uploads") so the + // serve endpoint at /api/v1/chat/files/{convId}/{storedName} works + // without any extra wiring. The conversationId may contain ':' + // (e.g. "wecom:XuZhanFu" or "wecom:group:abc"); Path resolution + // tolerates this on macOS/Linux but Windows would reject the + // colon — for now we keep parity with the existing chat-uploads + // layout and revisit if Windows support comes up. + Path uploadDir = Path.of("data", "chat-uploads", conversationId); + Files.createDirectories(uploadDir); + + // 1. HTTP GET 下载文件 + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(url)) + .timeout(Duration.ofSeconds(30)) + .GET() + .build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofInputStream()); + byte[] encryptedData = response.body().readAllBytes(); + + byte[] fileData; + // 2. AES 解密(如果提供了 aesKey) + if (aesKey != null && !aesKey.isBlank()) { + fileData = decryptAes256Cbc(encryptedData, aesKey); + } else { + fileData = encryptedData; + } + + // 3. Magic-byte sniff to recover a real extension when WeCom + // didn't include filename in the body (forwarded files often + // arrive nameless — saving them as "file.bin" misroutes the + // agent because every PDF tool keys off the .pdf extension). + byte[] head = new byte[Math.min(12, fileData.length)]; + System.arraycopy(fileData, 0, head, 0, head.length); + MagicSniff sniff = sniffMagic(head); + // ZIP container needs a deeper look — DOCX/XLSX/PPTX/ODF/EPUB/JAR + // all share the PK\x03\x04 magic. Peek inside the first few + // entries to pick the specific kind. + if (".zip".equals(sniff.extension())) { + sniff = refineZipKind(fileData, sniff); + } + + // 4. Compose a URL-safe storedName. If the hint is generic + // (e.g. "file.bin"), prefer the sniffed extension. + String urlHash = md5Hex(url).substring(0, 8); + String hintRaw = (fileNameHint == null ? "media" : fileNameHint).trim(); + String safeName = hintRaw.replaceAll("[^a-zA-Z0-9._-]", "_"); + if (safeName.isBlank()) safeName = "media"; + // "file.bin" is the WeCom-no-filename sentinel; if magic gave us + // something better, replace the extension. Same when hint had no + // extension at all. + boolean hintIsGeneric = safeName.equals("file.bin") || safeName.equals("media") + || !safeName.contains("."); + if (hintIsGeneric && !".bin".equals(sniff.extension())) { + safeName = stripExtension(safeName) + sniff.extension(); + } + String storedName = "wecom_" + urlHash + "_" + safeName; + Path filePath = uploadDir.resolve(storedName); + Files.write(filePath, fileData); + + String fileUrl = "/api/v1/chat/files/" + conversationId + "/" + storedName; + log.info("[wecom] Inbound media saved: {} ({} bytes, sniffed={}), serve URL={}", + filePath, fileData.length, sniff.contentType(), fileUrl); + return new InboundMediaResult( + filePath.toAbsolutePath().toString(), + storedName, + fileUrl, + fileData.length, + safeName, + sniff.contentType()); + } catch (Exception e) { + log.error("[wecom] Failed to download inbound media: {}", e.getMessage(), e); + return null; + } + } + + /** + * 下载并解密企业微信媒体文件(旧版本,保留给 outbound / 其他场景使用) *

* AES-256-CBC 解密:base64 decode aesKey → IV = 前 16 字节 → PKCS#7 去填充 * @@ -1544,6 +3300,20 @@ public class WeComChannelAdapter extends AbstractChannelAdapter { return CHANNEL_TYPE; } + /** + * WeCom's AI-bot transport is WebSocket-only ({@code wss://openws.work.weixin.qq.com} + * with {@code aibot_subscribe} authenticated by {@code bot_id + secret}) — there is + * no HTTP webhook fallback. Multiple nodes subscribing with the same + * credentials would each receive every {@code aibot_msg_callback} and race + * to send {@code aibot_respond_msg}, producing duplicate replies and + * eventual gateway rejection. The leader gate ensures only one node + * holds the subscription at a time. + */ + @Override + public boolean requiresSingleLeader() { + return true; + } + // ==================== 工具方法 ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComKeepaliveScheduler.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComKeepaliveScheduler.java new file mode 100644 index 00000000..b53e6db6 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/WeComKeepaliveScheduler.java @@ -0,0 +1,167 @@ +package vip.mate.channel.wecom; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + +/** + * Periodically refreshes a WeCom AI Bot {@code stream} reply with the + * "🤔 思考中..." placeholder text so WeCom's server-side does not drop + * the stream slot while a long-running agent task is still computing. + * + *

Why this exists: WeCom's stream slot has an undocumented TTL + * (empirically observed at ~60-120s of silence). When the slot drops, + * the eventual {@code finish=true} chunk is silently rejected — the + * user sees "🤔 思考中..." stuck forever. RFC-32 §2.1.2 (R-7 / B-5). + * + *

Constants (chosen empirically based on observed slot lifetime): + *

    + *
  • 20s refresh interval — well under the observed 60s minimum drop
  • + *
  • 180s force-finish ceiling — bound the worst-case "stuck" UX even + * if the agent task hangs forever; lets the next user reply use a + * fresh stream slot rather than reuse a closed one
  • + *
+ * + *

Force-finish invariant: after the 180s ceiling fires, the + * scheduler calls {@link WeComChannelAdapter#invalidateReplyContext} + * to evict the {@code (frameReqId, processingStreamId)} pair from + * {@code replyContexts} — so when the eventual real reply arrives, + * {@code renderAndSend} sees no context and falls through to a fresh + * {@code sendMessage} path instead of reusing the dead stream id. + * RFC-32 §2.1.2 / R-7 closes this race; the adapter's + * {@code invalidateReplyContext} is the contract. + */ +@Slf4j +@Component +public class WeComKeepaliveScheduler { + + /** Refresh interval (seconds). */ + static final long REFRESH_INTERVAL_SECONDS = 20; + + /** Hard ceiling — after this many seconds, force-finish the stream. */ + static final long MAX_DURATION_SECONDS = 180; + + /** Placeholder text written on every refresh tick + on force-finish. */ + static final String PROCESSING_TEXT = "🤔 思考中..."; + + /** One-shot state per active stream. Held by reference inside the scheduled task. */ + private static final class StreamState { + final WeComChannelAdapter adapter; + final String reqId; + final String streamId; + final String replyToken; + final long startedAt; + volatile ScheduledFuture future; + StreamState(WeComChannelAdapter a, String r, String s, String t) { + this.adapter = a; this.reqId = r; this.streamId = s; this.replyToken = t; + this.startedAt = System.currentTimeMillis(); + } + } + + private final Map states = new ConcurrentHashMap<>(); + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(2, r -> { + Thread t = new Thread(r, "wecom-keepalive"); + t.setDaemon(true); + return t; + }); + + /** + * Begin keepalive for a freshly-issued processing stream. No-op if + * called twice for the same {@code streamId} (just logs and keeps + * the existing schedule alive). + * + * @param adapter the live WeCom adapter (carries replyStream + invalidateReplyContext) + * @param reqId the inbound message frame's req_id (binds the + * outbound reply_stream chunks) + * @param streamId the same stream_id used in the initial "🤔 思考中..." chunk + * @param replyToken target id used to look up reply context on + * invalidation (typically chatId for groups, + * senderId for direct messages — same value passed + * to {@code replyContexts.put}) + */ + public void start(WeComChannelAdapter adapter, String reqId, String streamId, String replyToken) { + if (adapter == null || reqId == null || streamId == null + || reqId.isBlank() || streamId.isBlank()) { + log.debug("[wecom-keepalive] start ignored — null/blank arg(s)"); + return; + } + if (states.containsKey(streamId)) { + log.debug("[wecom-keepalive] start ignored — stream {} already tracked", streamId); + return; + } + StreamState st = new StreamState(adapter, reqId, streamId, replyToken); + st.future = scheduler.scheduleAtFixedRate( + () -> tick(st), + REFRESH_INTERVAL_SECONDS, + REFRESH_INTERVAL_SECONDS, + TimeUnit.SECONDS); + states.put(streamId, st); + log.debug("[wecom-keepalive] started for stream={} reqId={}", streamId, reqId); + } + + /** + * Stop keepalive for a stream — call this immediately before sending + * the real reply so the next refresh tick doesn't race the + * {@code finish=true} chunk on the same stream. + */ + public void stop(String streamId) { + if (streamId == null || streamId.isBlank()) return; + StreamState st = states.remove(streamId); + if (st != null && st.future != null) { + st.future.cancel(false); + log.debug("[wecom-keepalive] stopped for stream={}", streamId); + } + } + + /** + * Drop every tracked stream and cancel its schedule. Called from + * {@code releaseConnectionResources} so reconnects start clean. + */ + public void shutdownAll() { + for (StreamState st : states.values()) { + if (st.future != null) st.future.cancel(false); + } + states.clear(); + } + + private void tick(StreamState st) { + long elapsedSec = (System.currentTimeMillis() - st.startedAt) / 1000; + if (elapsedSec >= MAX_DURATION_SECONDS) { + // Force-finish: send finish=true on the same stream so the + // server-side closes the slot cleanly, then evict the + // replyContext entry so the eventual real reply takes the + // fresh-stream path. + try { + st.adapter.replyStreamFinishForKeepalive(st.reqId, st.streamId, PROCESSING_TEXT); + } catch (Exception e) { + log.debug("[wecom-keepalive] force-finish replyStream failed for {}: {}", + st.streamId, e.getMessage()); + } + try { + st.adapter.invalidateReplyContext(st.replyToken, st.streamId); + } catch (Exception e) { + log.debug("[wecom-keepalive] invalidateReplyContext failed for {}: {}", + st.streamId, e.getMessage()); + } + stop(st.streamId); + log.info("[wecom-keepalive] force-finished stream {} after {}s ceiling", + st.streamId, MAX_DURATION_SECONDS); + return; + } + try { + st.adapter.replyStreamRefreshForKeepalive(st.reqId, st.streamId, PROCESSING_TEXT); + } catch (Exception e) { + log.debug("[wecom-keepalive] refresh failed for {}: {}", st.streamId, e.getMessage()); + } + } + + // ---- Test hooks ---- + + int activeStreamCount() { return states.size(); } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/CardOversizedException.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/CardOversizedException.java new file mode 100644 index 00000000..1ed8cf03 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/CardOversizedException.java @@ -0,0 +1,15 @@ +package vip.mate.channel.wecom.cards; + +/** + * Thrown when a card payload would exceed a WeCom-imposed size limit + * (most commonly: button.key serialised JSON > 1024 bytes). + * + *

Catchable so that WeCom card renderers can fall back to the + * abstract-class text path on overflow rather than letting the entire + * approval flow drop. RFC-32 §2.1.1 calls this out explicitly. + */ +public class CardOversizedException extends RuntimeException { + public CardOversizedException(String message) { + super(message); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardDispatcher.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardDispatcher.java new file mode 100644 index 00000000..0b7bdcb4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardDispatcher.java @@ -0,0 +1,100 @@ +package vip.mate.channel.wecom.cards; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.channel.wecom.cards.tool_guard.ToolGuardCardKindFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Routing-only dispatcher for WeCom interactive template cards. + * + *

Maintains two indexes keyed by disjoint identifiers: + *

    + *
  • Outbound: {@code metadata.message_type} from the agent + * runtime → {@link WeComCardKind#renderer()}. Currently the only + * direct outbound caller is {@link + * vip.mate.channel.wecom.WeComChannelAdapter#sendApprovalNotice} + * which doesn't yet read message_type — but having the index lets + * us add future card kinds without touching the adapter.
  • + *
  • Inbound: prefix of the + * {@code template_card_event.task_id} → {@link WeComCardKind#handler()}. + * Card kinds must use disjoint prefixes; collision throws + * at registration time.
  • + *
+ * + *

The {@code @Component} is autowired; current contributors are + * collected via {@link #registerKinds()} which calls factory beans for + * each kind. Adding a new card kind: implement {@code WeComCardKind} + + * a factory bean returning it + add a line to {@link #registerKinds()}. + */ +@Slf4j +@Component +public class WeComCardDispatcher { + + private final Map byMessageType = new HashMap<>(); + private final Map byTaskIdPrefix = new HashMap<>(); + + private final ToolGuardCardKindFactory toolGuardFactory; + + public WeComCardDispatcher(ToolGuardCardKindFactory toolGuardFactory) { + this.toolGuardFactory = toolGuardFactory; + registerKinds(); + } + + private void registerKinds() { + // Currently single kind. Add lines here as new card kinds land + // (poll cards / info-request cards / etc.). Order doesn't matter: + // the disjoint-prefix invariant prevents ambiguity at lookup. + register(toolGuardFactory.create()); + } + + private void register(WeComCardKind kind) { + if (byMessageType.containsKey(kind.messageType())) { + throw new IllegalStateException( + "duplicate card kind for messageType '" + kind.messageType() + + "': existing=" + byMessageType.get(kind.messageType()).name() + + ", new=" + kind.name()); + } + if (byTaskIdPrefix.containsKey(kind.taskIdPrefix())) { + throw new IllegalStateException( + "duplicate card kind for taskIdPrefix '" + kind.taskIdPrefix() + + "': existing=" + byTaskIdPrefix.get(kind.taskIdPrefix()).name() + + ", new=" + kind.name()); + } + byMessageType.put(kind.messageType(), kind); + byTaskIdPrefix.put(kind.taskIdPrefix(), kind); + log.info("[wecom-cards] Registered card kind: name={} messageType={} taskIdPrefix={}", + kind.name(), kind.messageType(), kind.taskIdPrefix()); + } + + /** + * Look up a card kind by outbound {@code metadata.message_type}. + */ + public Optional lookupByMessageType(String messageType) { + if (messageType == null || messageType.isBlank()) return Optional.empty(); + return Optional.ofNullable(byMessageType.get(messageType)); + } + + /** + * Look up a card kind by inbound {@code template_card_event.task_id}'s + * prefix. O(N) over registered kinds (N is small — currently 1). + */ + public Optional lookupByTaskId(String taskId) { + if (taskId == null || taskId.isBlank()) return Optional.empty(); + for (Map.Entry e : byTaskIdPrefix.entrySet()) { + if (taskId.startsWith(e.getKey())) { + return Optional.of(e.getValue()); + } + } + return Optional.empty(); + } + + /** Visible for tests / logs. */ + public List registeredKindNames() { + return byMessageType.values().stream().map(WeComCardKind::name).toList(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardHandler.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardHandler.java new file mode 100644 index 00000000..bf1216d0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardHandler.java @@ -0,0 +1,38 @@ +package vip.mate.channel.wecom.cards; + +import vip.mate.channel.wecom.WeComChannelAdapter; + +import java.util.Map; + +/** + * Processes an inbound WeCom {@code template_card_event} frame for one + * kind of card. + * + *

Implementations must respect the 5-second WeCom window: render and + * dispatch the resolved-state card update (via + * {@link WeComChannelAdapter#updateTemplateCard}) inside that window, + * THEN enqueue any agent-side command (e.g. {@code /approve }). + * The window starts the moment the event frame is received, so any + * pre-update validation must be cheap (DB lookup + identity check is + * fine; LLM round-trip is not). + */ +@FunctionalInterface +public interface WeComCardHandler { + /** + * @param adapter the live WeCom adapter (provides + * {@code updateTemplateCard}, {@code messageRouter} + * for command injection, etc.) + * @param frame the raw inbound frame including {@code headers.req_id} + * needed by {@code updateTemplateCard} + * @param tce the parsed {@code event.template_card_event} sub-object + * (already extracted by the dispatcher; contains + * {@code task_id} / {@code event_key}) + * @param fromBlock the {@code body.from} sub-object (carries + * {@code userid} of the clicker — needed for + * identity validation against original requester) + */ + void handle(WeComChannelAdapter adapter, + Map frame, + Map tce, + Map fromBlock); +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardKind.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardKind.java new file mode 100644 index 00000000..64bc71fa --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardKind.java @@ -0,0 +1,52 @@ +package vip.mate.channel.wecom.cards; + +/** + * Description of one kind of interactive WeCom template card the + * dispatcher knows how to route, along with the two functional callbacks + * that handle the outbound render and the inbound click event. + * + *

Kept as a simple record so adding a new card type (e.g. a poll + * card, an info-request card) is just: implement renderer/handler, + * register a new {@code WeComCardKind} in + * {@link WeComCardDispatcher#registerKinds()}. + * + * @param name short human-readable name for logs + * @param messageType matches {@code metadata.message_type} on the + * outbound event coming from the agent runtime + * (drives the {@code render} dispatch) + * @param taskIdPrefix matches the prefix of the inbound + * {@code template_card_event.task_id} (drives the + * {@code handle} dispatch). Card kinds must use + * disjoint prefixes; the dispatcher rejects + * registration of a colliding prefix. + * @param renderer converts a pending business object (e.g. + * {@code ApprovalNotice}) into a WeCom template_card + * payload Map. Throws {@link CardOversizedException} + * to signal "this kind cannot render now, fall back + * to text". + * @param handler processes an inbound {@code template_card_event} + * frame: validate identity → render resolved card → + * enqueue any follow-up command. Implementations + * MUST complete the render-resolved-card step inside + * the 5s WeCom protocol window; the agent enqueue + * step can be slower. + */ +public record WeComCardKind( + String name, + String messageType, + String taskIdPrefix, + WeComCardRenderer renderer, + WeComCardHandler handler +) { + public WeComCardKind { + if (name == null || name.isBlank()) { + throw new IllegalArgumentException("WeComCardKind.name must not be blank"); + } + if (messageType == null || messageType.isBlank()) { + throw new IllegalArgumentException("WeComCardKind.messageType must not be blank"); + } + if (taskIdPrefix == null || taskIdPrefix.isBlank()) { + throw new IllegalArgumentException("WeComCardKind.taskIdPrefix must not be blank"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardRenderer.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardRenderer.java new file mode 100644 index 00000000..9248f25d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/WeComCardRenderer.java @@ -0,0 +1,25 @@ +package vip.mate.channel.wecom.cards; + +import vip.mate.channel.notification.ApprovalNotice; + +import java.util.Map; + +/** + * Builds a WeCom template_card payload Map from a business object. + * + *

Implementations may throw {@link CardOversizedException} to signal + * the caller to fall back to a non-card path (e.g. text approval + * notice). All other throw paths surface as bugs. + * + *

Currently parameterised on {@link ApprovalNotice} since tool-guard + * is the only card kind in PR-1; future kinds will likely accept a + * different input or take {@code Object} and self-cast. + */ +@FunctionalInterface +public interface WeComCardRenderer { + /** + * Build the template_card body Map ready to drop into + * {@code aibot_respond_msg.body.template_card}. + */ + Map render(ApprovalNotice notice) throws CardOversizedException; +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKey.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKey.java new file mode 100644 index 00000000..43687cb8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKey.java @@ -0,0 +1,115 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import vip.mate.channel.wecom.cards.CardOversizedException; + +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * JSON encode/decode helper for the {@code key} field on each + * tool-guard approval card button. + * + *

WeCom enforces a hard 1024-byte ceiling on each + * {@code button.key} — the field is what the server echoes back as + * {@code event_key} when the user clicks. We pack the action plus the + * minimal context we need to recover the pending approval (the + * {@code pendingId} alone is enough — mateclaw's + * {@code ApprovalService.findById} resolves the rest, including the + * original requester). Sender/chat context is intentionally not packed + * — the inbound handler runs in-process and can do a synchronous DB + * lookup, leaving headroom in the 1024-byte budget for long tool names + * / Chinese characters. + * + *

Encoding is stable (LinkedHashMap → consistent key order so byte- + * length is predictable). Decoding tolerates extra fields — useful if + * a future change adds optional context. + */ +public final class ToolGuardButtonKey { + + /** Hard byte limit for the {@code button.key} field; verified against WeCom protocol. */ + public static final int MAX_KEY_BYTES = 1024; + + public enum Action { + APPROVE("approve"), + DENY("deny"); + + public final String wireValue; + Action(String v) { this.wireValue = v; } + + public static Action fromWire(String v) { + if ("approve".equalsIgnoreCase(v)) return APPROVE; + if ("deny".equalsIgnoreCase(v)) return DENY; + return null; + } + } + + /** Decoded button-key payload. {@code null} if parse fails or action invalid. */ + public record Decoded(Action action, String pendingId, String toolName, String severity) {} + + private final ObjectMapper objectMapper; + + public ToolGuardButtonKey(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + /** + * Encode an approval-button {@code key}. + * + * @throws CardOversizedException if the resulting JSON exceeds 1024 + * bytes (caller must fall back to text approval path) + */ + public String encode(Action action, String pendingId, String toolName, String severity) { + // LinkedHashMap so the key order on the wire is stable across calls; + // makes byte-length predictable and snapshot-testable. + Map payload = new LinkedHashMap<>(); + payload.put("a", action.wireValue); // action + payload.put("rid", pendingId); // request/pending id + payload.put("tool", toolName); // for log readability when WeCom replays event_key + payload.put("sev", severity == null ? "" : severity); + try { + String json = objectMapper.writeValueAsString(payload); + int bytes = json.getBytes(StandardCharsets.UTF_8).length; + if (bytes > MAX_KEY_BYTES) { + throw new CardOversizedException( + "tool_guard button.key payload " + bytes + " bytes > limit " + MAX_KEY_BYTES); + } + return json; + } catch (CardOversizedException e) { + throw e; + } catch (Exception e) { + throw new CardOversizedException("failed to serialise button.key: " + e.getMessage()); + } + } + + /** + * Decode the {@code event_key} echoed back by WeCom on button click. + * Returns {@code null} if the payload is malformed or the action + * unrecognised. Callers should treat null as "ignore this event". + */ + public Decoded decode(String eventKey) { + if (eventKey == null || eventKey.isBlank()) return null; + try { + Map raw = objectMapper.readValue(eventKey, new TypeReference<>() {}); + Action action = Action.fromWire(asString(raw.get("a"))); + if (action == null) return null; + String pendingId = asString(raw.get("rid")); + if (pendingId == null || pendingId.isBlank()) return null; + return new Decoded( + action, + pendingId, + asString(raw.getOrDefault("tool", "")), + asString(raw.getOrDefault("sev", ""))); + } catch (Exception e) { + // Malformed payload — treat as "ignore". The caller's + // log.debug at handler entry covers visibility. + return null; + } + } + + private static String asString(Object o) { + return o == null ? null : o.toString(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandler.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandler.java new file mode 100644 index 00000000..ea1b934f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandler.java @@ -0,0 +1,225 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import lombok.extern.slf4j.Slf4j; +import vip.mate.approval.ApprovalService; +import vip.mate.approval.PendingApproval; +import vip.mate.channel.ChannelMessage; +import vip.mate.channel.wecom.WeComChannelAdapter; +import vip.mate.channel.wecom.cards.WeComCardHandler; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Process an inbound {@code template_card_event} frame for a tool-guard + * approval card. + * + *

Step ordering — validate before render (RFC-32 v2.1 / R-5): + * v2.0's draft step order was "render resolved card → inject command → + * router validates identity". That meant a non-original-requester click + * would briefly show "✅ 已批准 by 李四" on the card before the router + * silently dropped the injected command. v2.1 reorders to: + *

    + *
  1. Decode {@code event_key} → null check
  2. + *
  3. Look up {@code PendingApproval} by id
  4. + *
  5. Identity check: pending.userId vs clicker
  6. + *
  7. Render the appropriate resolved-state card (success / unauthorized / expired)
  8. + *
  9. Inject {@code /approve} or {@code /deny} command into the router + * — only for authorised clicks
  10. + *
+ * + *

Steps 1-4 must complete inside the WeCom 5-second window for the + * card update; step 5 can be slower. + */ +@Slf4j +public class ToolGuardCardHandler implements WeComCardHandler { + + private final ApprovalService approvalService; + private final ToolGuardButtonKey buttonKey; + + public ToolGuardCardHandler(ApprovalService approvalService, ToolGuardButtonKey buttonKey) { + this.approvalService = approvalService; + this.buttonKey = buttonKey; + } + + @SuppressWarnings("unchecked") + @Override + public void handle(WeComChannelAdapter adapter, + Map frame, + Map tce, + Map fromBlock) { + String eventReqId = extractEventReqId(frame); + String taskId = (String) tce.getOrDefault("task_id", ""); + String eventKey = (String) tce.getOrDefault("event_key", ""); + + // ---- 1. Decode event_key ---- + ToolGuardButtonKey.Decoded decoded = buttonKey.decode(eventKey); + if (decoded == null) { + log.warn("[wecom-toolguard] Could not decode event_key, ignoring task_id={}", taskId); + return; + } + String pendingId = decoded.pendingId(); + ToolGuardButtonKey.Action action = decoded.action(); + String clickerUserId = stringOrEmpty(fromBlock.get("userid")); + + // ---- 2. Look up pending approval ---- + Optional opt = approvalService.getPending(pendingId); + if (opt.isEmpty() || !"pending".equals(opt.get().getStatus())) { + // Either: GC'd it / approved/denied via another path / never existed + log.info("[wecom-toolguard] Pending {} not found or already resolved (action={}, clicker={})", + pendingId, action, abbrev(clickerUserId)); + renderExpired(adapter, eventReqId, taskId, decoded.toolName()); + return; + } + PendingApproval pending = opt.get(); + + // ---- 3. Identity check ---- + String originalRequester = pending.getUserId(); + boolean isAuthorized = originalRequester == null + || "system".equals(originalRequester) + || originalRequester.equals(clickerUserId); + if (!isAuthorized) { + log.warn("[wecom-toolguard] Unauthorised click: clicker={} != requester={}, pending={}", + abbrev(clickerUserId), abbrev(originalRequester), pendingId); + renderUnauthorised(adapter, eventReqId, taskId, decoded.toolName(), originalRequester); + return; + } + + // ---- 4. Render resolved card (must finish inside the 5s WeCom window) ---- + renderResolved(adapter, eventReqId, taskId, decoded.toolName(), action, clickerUserId); + + // ---- 5. Inject the /approve or /deny command into the message router ---- + // Re-routing the click as a synthetic user message means we reuse the + // existing approval validation + state-machine path + // (ChannelMessageRouter.processMessage), so any future change to the + // approval flow keeps working without a parallel button-click code path. + String commandText = (action == ToolGuardButtonKey.Action.APPROVE ? "/approve " : "/deny ") + + pendingId; + ChannelMessage synthetic = buildSynthetic(commandText, clickerUserId, pending, frame); + try { + adapter.injectSyntheticMessage(synthetic); + log.info("[wecom-toolguard] Injected '{}' for pending={}, clicker={}", + action == ToolGuardButtonKey.Action.APPROVE ? "/approve" : "/deny", + pendingId, abbrev(clickerUserId)); + } catch (Exception e) { + // Never let an enqueue failure leave the card looking applied. + // The card already shows resolved-state, but the agent won't see + // the approve/deny — operator log is the safety net. + log.error("[wecom-toolguard] Failed to inject command for pending={}: {}", + pendingId, e.getMessage(), e); + } + } + + // ------------------------------------------------------------------ + // Card rendering helpers + // ------------------------------------------------------------------ + + private static void renderResolved(WeComChannelAdapter adapter, String eventReqId, + String taskId, String toolName, + ToolGuardButtonKey.Action action, String clicker) { + String title = action == ToolGuardButtonKey.Action.APPROVE + ? "✅ 已批准" + : "🚫 已拒绝"; + String desc = action == ToolGuardButtonKey.Action.APPROVE + ? "Tool " + toolName + " 已批准" + : "Tool " + toolName + " 已拒绝"; + try { + adapter.updateTemplateCard(eventReqId, + ToolGuardCardRenderer.buildResolvedCard(taskId, title, desc)); + } catch (Exception e) { + log.warn("[wecom-toolguard] update_template_card (resolved) failed: {}", e.getMessage()); + } + } + + private static void renderUnauthorised(WeComChannelAdapter adapter, String eventReqId, + String taskId, String toolName, String originalRequester) { + String requesterLabel = originalRequester == null ? "原请求者" : abbrev(originalRequester); + try { + adapter.updateTemplateCard(eventReqId, + ToolGuardCardRenderer.buildResolvedCard(taskId, + "❌ 仅原请求者可审批", + "请由 " + requesterLabel + " 操作")); + } catch (Exception e) { + log.warn("[wecom-toolguard] update_template_card (unauthorised) failed: {}", e.getMessage()); + } + } + + private static void renderExpired(WeComChannelAdapter adapter, String eventReqId, + String taskId, String toolName) { + try { + adapter.updateTemplateCard(eventReqId, + ToolGuardCardRenderer.buildResolvedCard(taskId, + "⌛ 审批已过期", + "Tool " + toolName + " 的审批已过期或被处理")); + } catch (Exception e) { + log.warn("[wecom-toolguard] update_template_card (expired) failed: {}", e.getMessage()); + } + } + + // ------------------------------------------------------------------ + // Synthetic message construction + // ------------------------------------------------------------------ + + /** + * Build a {@link ChannelMessage} that looks like the clicker just sent + * "/approve " (or /deny). The router's existing approval gate + * picks it up via {@code processMessage} and runs the same identity- + * check + state-machine that text commands hit. + */ + @SuppressWarnings("unchecked") + private static ChannelMessage buildSynthetic(String commandText, String clickerUserId, + PendingApproval pending, + Map eventFrame) { + // The conversation behind the original card is whichever WeCom chat + // the {@code template_card_event} arrived from. body.chattype + + // body.chatid let us reconstruct the same conversationId the + // original message used. + Map body = (Map) eventFrame.getOrDefault("body", Map.of()); + String chatType = stringOrEmpty(body.get("chattype")); + String chatId = stringOrEmpty(body.get("chatid")); + boolean isGroup = "group".equals(chatType); + String effectiveChatId = isGroup && !chatId.isBlank() ? chatId : null; + String replyToken = isGroup && !chatId.isBlank() ? chatId : clickerUserId; + + return ChannelMessage.builder() + .channelType("wecom") + .senderId(clickerUserId) + .senderName(clickerUserId) + .chatId(effectiveChatId) + .content(commandText) + .contentType("text") + .contentParts(List.of()) + .inputMode("text") + .timestamp(LocalDateTime.now()) + .replyToken(replyToken) + // Tag rawPayload so any downstream code that wants to + // distinguish real messages from button-click injections + // can read this flag instead of inspecting senderId. + .rawPayload(Map.of( + "wecom_button_click", true, + "wecom_pending_id", pending.getPendingId() + )) + .build(); + } + + // ------------------------------------------------------------------ + // Helpers + // ------------------------------------------------------------------ + + @SuppressWarnings("unchecked") + private static String extractEventReqId(Map frame) { + Map headers = (Map) frame.getOrDefault("headers", Map.of()); + return stringOrEmpty(headers.get("req_id")); + } + + private static String stringOrEmpty(Object v) { + return v == null ? "" : v.toString(); + } + + private static String abbrev(String s) { + if (s == null || s.length() <= 8) return s == null ? "" : s; + return s.substring(0, 8) + "…"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardKindFactory.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardKindFactory.java new file mode 100644 index 00000000..e36e5422 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardKindFactory.java @@ -0,0 +1,56 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; +import vip.mate.approval.ApprovalService; +import vip.mate.channel.wecom.cards.WeComCardKind; + +/** + * Spring-managed factory that produces the tool-guard card kind for + * {@link vip.mate.channel.wecom.cards.WeComCardDispatcher}. + * + *

Plain {@code @Component} so the dispatcher can constructor-inject + * it. Each call to {@link #create()} returns a freshly constructed + * {@link WeComCardKind}; the dispatcher keeps the result and queries it + * for life of the JVM. + */ +@Component +public class ToolGuardCardKindFactory { + + /** + * Same value as {@link ToolGuardCardRenderer#TASK_ID_PREFIX}. Kept + * here too so the {@link WeComCardKind#taskIdPrefix()} index can be + * declared from the factory without reaching into the renderer. + */ + public static final String TASK_ID_PREFIX = ToolGuardCardRenderer.TASK_ID_PREFIX; + + /** + * Outbound metadata.message_type matched by the dispatcher when the + * agent runtime emits an approval-pending event. Currently the WeCom + * adapter's sendApprovalNotice override doesn't read message_type + * (it always renders tool-guard), but having the index lets future + * card kinds plug in cleanly. + */ + public static final String MESSAGE_TYPE = "tool_guard_approval"; + + private final ApprovalService approvalService; + private final ObjectMapper objectMapper; + + public ToolGuardCardKindFactory(ApprovalService approvalService, ObjectMapper objectMapper) { + this.approvalService = approvalService; + this.objectMapper = objectMapper; + } + + public WeComCardKind create() { + ToolGuardButtonKey buttonKey = new ToolGuardButtonKey(objectMapper); + ToolGuardCardRenderer renderer = new ToolGuardCardRenderer(buttonKey); + ToolGuardCardHandler handler = new ToolGuardCardHandler(approvalService, buttonKey); + return new WeComCardKind( + "tool_guard_approval", + MESSAGE_TYPE, + TASK_ID_PREFIX, + renderer, + handler + ); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRenderer.java b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRenderer.java new file mode 100644 index 00000000..0addd2a5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRenderer.java @@ -0,0 +1,147 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import vip.mate.channel.notification.ApprovalNotice; +import vip.mate.channel.wecom.cards.CardOversizedException; +import vip.mate.channel.wecom.cards.WeComCardRenderer; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Build the WeCom {@code button_interaction} approval card payload from + * an {@link ApprovalNotice}. + * + *

Card structure (matches the WeCom official protocol): + *

+ * {
+ *   "card_type": "button_interaction",
+ *   "task_id": "tg_approval_<pendingId>",
+ *   "main_title": {
+ *     "title": "🛡️ 工具审批",
+ *     "desc":  "<toolName> | <severityLabel>"
+ *   },
+ *   "button_list": [
+ *     { "text": "批准", "style": 1, "key": "<encoded JSON>" },
+ *     { "text": "拒绝", "style": 2, "key": "<encoded JSON>" }
+ *   ]
+ * }
+ * 
+ * + *

If either button.key would exceed the 1024-byte WeCom limit, the + * encoder throws {@link CardOversizedException} and the calling adapter + * falls back to the abstract-class text-approval path. + */ +public class ToolGuardCardRenderer implements WeComCardRenderer { + + /** + * Prefix on the card's {@code task_id}; the inbound dispatcher matches + * this to find the right handler. Same value as + * {@link ToolGuardCardKindFactory#TASK_ID_PREFIX}. + */ + public static final String TASK_ID_PREFIX = "tg_approval_"; + + private final ToolGuardButtonKey buttonKey; + + public ToolGuardCardRenderer(ToolGuardButtonKey buttonKey) { + this.buttonKey = buttonKey; + } + + @Override + public Map render(ApprovalNotice notice) throws CardOversizedException { + String pendingId = notice.pendingId(); + String toolName = nullSafe(notice.toolName(), "tool"); + String severity = nullSafe(notice.maxSeverity(), "MEDIUM"); + + // Encode buttons first so a 1024-byte overflow throws BEFORE we + // build any of the cosmetic card structure. Same payload shape on + // both buttons differs only in the action wire value. + String approveKey = buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, pendingId, toolName, severity); + String denyKey = buttonKey.encode( + ToolGuardButtonKey.Action.DENY, pendingId, toolName, severity); + + // Use LinkedHashMap so the JSON serialisation order is stable — + // helps when log-grepping outbound frames against snapshots. + Map mainTitle = new LinkedHashMap<>(); + mainTitle.put("title", "🛡️ 工具审批"); + mainTitle.put("desc", buildSubtitle(toolName, severity)); + + Map approveBtn = new LinkedHashMap<>(); + approveBtn.put("text", "批准"); + approveBtn.put("style", 1); + approveBtn.put("key", approveKey); + + Map denyBtn = new LinkedHashMap<>(); + denyBtn.put("text", "拒绝"); + denyBtn.put("style", 2); + denyBtn.put("key", denyKey); + + Map card = new LinkedHashMap<>(); + card.put("card_type", "button_interaction"); + card.put("task_id", TASK_ID_PREFIX + pendingId); + card.put("main_title", mainTitle); + card.put("button_list", List.of(approveBtn, denyBtn)); + return card; + } + + /** + * Build a {@code text_notice} resolved-state card to update the + * original button card after a click. WeCom's card protocol requires + * {@code text_notice} cards to carry a {@code card_action} of type 1 + * or 2 (type 0 is rejected by the bot endpoint), so we provide a + * harmless project URL. + * + * @param taskId same task_id as the original card so the + * update targets the right message + * @param title one-line headline (e.g. "✅ 已批准 by 张三") + * @param desc optional detail line; truncated to 30 chars + * to stay inside WeCom's main_title.desc limit + */ + public static Map buildResolvedCard(String taskId, String title, String desc) { + Map mainTitle = new LinkedHashMap<>(); + mainTitle.put("title", title == null ? "" : title); + mainTitle.put("desc", truncate(desc == null ? "" : desc, 30)); + + Map cardAction = new LinkedHashMap<>(); + cardAction.put("type", 1); + cardAction.put("url", "https://mateclaw.vip"); + + Map card = new LinkedHashMap<>(); + card.put("card_type", "text_notice"); + card.put("task_id", taskId); + card.put("main_title", mainTitle); + card.put("card_action", cardAction); + return card; + } + + private static String buildSubtitle(String toolName, String severity) { + // Keep the subtitle short — WeCom truncates aggressively. Format: + // " | ". Translate severity to a single-word Chinese + // label so it reads naturally in the card. + return toolName + " | " + severityShortLabel(severity); + } + + private static String severityShortLabel(String severity) { + if (severity == null) return "MEDIUM"; + return switch (severity.toUpperCase()) { + case "CRITICAL" -> "🔴 极高"; + case "HIGH" -> "🟠 高"; + case "MEDIUM" -> "🟡 中"; + case "LOW" -> "🔵 低"; + case "INFO" -> "⚪ 提示"; + default -> severity; + }; + } + + private static String truncate(String s, int max) { + if (s == null) return ""; + if (s.length() <= max) return s; + if (max <= 1) return s.substring(0, max); + return s.substring(0, max - 1) + "…"; + } + + private static String nullSafe(String v, String fallback) { + return (v == null || v.isBlank()) ? fallback : v; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/config/ConversationWindowProperties.java b/mateclaw-server/src/main/java/vip/mate/config/ConversationWindowProperties.java index 32b52514..e69483ae 100644 --- a/mateclaw-server/src/main/java/vip/mate/config/ConversationWindowProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/config/ConversationWindowProperties.java @@ -37,4 +37,39 @@ public class ConversationWindowProperties { /** 摘要 token 预算下限(字数) */ private int summaryBudgetFloor = 500; + + /** + * Minimum prefix size (in messages) the pair-safe boundary must leave + * before compaction is allowed to run. After enforcing tool-call/response + * pair integrity the boundary may collapse so far forward that only a + * handful of messages remain in the prefix — at that point the + * compaction cost (a structured-summary LLM call) outweighs any token + * savings, and we may as well skip this turn. + * + *

Default 2 means "at least two old messages worth condensing". + * Set to 0 to always attempt compaction whenever a pair-safe cut exists. + */ + private int pairSafeMinPrefixToCompact = 2; + + /** + * After compaction, re-inject the first user message of the compressed + * prefix so the original goal stays anchored in the prompt even when a + * long task has paged through dozens of turns. Without an anchor the + * structured summary alone can drift, and the model may forget what was + * being asked. Injected as a {@link org.springframework.ai.chat.messages.UserMessage} + * (never SystemMessage) so historical user input cannot be promoted to a + * system-level instruction. + */ + private boolean firstUserAnchorEnabled = true; + + /** + * Maximum tokens the anchor body is allowed to consume in the prompt. + * The first user message is often short ("write me a CLI tool that…"), + * but power users sometimes paste multi-KB specs. When the body fits + * the budget it stays verbatim; when it is up to 3× over, it is + * head+tail truncated to this budget; when it is more than 3× over, + * it degrades to a 200-char pointer line so the model still knows the + * original goal existed without blowing prompt-cache or summary budget. + */ + private int firstUserAnchorMaxTokens = 400; } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java b/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java index 5384bc9c..960d5238 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/controller/ModelConfigController.java @@ -204,10 +204,12 @@ public class ModelConfigController { // ==================== Embedding 模型管理 ==================== - @Operation(summary = "按类型筛选模型(chat / embedding)") + @Operation(summary = "按类型筛选模型(chat / embedding),可选 modality 过滤") @GetMapping("/by-type") - public R> listByType(@RequestParam(defaultValue = "chat") String modelType) { - return R.ok(modelConfigService.listByType(modelType)); + public R> listByType( + @RequestParam(defaultValue = "chat") String modelType, + @RequestParam(required = false) String modality) { + return R.ok(modelConfigService.listByType(modelType, modality)); } @Operation(summary = "测试 Embedding 模型连通性(嵌入一个短文本验证 API key)") diff --git a/mateclaw-server/src/main/java/vip/mate/llm/embedding/EmbeddingModelFactory.java b/mateclaw-server/src/main/java/vip/mate/llm/embedding/EmbeddingModelFactory.java index 82b83347..a8436f95 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/embedding/EmbeddingModelFactory.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/embedding/EmbeddingModelFactory.java @@ -2,8 +2,8 @@ package vip.mate.llm.embedding; import com.alibaba.cloud.ai.autoconfigure.dashscope.DashScopeConnectionProperties; import com.alibaba.cloud.ai.dashscope.api.DashScopeApi; -import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingModel; -import com.alibaba.cloud.ai.dashscope.embedding.DashScopeEmbeddingOptions; +import com.alibaba.cloud.ai.dashscope.embedding.text.DashScopeEmbeddingModel; +import com.alibaba.cloud.ai.dashscope.embedding.text.DashScopeEmbeddingOptions; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.document.MetadataMode; diff --git a/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderRequirements.java b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderRequirements.java new file mode 100644 index 00000000..58790110 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/failover/ProviderRequirements.java @@ -0,0 +1,90 @@ +package vip.mate.llm.failover; + +import vip.mate.llm.model.ModelProviderEntity; + +import java.util.Map; + +/** + * Decide which fields a {@link ModelProviderEntity} needs to be considered + * "configured", based on the row's columns rather than its protocol enum. + * + *

Why row-based: every OpenAI-compatible provider (OpenAI / Kimi / DeepSeek + * cloud as well as llama.cpp / lmstudio / vllm / ollama local) shares the + * single {@code OPENAI_COMPATIBLE} protocol value. A protocol-keyed lookup + * cannot distinguish "cloud, needs api key" from "local, needs base url". + * The discriminating signals all live on the row: {@code requireApiKey}, + * {@code isLocal}, {@code isCustom}, {@code authType}, {@code providerId}. + * + *

Hint key + args (no raw text) so the frontend renders via i18n + * {@code t(key, args)} without leaking Chinese into the English locale. + */ +public final class ProviderRequirements { + + /** What this provider row needs to be considered configured. */ + public record Required( + boolean needsApiKey, + boolean needsBaseUrl, + String hintKey, // i18n key, null when no hint applies + Map hintArgs // template params for vue-i18n; never raw text + ) {} + + private static final Required NONE = new Required(false, false, null, Map.of()); + + private ProviderRequirements() {} + + /** + * Compute the required-fields verdict for a provider row. + * + * Decision tree: + * - authType == "oauth" -> no api key, no base url (OAuth handled elsewhere) + * - requireApiKey == true -> needs api key + * - isLocal || isCustom -> needs base url (no sane SDK default) + * - cloud built-ins -> SDK ships hard-coded base url; no base url needed + * + * Hint key picked from a small providerId-substring map for the most common + * local providers; everything else falls back to a generic OpenAI-compatible + * hint so the user always sees an actionable example URL. + */ + public static Required of(ModelProviderEntity provider) { + if (provider == null) return NONE; + + // OAuth providers store credentials elsewhere (DB column or disk). Neither + // api key nor base url applies to the configured check. + if ("oauth".equals(provider.getAuthType())) { + return NONE; + } + + boolean needsApiKey = Boolean.TRUE.equals(provider.getRequireApiKey()); + boolean isLocal = Boolean.TRUE.equals(provider.getIsLocal()); + boolean isCustom = Boolean.TRUE.equals(provider.getIsCustom()); + boolean needsBaseUrl = isLocal || isCustom; + + if (!needsBaseUrl) { + return new Required(needsApiKey, false, null, Map.of()); + } + + // Pick a hint by providerId substring. Order matters: more specific names + // first so "lm-studio" doesn't accidentally match a generic prefix later. + String pid = provider.getProviderId() == null ? "" : provider.getProviderId().toLowerCase(); + String hintKey; + Map hintArgs; + if (pid.contains("ollama")) { + hintKey = "provider.hint.ollamaBaseUrlExample"; + hintArgs = Map.of("example", "http://127.0.0.1:11434"); + } else if (pid.contains("lmstudio") || pid.contains("lm-studio") || pid.contains("lm_studio")) { + hintKey = "provider.hint.lmstudioBaseUrlExample"; + hintArgs = Map.of("example", "http://127.0.0.1:1234/v1"); + } else if (pid.contains("llamacpp") || pid.contains("llama-cpp") || pid.contains("llama_cpp") + || pid.contains("llama.cpp")) { + hintKey = "provider.hint.llamacppBaseUrlExample"; + hintArgs = Map.of("example", "http://127.0.0.1:8080/v1"); + } else if (pid.contains("vllm")) { + hintKey = "provider.hint.vllmBaseUrlExample"; + hintArgs = Map.of("example", "http://127.0.0.1:8000/v1"); + } else { + hintKey = "provider.hint.openaiCompatBaseUrlExample"; + hintArgs = Map.of("example", "http://127.0.0.1:8080/v1"); + } + return new Required(needsApiKey, true, hintKey, hintArgs); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/CreateCustomProviderRequest.java b/mateclaw-server/src/main/java/vip/mate/llm/model/CreateCustomProviderRequest.java index 7a910524..66da006a 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/CreateCustomProviderRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/CreateCustomProviderRequest.java @@ -12,5 +12,6 @@ public class CreateCustomProviderRequest { private String apiKeyPrefix; private String protocol; private String chatModel; + private Boolean requireApiKey; private List models; } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java index 97982771..fffe412d 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderConfigRequest.java @@ -11,6 +11,7 @@ public class ProviderConfigRequest { private String protocol; private String chatModel; private Map generateKwargs; + private Boolean requireApiKey; /** * RFC-009 P3.5: provider's position in the multi-model failover chain. * {@code 0} = excluded; positive ints define ascending try-order. When diff --git a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java index c5cfd317..1fac6e52 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/model/ProviderInfoDTO.java @@ -3,6 +3,7 @@ package vip.mate.llm.model; import lombok.Data; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -41,4 +42,35 @@ public class ProviderInfoDTO { private Long cooldownRemainingMs; /** RFC-074: whether the user has explicitly enabled this provider. False = lives in the catalog drawer only. */ private Boolean enabled; + + // Issue #81: derived fields powering the chat-console liveness-aware popup. + // All six are computed from existing columns; none are persisted. + + /** Credential status: CONFIGURED / MISSING / NOT_REQUIRED / OAUTH_PENDING. */ + private String authStatus; + + /** Base URL completeness: null when not applicable; true/false when applicable. */ + private Boolean baseUrlComplete; + + /** Comma-joined missing field names ("apiKey", "baseUrl"); empty when nothing missing. */ + private String missingFields; + + /** + * Machine-readable next-step key. Switches the chat popup's primary button text + handler. + * Values: fill_base_url / fill_api_key / start_oauth / configure_required_fields / + * test_connection / pull_model / wait_cooldown / reprobe / none. + */ + private String suggestedAction; + + /** + * i18n key for an actionable hint (e.g. "provider.hint.llamacppBaseUrlExample"). + * Frontend renders via t(key, args). Null when no hint applies. + */ + private String suggestedActionHintKey; + + /** + * Template parameters for {@link #suggestedActionHintKey}. Frontend passes + * this directly to vue-i18n. Empty map means no parameters. + */ + private Map suggestedActionHintArgs = new LinkedHashMap<>(); } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/oauth/OpenAIOAuthService.java b/mateclaw-server/src/main/java/vip/mate/llm/oauth/OpenAIOAuthService.java index b21c8119..06b5272b 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/oauth/OpenAIOAuthService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/oauth/OpenAIOAuthService.java @@ -13,6 +13,7 @@ import org.springframework.web.client.RestClient; import vip.mate.exception.MateClawException; import vip.mate.llm.model.ModelProviderEntity; import vip.mate.llm.repository.ModelProviderMapper; +import vip.mate.llm.service.ModelProviderService; import java.io.OutputStream; import java.net.InetSocketAddress; @@ -73,9 +74,11 @@ public class OpenAIOAuthService { private static final String SCOPES = "openid profile email offline_access"; private static final String PROVIDER_ID = "openai-chatgpt"; private static final int CALLBACK_PORT = 1455; + private static final String DEFAULT_CALLBACK_BIND_HOST = "127.0.0.1"; private final ModelProviderMapper modelProviderMapper; private final ObjectMapper objectMapper; + private final ModelProviderService modelProviderService; private final RestClient restClient = RestClient.create(); /** state → code_verifier 缓存 */ @@ -240,15 +243,17 @@ public class OpenAIOAuthService { // Try to bind synchronously up front so callers can detect failure. HttpServer server; + String bindHost = resolveCallbackBindHost(); try { - server = HttpServer.create(new InetSocketAddress("127.0.0.1", CALLBACK_PORT), 0); + server = HttpServer.create(new InetSocketAddress(bindHost, CALLBACK_PORT), 0); } catch (java.net.BindException e) { - log.warn("OAuth callback bind failed on port {} (in-use or restricted): {}", - CALLBACK_PORT, e.getMessage()); + log.warn("OAuth callback bind failed on {}:{} (in-use or restricted): {}", + bindHost, CALLBACK_PORT, e.getMessage()); pendingStates.remove(expectedState); return false; } catch (java.io.IOException e) { - log.warn("OAuth callback HttpServer.create IO error: {}", e.getMessage()); + log.warn("OAuth callback HttpServer.create IO error on {}:{}: {}", + bindHost, CALLBACK_PORT, e.getMessage()); pendingStates.remove(expectedState); return false; } @@ -311,7 +316,8 @@ public class OpenAIOAuthService { boundServer.start(); activeCallbackServer = boundServer; - log.info("OAuth 回调服务器已启动在 http://127.0.0.1:{}", CALLBACK_PORT); + log.info("OAuth 回调服务器已启动,监听 {}:{},浏览器回调地址 {}", + bindHost, CALLBACK_PORT, REDIRECT_URI); // 3 分钟超时自动关闭 CompletableFuture.delayedExecutor(3, TimeUnit.MINUTES).execute(() -> { @@ -490,6 +496,7 @@ public class OpenAIOAuthService { provider.setOauthAccountId(accountId); } modelProviderMapper.updateById(provider); + modelProviderService.activateFirstModelIfDefaultUnavailable(PROVIDER_ID); log.info("OpenAI OAuth token 已保存,expires_in={}s, accountId={}", expiresIn, accountId); } @@ -536,6 +543,15 @@ public class OpenAIOAuthService { } } + String resolveCallbackBindHost() { + String configured = System.getProperty("mateclaw.oauth.openai.callback-bind-host", + System.getenv("MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST")); + if (!StringUtils.hasText(configured)) { + return DEFAULT_CALLBACK_BIND_HOST; + } + return configured.trim(); + } + // ==================== PKCE 工具 ==================== private String generateCodeVerifier() { diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java new file mode 100644 index 00000000..6521009d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MediaCaptionService.java @@ -0,0 +1,141 @@ +package vip.mate.llm.routing; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.content.Media; +import org.springframework.core.io.FileSystemResource; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; +import org.springframework.util.MimeType; +import vip.mate.llm.chatmodel.ProviderChatModelFactory; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; + +/** + * Caption an image attachment using a configured vision model so a text-only + * primary model can still reason about it. + * + *

The service is the execution arm of the sidecar strategy chosen by + * {@link MultimodalRouter}: pick a vision-capable model, send a single + * structured prompt with the image attached, return the description text. + * + *

v1 has no caching layer — every call hits the vision model. The cache + * (keyed by {@code sha256(file_bytes) + visionModelId + locale}) is reserved + * for the next iteration; the API shape exposes {@code cacheHit} so callers + * already record the field in routing metadata. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MediaCaptionService { + + private final ProviderChatModelFactory chatModelFactory; + private final RetryTemplate retryTemplate; + + public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) { + if (visionModel == null || imagePart == null) { + return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null")); + } + Path mediaPath = resolveMediaPath(imagePart); + if (mediaPath == null) { + return CaptionResult.failure(0, new IllegalStateException( + "Image file not found for attachment: " + imagePart.getFileName())); + } + String contentType = imagePart.getContentType(); + if (contentType == null || "image/*".equals(contentType)) { + contentType = "image/jpeg"; + } + long start = System.currentTimeMillis(); + try { + ChatModel chatModel = chatModelFactory.buildFor(visionModel, retryTemplate); + ChatClient client = ChatClient.create(chatModel); + UserMessage userMessage = UserMessage.builder() + .text(buildPrompt(locale, imagePart.getFileName())) + .media(List.of(new Media(MimeType.valueOf(contentType), new FileSystemResource(mediaPath)))) + .build(); + String description = client.prompt() + .messages(userMessage) + .call() + .content(); + long elapsed = System.currentTimeMillis() - start; + String trimmed = description == null ? "" : description.trim(); + if (trimmed.isEmpty()) { + return CaptionResult.failure(elapsed, + new IllegalStateException("Vision model returned empty description")); + } + return CaptionResult.success(trimmed, elapsed, false); + } catch (Exception e) { + long elapsed = System.currentTimeMillis() - start; + log.warn("Caption call failed for {} via {}/{}: {}", + imagePart.getFileName(), visionModel.getProvider(), visionModel.getModelName(), + e.getMessage()); + return CaptionResult.failure(elapsed, e); + } + } + + /** + * Locale-aware prompt. Defaults to Chinese when the locale is null or unrecognized + * (matches the primary user base) but switches to English so vision-model output + * matches the chat language and avoids polluting English-only contexts. + */ + private String buildPrompt(Locale locale, String fileName) { + boolean english = locale != null && Locale.ENGLISH.getLanguage().equalsIgnoreCase(locale.getLanguage()); + String fileHint = (fileName == null || fileName.isBlank()) ? "" : " (" + fileName + ")"; + if (english) { + return "Describe this image" + fileHint + + " concisely: list the main objects, scene, any visible text (OCR), " + + "and notable actions or emotions. Keep the answer under 300 words. " + + "Reply with the description only — no preamble."; + } + return "请用一段简洁的中文描述这张图片" + fileHint + + ":列出主要物体、场景、画面中可见的文字(OCR)、以及人物动作或情绪。" + + "不超过 300 字。直接给出描述,不要寒暄。"; + } + + /** + * Mirrors {@code BaseAgent.resolveImagePath} but standalone — caption service + * is reused outside the agent context (e.g. tests, future preflight endpoint). + */ + private Path resolveMediaPath(MessageContentPart part) { + Path resolved = tryResolve(part.getPath()); + if (resolved != null) return resolved; + return tryResolve(part.getMediaId()); + } + + private Path tryResolve(String relativePath) { + if (relativePath == null || relativePath.isBlank()) return null; + Path path = Paths.get(relativePath); + if (path.isAbsolute() && Files.exists(path)) return path; + Path workdir = Paths.get(System.getProperty("user.dir")).resolve(relativePath); + if (Files.exists(workdir)) return workdir; + return null; + } + + public record CaptionResult( + String description, + boolean cacheHit, + long elapsedMs, + Throwable failure + ) { + public boolean isFailure() { + return failure != null; + } + + public static CaptionResult success(String description, long elapsedMs, boolean cacheHit) { + return new CaptionResult(description, cacheHit, elapsedMs, null); + } + + public static CaptionResult failure(long elapsedMs, Throwable failure) { + return new CaptionResult(null, false, elapsedMs, failure); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java new file mode 100644 index 00000000..9a51d46c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/MultimodalRouter.java @@ -0,0 +1,166 @@ +package vip.mate.llm.routing; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; +import vip.mate.llm.routing.model.MultimodalRoutingDecision.SkippedAttachment; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.llm.service.ModelCapabilityService.Modality; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.util.ArrayList; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** + * Decides how to handle attachments whose modality outruns the agent's primary model. + * + *

The router is a pure decision step: it inspects the parts list and the primary + * model's capability set, then returns a {@link MultimodalRoutingDecision}. Caller is + * responsible for executing the decision (e.g. invoking the caption service when + * strategy is SIDECAR). + * + *

v1 only supports image sidecar. Video attachments fall through to the NONE + * branch with an explanatory skip reason — the next iteration will add a video + * captioning path once a strategy for frame sampling is in place. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class MultimodalRouter { + + private final SystemSettingService systemSettingService; + private final ModelConfigService modelConfigService; + private final ModelCapabilityService capabilityService; + + public MultimodalRoutingDecision route(List parts, ModelConfigEntity primary) { + Set required = collectRequiredModalities(parts); + if (required.isEmpty()) return MultimodalRoutingDecision.none(); + + EnumSet primaryCaps = primary == null + ? EnumSet.noneOf(Modality.class) + : capabilityService.resolve(primary.getModelName(), primary.getModalities()); + if (primaryCaps.containsAll(required)) return MultimodalRoutingDecision.none(); + + EnumSet missing = EnumSet.copyOf(required); + missing.removeAll(primaryCaps); + + List skipped = new ArrayList<>(); + ModelConfigEntity sidecarModel = null; + + // VISION sidecar: resolve configured default vision model. + if (missing.contains(Modality.VISION)) { + ModelConfigEntity candidate = resolveSidecar(Modality.VISION); + if (candidate != null) { + sidecarModel = candidate; + } else { + String reason = describeMissingSidecar(Modality.VISION); + for (MessageContentPart p : imageParts(parts)) { + skipped.add(new SkippedAttachment("image", p.getFileName(), reason)); + } + } + } + + // VIDEO: v1 has no sidecar implementation. Mark as skipped so the UI can + // tell the user to switch to a video-capable primary model. Reserved for + // a follow-up RFC. + if (missing.contains(Modality.VIDEO)) { + for (MessageContentPart p : videoParts(parts)) { + skipped.add(new SkippedAttachment("video", p.getFileName(), + "video_sidecar_not_supported_in_v1")); + } + } + + if (sidecarModel != null) { + return MultimodalRoutingDecision.sidecar(sidecarModel, required, missing); + } + return MultimodalRoutingDecision.noneWithSkipped(required, missing, skipped); + } + + private Set collectRequiredModalities(List parts) { + if (parts == null || parts.isEmpty()) return Set.of(); + EnumSet required = EnumSet.noneOf(Modality.class); + for (MessageContentPart part : parts) { + if (part == null) continue; + String type = part.getType(); + String contentType = part.getContentType(); + if (isImagePart(type, contentType)) required.add(Modality.VISION); + else if (isVideoPart(type, contentType)) required.add(Modality.VIDEO); + else if (isAudioPart(type, contentType)) required.add(Modality.AUDIO); + } + return required; + } + + private boolean isImagePart(String type, String contentType) { + if ("image".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("image/"); + } + + private boolean isVideoPart(String type, String contentType) { + if ("video".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("video/"); + } + + private boolean isAudioPart(String type, String contentType) { + if ("audio".equals(type)) return true; + return "file".equals(type) && contentType != null && contentType.startsWith("audio/"); + } + + private List imageParts(List parts) { + return parts.stream() + .filter(p -> p != null && isImagePart(p.getType(), p.getContentType())) + .toList(); + } + + private List videoParts(List parts) { + return parts.stream() + .filter(p -> p != null && isVideoPart(p.getType(), p.getContentType())) + .toList(); + } + + /** + * Resolve the configured sidecar model for a modality. Returns null when: + * - the setting is empty / blank; + * - the referenced row no longer exists or has been disabled; + * - the row's resolved capability set does not actually contain the modality. + * The caller treats null as "ask the user to configure one." + */ + private ModelConfigEntity resolveSidecar(Modality modality) { + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long modelId = switch (modality) { + case VISION -> settings.getDefaultVisionModelId(); + case VIDEO -> settings.getDefaultVideoModelId(); + default -> null; + }; + if (modelId == null) return null; + ModelConfigEntity model; + try { + model = modelConfigService.getModel(modelId); + } catch (Exception e) { + log.debug("Configured sidecar model id={} could not be loaded: {}", modelId, e.getMessage()); + return null; + } + if (model == null || !Boolean.TRUE.equals(model.getEnabled())) return null; + if (!capabilityService.supports(model.getModelName(), model.getModalities(), modality)) { + log.warn("Configured sidecar model {}/{} does not actually support {} — ignoring", + model.getProvider(), model.getModelName(), modality); + return null; + } + return model; + } + + private String describeMissingSidecar(Modality modality) { + SystemSettingsDTO settings = systemSettingService.getSettings(); + Long configured = modality == Modality.VISION + ? settings.getDefaultVisionModelId() + : settings.getDefaultVideoModelId(); + if (configured == null) return modality.name().toLowerCase() + "_model_not_configured"; + return modality.name().toLowerCase() + "_model_unavailable"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java b/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java new file mode 100644 index 00000000..ef3c6691 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/llm/routing/model/MultimodalRoutingDecision.java @@ -0,0 +1,89 @@ +package vip.mate.llm.routing.model; + +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelCapabilityService.Modality; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Outcome of routing a single user turn that may carry image / video / audio attachments. + * + *

The decision is purely descriptive — execution (loading captions, mutating + * the user message) happens in the caller. Treat instances as immutable; the + * static factories cover the only valid shapes. + */ +public record MultimodalRoutingDecision( + Strategy strategy, + ModelConfigEntity sidecarModel, + Set requiredModalities, + Set primaryMissing, + List skipped +) { + + public enum Strategy { + /** Primary model handles the turn directly (or no attachments at all). */ + NONE, + /** A separate vision/video model captions attachments; primary stays. */ + SIDECAR, + /** Reserved: switch the whole turn to a multimodal model. v1 does not emit. */ + NATIVE + } + + public record SkippedAttachment(String type, String fileName, String reason) {} + + public static MultimodalRoutingDecision none() { + return new MultimodalRoutingDecision( + Strategy.NONE, null, Set.of(), Set.of(), List.of()); + } + + public static MultimodalRoutingDecision noneWithSkipped( + Set required, + Set missing, + List skipped) { + return new MultimodalRoutingDecision( + Strategy.NONE, null, required, missing, skipped); + } + + public static MultimodalRoutingDecision sidecar( + ModelConfigEntity sidecarModel, + Set required, + Set missing) { + return new MultimodalRoutingDecision( + Strategy.SIDECAR, sidecarModel, required, missing, List.of()); + } + + /** + * Serialize to a flat map for emission as a graph event payload. + * Only includes keys present in this decision so the resulting + * {@code metadata.routing} JSON stays compact for the chat UI. + */ + public Map toMap() { + Map m = new LinkedHashMap<>(); + m.put("strategy", strategy.name().toLowerCase()); + if (sidecarModel != null) { + m.put("sidecarModelId", sidecarModel.getId()); + m.put("sidecarModel", sidecarModel.getModelName()); + m.put("sidecarProvider", sidecarModel.getProvider()); + } + if (!requiredModalities.isEmpty()) { + m.put("requiredModalities", requiredModalities.stream().map(Enum::name).toList()); + } + if (!primaryMissing.isEmpty()) { + m.put("primaryMissing", primaryMissing.stream().map(Enum::name).toList()); + } + if (!skipped.isEmpty()) { + m.put("skipped", skipped.stream().map(s -> { + Map entry = new LinkedHashMap<>(); + entry.put("type", s.type()); + if (s.fileName() != null) entry.put("fileName", s.fileName()); + entry.put("reason", s.reason()); + return entry; + }).toList()); + } + return Collections.unmodifiableMap(m); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java index 1759b5c3..c278eb8d 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelConfigService.java @@ -23,6 +23,7 @@ public class ModelConfigService { private final ModelConfigMapper modelConfigMapper; private final ApplicationEventPublisher eventPublisher; + private final ModelCapabilityService modelCapabilityService; /** * Lazy to break circular dependency: ModelProviderService → ModelConfigService. @@ -42,10 +43,10 @@ public class ModelConfigService { public List listEnabledModels() { return modelConfigMapper.selectList(new LambdaQueryWrapper() .eq(ModelConfigEntity::getEnabled, true) - .eq(ModelConfigEntity::getProvider, "dashscope") // 仅 chat 类型(排除 embedding),NULL 兼容老数据 .and(w -> w.isNull(ModelConfigEntity::getModelType) .or().eq(ModelConfigEntity::getModelType, "chat")) + .orderByAsc(ModelConfigEntity::getProvider) .orderByDesc(ModelConfigEntity::getIsDefault) .orderByAsc(ModelConfigEntity::getName)); } @@ -67,17 +68,40 @@ public class ModelConfigService { * */ public List listByType(String modelType) { + return listByType(modelType, null); + } + + /** + * Optional modality filter (case-insensitive: {@code "vision" / "video" / "audio"}). + * When non-null, only enabled rows whose resolved capability set contains the + * requested modality survive — used by the multimodal sidecar settings UI to + * populate "default vision model" / "default video model" dropdowns. + */ + public List listByType(String modelType, String modality) { + List rows; if ("chat".equals(modelType)) { - return modelConfigMapper.selectList(new LambdaQueryWrapper() + rows = modelConfigMapper.selectList(new LambdaQueryWrapper() .and(w -> w.isNull(ModelConfigEntity::getModelType) .or().eq(ModelConfigEntity::getModelType, "chat")) .orderByDesc(ModelConfigEntity::getIsDefault) .orderByAsc(ModelConfigEntity::getName)); + } else { + rows = modelConfigMapper.selectList(new LambdaQueryWrapper() + .eq(ModelConfigEntity::getModelType, modelType) + .orderByDesc(ModelConfigEntity::getIsDefault) + .orderByAsc(ModelConfigEntity::getName)); } - return modelConfigMapper.selectList(new LambdaQueryWrapper() - .eq(ModelConfigEntity::getModelType, modelType) - .orderByDesc(ModelConfigEntity::getIsDefault) - .orderByAsc(ModelConfigEntity::getName)); + if (modality == null || modality.isBlank()) return rows; + ModelCapabilityService.Modality required; + try { + required = ModelCapabilityService.Modality.valueOf(modality.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + return rows; + } + return rows.stream() + .filter(m -> Boolean.TRUE.equals(m.getEnabled())) + .filter(m -> modelCapabilityService.supports(m.getModelName(), m.getModalities(), required)) + .toList(); } /** @@ -107,7 +131,7 @@ public class ModelConfigService { .and(w -> w.isNull(ModelConfigEntity::getModelType) .or().eq(ModelConfigEntity::getModelType, "chat")) .last("LIMIT 1")); - if (defaultMarked != null && isProviderConfigured(defaultMarked.getProvider())) { + if (defaultMarked != null && isProviderEnabledAndConfigured(defaultMarked.getProvider())) { return defaultMarked; } @@ -120,7 +144,7 @@ public class ModelConfigService { .orderByDesc(ModelConfigEntity::getIsDefault) .orderByAsc(ModelConfigEntity::getName)); for (ModelConfigEntity candidate : candidates) { - if (isProviderConfigured(candidate.getProvider())) { + if (isProviderEnabledAndConfigured(candidate.getProvider())) { return candidate; } } @@ -141,12 +165,12 @@ public class ModelConfigService { * dependency. Falls back to {@code true} when the service is not yet available * (e.g., during early bootstrap) so we don't accidentally block startup. */ - private boolean isProviderConfigured(String providerId) { + private boolean isProviderEnabledAndConfigured(String providerId) { if (modelProviderService == null || providerId == null) { return true; } try { - return modelProviderService.isProviderConfigured(providerId); + return modelProviderService.isProviderEnabledAndConfigured(providerId); } catch (Exception e) { return true; // conservative: don't filter if lookup fails } diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java index 3319ae1d..c2921f7f 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelDiscoveryService.java @@ -6,6 +6,7 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; +import org.springframework.http.client.JdkClientHttpRequestFactory; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; import org.springframework.web.client.RestClient; @@ -13,6 +14,7 @@ import vip.mate.exception.MateClawException; import vip.mate.llm.model.*; import vip.mate.llm.oauth.OpenAIOAuthService; +import java.net.http.HttpClient; import java.time.Duration; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -413,7 +415,7 @@ public class ModelDiscoveryService { } String apiKey = provider.getApiKey(); - RestClient client = RestClient.builder() + RestClient client = openAiCompatibleClientBuilder() .baseUrl(baseUrl) .defaultHeader(HttpHeaders.ACCEPT, MediaType.APPLICATION_JSON_VALUE) .build(); @@ -623,7 +625,7 @@ public class ModelDiscoveryService { Map kwargs = modelProviderService.readProviderGenerateKwargs(provider); String completionsPath = resolveCompletionsPath(baseUrl, kwargs); - RestClient.RequestHeadersSpec spec = RestClient.builder() + RestClient.RequestHeadersSpec spec = openAiCompatibleClientBuilder() .baseUrl(baseUrl) .defaultHeader(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE) .build() @@ -915,6 +917,23 @@ public class ModelDiscoveryService { return normalized; } + /** + * Build a RestClient.Builder pinned to HTTP/1.1 for self-hosted OpenAI-compatible + * servers. Java's HttpClient defaults to HTTP/2 and over cleartext attempts an + * H2C upgrade ({@code Upgrade: h2c, Connection: Upgrade, HTTP2-Settings: ...}). + * Uvicorn-based stacks (vLLM, lmstudio, llama.cpp, ollama) reject the upgrade + * by closing the socket mid-handshake — surfacing as either + * "header parser received no bytes" on the chat path or, more subtly, a + * 400 with body=None on the test path because the body never makes it past + * the upgrade negotiation. + */ + private RestClient.Builder openAiCompatibleClientBuilder() { + HttpClient httpClient = HttpClient.newBuilder() + .version(HttpClient.Version.HTTP_1_1) + .build(); + return RestClient.builder().requestFactory(new JdkClientHttpRequestFactory(httpClient)); + } + @SuppressWarnings("unchecked") private void applyCustomHeaders(RestClient.RequestHeadersSpec spec, Map kwargs) { if (kwargs == null) { diff --git a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java index f7d2cade..03ed03b1 100644 --- a/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java +++ b/mateclaw-server/src/main/java/vip/mate/llm/service/ModelProviderService.java @@ -14,6 +14,7 @@ import vip.mate.llm.event.ModelConfigChangedEvent; import vip.mate.llm.failover.AvailableProviderPool; import vip.mate.llm.failover.ProviderHealthTracker; import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.failover.ProviderRequirements; import vip.mate.llm.model.*; import vip.mate.llm.repository.ModelProviderMapper; @@ -128,6 +129,9 @@ public class ModelProviderService { provider.setBaseUrl(request.getBaseUrl()); provider.setChatModel(ModelProtocol.resolveChatModel(request.getProtocol(), request.getChatModel())); provider.setGenerateKwargs(writeJson(request.getGenerateKwargs())); + if (request.getRequireApiKey() != null) { + provider.setRequireApiKey(request.getRequireApiKey()); + } // RFC-009 P3.5: only update fallback priority when the caller explicitly // sends a value. null leaves it untouched (existing chain unchanged). if (request.getFallbackPriority() != null) { @@ -168,7 +172,7 @@ public class ModelProviderService { provider.setSupportModelDiscovery(false); provider.setSupportConnectionCheck(false); provider.setFreezeUrl(false); - provider.setRequireApiKey(true); + provider.setRequireApiKey(request.getRequireApiKey() == null || Boolean.TRUE.equals(request.getRequireApiKey())); modelProviderMapper.insert(provider); if (request.getModels() != null) { @@ -234,18 +238,31 @@ public class ModelProviderService { public boolean isProviderAvailable(String providerId) { ModelProviderEntity provider = getProvider(providerId); - return isProviderConfigured(provider) && hasModels(providerId); + return isProviderEnabledAndConfigured(provider) && hasModels(providerId); } public String getProviderUnavailableReason(String providerId) { ModelProviderEntity provider = getProvider(providerId); + if (!Boolean.TRUE.equals(provider.getEnabled())) { + return "Provider 未启用"; + } if (!isProviderConfigured(provider)) { - if (Boolean.TRUE.equals(provider.getRequireApiKey())) { - return "Provider 未配置有效的 API Key"; + // Issue #81: emit a precise reason based on which row-level fields are + // missing, rather than the previous protocol-blind heuristic. The new + // frontend reads suggestedActionHintKey/Args; this string remains for + // logs and legacy callers. + ProviderRequirements.Required req = ProviderRequirements.of(provider); + boolean hasBaseUrl = StringUtils.hasText(provider.getBaseUrl()); + boolean hasApiKey = hasUsableApiKey(provider.getApiKey()); + if (req.needsBaseUrl() && !hasBaseUrl && req.needsApiKey() && !hasApiKey) { + return "Provider 未配置 Base URL 和 API Key"; } - if (Boolean.TRUE.equals(provider.getIsCustom()) || !Boolean.TRUE.equals(provider.getIsLocal())) { + if (req.needsBaseUrl() && !hasBaseUrl) { return "Provider 未配置 Base URL"; } + if (req.needsApiKey() && !hasApiKey) { + return "Provider 未配置 API Key"; + } return "Provider 未完成配置"; } if (!hasModels(providerId)) { @@ -316,7 +333,7 @@ public class ModelProviderService { } private void tryAutoActivateModel(String providerId, ModelProviderEntity provider) { - if (!isProviderConfigured(provider)) { + if (!isProviderEnabledAndConfigured(provider)) { return; } List providerModels = modelConfigService.listModelsByProvider(providerId); @@ -327,7 +344,7 @@ public class ModelProviderService { try { ModelConfigEntity currentDefault = modelConfigService.getDefaultModel(); ModelProviderEntity defaultProvider = modelProviderMapper.selectById(currentDefault.getProvider()); - if (!isProviderConfigured(defaultProvider)) { + if (!isProviderEnabledAndConfigured(defaultProvider)) { shouldAutoActivate = true; } } catch (MateClawException e) { @@ -339,6 +356,16 @@ public class ModelProviderService { } } + /** + * OAuth/device-code completion updates credentials outside the normal provider + * config endpoint. Reuse the same default-model promotion logic so a freshly + * connected OAuth provider is immediately selectable by chat. + */ + public void activateFirstModelIfDefaultUnavailable(String providerId) { + ModelProviderEntity provider = getProvider(providerId); + tryAutoActivateModel(providerId, provider); + } + private ModelProviderEntity getProvider(String providerId) { ModelProviderEntity provider = modelProviderMapper.selectById(providerId); if (provider == null) { @@ -416,9 +443,85 @@ public class ModelProviderService { } dto.setModels(builtinModels); dto.setExtraModels(extraModels); + applySuggestedAction(dto, provider, providerLiveness); return dto; } + /** + * Issue #81: derive the chat-popup recovery hint from row + liveness, so the + * frontend can render a precise "next step" instead of a generic + * "model unavailable" toast. Six fields populated: + * - authStatus: CONFIGURED / MISSING / NOT_REQUIRED / OAUTH_PENDING + * - baseUrlComplete: null when not applicable, true/false otherwise + * - missingFields: comma-joined ("apiKey", "baseUrl") for required-field UX + * - suggestedAction: machine-readable next-step key (frontend switches on this) + * - suggestedActionHintKey + suggestedActionHintArgs: i18n key/args, no raw text + */ + private void applySuggestedAction(ProviderInfoDTO dto, ModelProviderEntity provider, Liveness liveness) { + ProviderRequirements.Required req = ProviderRequirements.of(provider); + boolean hasBaseUrl = StringUtils.hasText(provider.getBaseUrl()); + boolean hasApiKey = hasUsableApiKey(provider.getApiKey()); + boolean hasModels = (dto.getModels() != null && !dto.getModels().isEmpty()) + || (dto.getExtraModels() != null && !dto.getExtraModels().isEmpty()); + + // 1. authStatus + if ("oauth".equals(provider.getAuthType())) { + dto.setAuthStatus(Boolean.TRUE.equals(dto.getOauthConnected()) ? "CONFIGURED" : "OAUTH_PENDING"); + } else if (req.needsApiKey()) { + dto.setAuthStatus(hasApiKey ? "CONFIGURED" : "MISSING"); + } else { + dto.setAuthStatus("NOT_REQUIRED"); + } + + // 2. baseUrlComplete: null when this provider doesn't need a base URL. + dto.setBaseUrlComplete(req.needsBaseUrl() ? hasBaseUrl : null); + + // 3. missingFields + java.util.List missing = new ArrayList<>(); + if (req.needsApiKey() && !hasApiKey) missing.add("apiKey"); + if (req.needsBaseUrl() && !hasBaseUrl) missing.add("baseUrl"); + dto.setMissingFields(String.join(",", missing)); + + // 4. suggestedAction + String action; + if (liveness == Liveness.UNCONFIGURED) { + if ("oauth".equals(provider.getAuthType())) { + action = "start_oauth"; + } else if (missing.size() == 1 && missing.get(0).equals("baseUrl")) { + action = "fill_base_url"; + } else if (missing.size() == 1 && missing.get(0).equals("apiKey")) { + action = "fill_api_key"; + } else { + action = "configure_required_fields"; + } + } else if (liveness == Liveness.REMOVED) { + action = "reprobe"; + } else if (liveness == Liveness.COOLDOWN) { + action = "wait_cooldown"; + } else if (liveness == Liveness.UNPROBED) { + action = "reprobe"; + } else if (liveness == Liveness.LIVE && !hasModels) { + action = Boolean.TRUE.equals(provider.getSupportModelDiscovery()) + ? "pull_model" + : "configure_required_fields"; + } else { + action = "none"; + } + dto.setSuggestedAction(action); + + // 5. hint key + args (NOT raw text). Frontend renders via t(key, args). + // Only emit hint when it actually applies to the action; suppress for + // REMOVED / COOLDOWN / UNPROBED to keep the popup clean. + if ("fill_base_url".equals(action) || "configure_required_fields".equals(action)) { + dto.setSuggestedActionHintKey(req.hintKey()); + dto.setSuggestedActionHintArgs(req.hintArgs() == null ? new java.util.LinkedHashMap<>() + : new java.util.LinkedHashMap<>(req.hintArgs())); + } else { + dto.setSuggestedActionHintKey(null); + dto.setSuggestedActionHintArgs(new java.util.LinkedHashMap<>()); + } + } + private boolean hasModels(String providerId) { return !modelConfigService.listModelsByProvider(providerId).isEmpty(); } @@ -427,13 +530,11 @@ public class ModelProviderService { if (provider == null) { return false; } - if (Boolean.TRUE.equals(provider.getIsLocal())) { - return true; - } - // OAuth 认证的 provider:检查 OAuth token 是否存在 + // OAuth providers store credentials elsewhere (DB column or disk for + // Claude Code). Resolve them via the OAuth service rather than the + // base-URL / api-key columns. if ("oauth".equals(provider.getAuthType())) { - // Claude Code OAuth (RFC-062) — token lives on disk, not in DB. if (CLAUDE_CODE_PROVIDER_ID.equals(provider.getProviderId())) { ClaudeCodeOAuthService svc = claudeCodeOAuthServiceProvider.getIfAvailable(); return svc != null && svc.isLoggedIn(); @@ -441,16 +542,21 @@ public class ModelProviderService { return StringUtils.hasText(provider.getOauthAccessToken()); } - boolean hasBaseUrl = StringUtils.hasText(provider.getBaseUrl()); - boolean hasApiKey = hasUsableApiKey(provider.getApiKey()); - - if (Boolean.TRUE.equals(provider.getIsCustom())) { - return hasBaseUrl && (!Boolean.TRUE.equals(provider.getRequireApiKey()) || hasApiKey); + // Issue #81: decide required fields from the provider row, not from the + // protocol enum. Every OpenAI-compatible provider (cloud or local) shares + // OPENAI_COMPATIBLE, so a protocol-keyed table cannot tell OpenAI cloud + // (needs api_key, no base url) apart from llama.cpp local (no api_key, + // needs base url). Without this, isLocal=true short-circuited to true + // for llama.cpp regardless of an empty Base URL, hiding the real cause + // behind a confusing REMOVED state. + ProviderRequirements.Required req = ProviderRequirements.of(provider); + if (req.needsApiKey() && !hasUsableApiKey(provider.getApiKey())) { + return false; } - if (Boolean.FALSE.equals(provider.getRequireApiKey())) { - return hasBaseUrl; + if (req.needsBaseUrl() && !StringUtils.hasText(provider.getBaseUrl())) { + return false; } - return hasApiKey; + return true; } public boolean hasUsableApiKey(String apiKey) { @@ -458,11 +564,30 @@ public class ModelProviderService { return false; } String normalized = apiKey.trim(); + // Reject masked display values (the UI sends "********" when the user + // didn't re-type the key) and known placeholder sentinels — without this + // check, the chat / embedding fallback chain happily forwards the + // placeholder to the LLM endpoint, which then returns a 401 at request + // time. "configure-in-admin-ui" is the application.yml default that + // keeps DashScopeChatAutoConfiguration happy at startup when no env var + // is set; "your-*-api-key-here" are legacy sentinels from earlier + // .env.example / application.yml versions. return !normalized.contains("*") + && !"configure-in-admin-ui".equalsIgnoreCase(normalized) && !"your-dashscope-api-key-here".equalsIgnoreCase(normalized) && !"your-api-key-here".equalsIgnoreCase(normalized); } + public boolean isProviderEnabledAndConfigured(String providerId) { + return isProviderEnabledAndConfigured(getProvider(providerId)); + } + + private boolean isProviderEnabledAndConfigured(ModelProviderEntity provider) { + return provider != null + && Boolean.TRUE.equals(provider.getEnabled()) + && isProviderConfigured(provider); + } + public Map readProviderGenerateKwargs(ModelProviderEntity provider) { return readJson(provider != null ? provider.getGenerateKwargs() : null); } diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java deleted file mode 100644 index 2fe75991..00000000 --- a/mateclaw-server/src/main/java/vip/mate/memory/fact/model/FactEntityRefEntity.java +++ /dev/null @@ -1,32 +0,0 @@ -package vip.mate.memory.fact.model; - -import com.baomidou.mybatisplus.annotation.*; -import lombok.Data; - -import java.time.LocalDateTime; - -/** - * Entity reference for multi-hop graph queries on facts. - * - * @author MateClaw Team - */ -@Data -@TableName("mate_fact_entity_ref") -public class FactEntityRefEntity { - - @TableId(type = IdType.AUTO) - private Long id; - - private Long factId; - - private String entityName; - - /** person, tool, project, concept */ - private String entityType; - - /** subject | object */ - private String role; - - @TableField(fill = FieldFill.INSERT) - private LocalDateTime createTime; -} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/query/FactQueryService.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/query/FactQueryService.java index f8a98bc5..d5ec9f0f 100644 --- a/mateclaw-server/src/main/java/vip/mate/memory/fact/query/FactQueryService.java +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/query/FactQueryService.java @@ -6,7 +6,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import vip.mate.memory.fact.model.FactContradictionEntity; import vip.mate.memory.fact.model.FactEntity; -import vip.mate.memory.fact.model.FactEntityRefEntity; import vip.mate.memory.fact.repository.FactMapper; import java.time.LocalDateTime; @@ -24,7 +23,6 @@ import java.util.List; public class FactQueryService { private final FactMapper factMapper; - private final vip.mate.memory.fact.repository.FactEntityRefMapper refMapper; private final vip.mate.memory.fact.repository.FactContradictionMapper contradictionMapper; /** @@ -41,27 +39,6 @@ public class FactQueryService { .last("LIMIT 20")); } - /** - * Find related facts via entity references (multi-hop). - */ - public List related(Long agentId, String entity, int hops) { - // Find fact IDs that reference this entity - List refs = refMapper.selectList( - new LambdaQueryWrapper() - .like(FactEntityRefEntity::getEntityName, entity) - .last("LIMIT 50")); - List factIds = refs.stream().map(FactEntityRefEntity::getFactId).distinct().toList(); - if (factIds.isEmpty()) return List.of(); - - return factMapper.selectList( - new LambdaQueryWrapper() - .eq(FactEntity::getAgentId, agentId) - .eq(FactEntity::getDeleted, 0) - .in(FactEntity::getId, factIds) - .orderByDesc(FactEntity::getTrust) - .last("LIMIT 20")); - } - /** * List unresolved contradictions for an agent. */ diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactEntityRefMapper.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactEntityRefMapper.java deleted file mode 100644 index 0e8829a9..00000000 --- a/mateclaw-server/src/main/java/vip/mate/memory/fact/repository/FactEntityRefMapper.java +++ /dev/null @@ -1,9 +0,0 @@ -package vip.mate.memory.fact.repository; - -import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import org.apache.ibatis.annotations.Mapper; -import vip.mate.memory.fact.model.FactEntityRefEntity; - -@Mapper -public interface FactEntityRefMapper extends BaseMapper { -} diff --git a/mateclaw-server/src/main/java/vip/mate/memory/fact/tool/FactQueryTool.java b/mateclaw-server/src/main/java/vip/mate/memory/fact/tool/FactQueryTool.java index 30aea726..c2732ee7 100644 --- a/mateclaw-server/src/main/java/vip/mate/memory/fact/tool/FactQueryTool.java +++ b/mateclaw-server/src/main/java/vip/mate/memory/fact/tool/FactQueryTool.java @@ -43,24 +43,6 @@ public class FactQueryTool { .collect(Collectors.joining("\n")); } - @Tool(description = "Find facts related to an entity via entity references (multi-hop graph query).") - public String fact_related( - @ToolParam(description = "Agent ID") Long agentId, - @ToolParam(description = "Entity name") String entity, - @ToolParam(description = "Number of hops (1-3)") int hops) { - if (!properties.getFact().isProjectionEnabled()) { - return "Fact projection is disabled."; - } - List facts = queryService.related(agentId, entity, Math.min(hops, 3)); - if (facts.isEmpty()) return "No related facts found for: " + entity; - - queryService.bumpUseCount(facts.stream().map(FactEntity::getId).toList()); - - return facts.stream() - .map(f -> String.format("- %s %s %s (trust=%.2f)", f.getSubject(), f.getPredicate(), f.getObjectValue(), f.getTrust())) - .collect(Collectors.joining("\n")); - } - @Tool(description = "List unresolved fact contradictions detected during Dream consolidation.") public String fact_list_contradictions( @ToolParam(description = "Agent ID") Long agentId) { diff --git a/mateclaw-server/src/main/java/vip/mate/skill/acp/AcpSkillBridge.java b/mateclaw-server/src/main/java/vip/mate/skill/acp/AcpSkillBridge.java index a200ec40..e5ae7a20 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/acp/AcpSkillBridge.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/acp/AcpSkillBridge.java @@ -301,6 +301,12 @@ public class AcpSkillBridge { s.setEnabled(Boolean.TRUE.equals(ep.getEnabled())); s.setBuiltin(Boolean.TRUE.equals(ep.getBuiltin())); s.setTags("acp"); + // Carry the backing endpoint's workspace through to the virtual + // SkillEntity so binding-time tenancy checks can compare it against + // the agent's workspace. Without this the bridge synthesizes rows + // with workspaceId = null and an agent in any workspace could bind + // any ACP endpoint regardless of where the endpoint was provisioned. + s.setWorkspaceId(ep.getWorkspaceId()); s.setSecurityScanStatus("PASSED"); // ACP endpoints are user-configured external CLIs, not skill scripts s.setConfigJson(buildConfigJson(ep)); s.setManifestJson(serializeManifest(buildManifest(ep))); diff --git a/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java b/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java index 979c21fa..d40d957d 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java @@ -23,6 +23,7 @@ import vip.mate.skill.synthesis.SkillSynthesisService; import vip.mate.skill.runtime.SkillRuntimeService; import vip.mate.skill.runtime.model.ResolvedSkill; import vip.mate.skill.workspace.BundledSkillSyncer; +import vip.mate.skill.workspace.SkillFileSyncer; import vip.mate.skill.workspace.SkillWorkspaceManager; import java.util.ArrayList; @@ -49,6 +50,7 @@ public class SkillController { private final SkillRuntimeService skillRuntimeService; private final SkillWorkspaceManager workspaceManager; private final BundledSkillSyncer bundledSkillSyncer; + private final SkillFileSyncer skillFileSyncer; private final SkillSynthesisService synthesisService; private final SkillDependencyChecker dependencyChecker; private final SkillLessonsService lessonsService; @@ -249,13 +251,88 @@ public class SkillController { @Operation(summary = "重新扫描单个技能(RFC-042 §2.3.4)") @PostMapping("/{id}/rescan") public R rescan(@PathVariable Long id) { + rejectVirtualSkillMutation(id); return R.ok(skillService.rescanSecurity(id)); } + @Operation(summary = "Re-sync this skill's bundle files from DB → local workspace cache", + description = "Use after an out-of-band scripts/ change or to recover a missing local cache " + + "in a multi-instance deployment. Pulls every mate_skill_file row owned by the skill " + + "down to disk; if no rows exist yet but local files do, ingests them into the canonical store.") + @PostMapping("/{id}/sync-files") + public R> syncFiles(@PathVariable Long id) { + rejectVirtualSkillMutation(id); + SkillEntity skill = skillService.getSkill(id); + var report = skillFileSyncer.syncOne(skill); + Map body = new LinkedHashMap<>(); + body.put("skillId", id); + body.put("name", skill.getName()); + body.put("filesMaterialized", report.filesMaterialized()); + body.put("filesAlreadyCurrent", report.filesAlreadyCurrent()); + body.put("filesBackfilledFromDisk", report.filesBackfilledFromDisk()); + body.put("backfilledFromDisk", report.didBackfillFromDisk()); + return R.ok(body); + } + + @Operation(summary = "Re-sync every skill's bundle files (admin)", + description = "Bulk variant of /sync-files; primarily for ops debugging when you suspect " + + "the local workspace is out of sync with the canonical store.") + @PostMapping("/sync-files") + public R> syncAllFiles() { + var report = skillFileSyncer.syncAll(); + Map body = new LinkedHashMap<>(); + body.put("skillsConsidered", report.skillsConsidered()); + body.put("skillsBackfilled", report.skillsBackfilled()); + body.put("filesMaterialized", report.filesMaterialized()); + body.put("filesAlreadyCurrent", report.filesAlreadyCurrent()); + body.put("filesBackfilledFromDisk", report.filesBackfilledFromDisk()); + return R.ok(body); + } + + /** + * Mutation paths refuse virtual MCP/ACP skill ids upfront. The bridge + * synthesizes those rows on the fly from the upstream connection + * config; persisting an update against {@code mate_skill} would + * either silently no-op (no row to update) or — as users have hit — + * throw "技能不存在" because the lookup precedes the update. Sending + * a clear 4xx with a redirect hint is the right shape: the user + * wants the icon / display name / etc. to stick, and the only place + * those fields persist for an MCP entry is the MCP connection page. + */ + private void rejectVirtualSkillMutation(Long id) { + if (vip.mate.skill.mcp.McpSkillBridge.isVirtualMcpSkillId(id) + || vip.mate.skill.acp.AcpSkillBridge.isVirtualAcpSkillId(id)) { + throw new vip.mate.exception.MateClawException( + "err.skill.virtual_readonly", + "MCP/ACP 衍生技能不可在此编辑——请到 Settings ▸ MCP/ACP 连接页修改"); + } + } + @Operation(summary = "获取已启用技能列表") @GetMapping("/enabled") public R> listEnabled() { - return R.ok(skillService.listEnabledSkills()); + // Mirror the merging the paginated /skills endpoint does so the agent + // edit picker (which calls this endpoint) sees MCP- and ACP-derived + // virtual skills alongside the persisted ones. The shadow base must + // include all real skill names — including disabled ones — so a + // disabled real skill correctly suppresses its same-named virtual + // twin, matching /skills and /counts. + List result = new ArrayList<>(skillService.listEnabledSkills()); + Set realNames = realSkillNames(); + + try { + result.addAll(filterShadowedVirtualSkills( + mcpSkillBridge.listMcpDerivedSkillEntities(), realNames)); + } catch (Exception e) { + // Bridge failure must not 500 the picker — same defensive stance as /counts. + } + try { + result.addAll(filterShadowedVirtualSkills( + acpSkillBridge.listAcpDerivedSkillEntities(), realNames)); + } catch (Exception e) { + // Bridge failure must not 500 the picker — same defensive stance as /counts. + } + return R.ok(result); } @Operation(summary = "按类型获取技能列表") @@ -300,6 +377,7 @@ public class SkillController { @Operation(summary = "更新技能") @PutMapping("/{id}") public R update(@PathVariable Long id, @RequestBody SkillEntity skill) { + rejectVirtualSkillMutation(id); skill.setId(id); return R.ok(skillService.updateSkill(skill)); } @@ -316,6 +394,7 @@ public class SkillController { @Operation(summary = "硬删除技能 (admin only — 物理删除 + 工作区清空)") @DeleteMapping("/{id}") public R delete(@PathVariable Long id) { + rejectVirtualSkillMutation(id); skillService.hardDeleteSkill(id); return R.ok(); } @@ -323,6 +402,7 @@ public class SkillController { @Operation(summary = "启用/禁用技能") @PutMapping("/{id}/toggle") public R toggle(@PathVariable Long id, @RequestParam boolean enabled) { + rejectVirtualSkillMutation(id); return R.ok(skillService.toggleSkill(id, enabled)); } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/installer/BuiltinSkillSeedService.java b/mateclaw-server/src/main/java/vip/mate/skill/installer/BuiltinSkillSeedService.java index 61f3446b..dd956087 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/installer/BuiltinSkillSeedService.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/installer/BuiltinSkillSeedService.java @@ -309,7 +309,16 @@ public class BuiltinSkillSeedService implements ApplicationRunner { row.setDescription(nullIfBlank(parsed.getDescription())); row.setSkillType(SKILL_TYPE_BUILTIN); row.setBuiltin(true); - row.setEnabled(true); + // Frontmatter `optional: true` flips the initial seed to enabled=false, + // so heavyweight bundled skills (paid CLI dependencies, external OAuth, + // niche integrations) ship dark — the user opts in from the Skills page + // when they actually want them. The default (frontmatter absent or + // false) preserves the historical "all bundled skills active" behavior. + // {@link #mergeIntoExisting} deliberately does NOT touch `enabled`, so + // once a user activates an optional skill, subsequent boots keep their + // choice and a downgrade in frontmatter never silently disables it. + boolean optional = booleanFromFrontmatter(parsed, "optional", false); + row.setEnabled(!optional); row.setSkillContent(content); row.setVersion(stringFromFrontmatter(parsed, "version", DEFAULT_VERSION)); row.setIcon(stringFromFrontmatter(parsed, "icon", DEFAULT_ICON)); @@ -406,6 +415,24 @@ public class BuiltinSkillSeedService implements ApplicationRunner { return dirty; } + /** + * Read a boolean frontmatter key, tolerant of the YAML / casual-string + * forms the parser might surface ({@code true} / {@code "true"} / + * {@code "yes"} / {@code "1"}). Anything else falls back to the supplied + * default so a typo doesn't silently flip behavior. + */ + private boolean booleanFromFrontmatter(SkillFrontmatterParser.ParsedSkillMd parsed, + String key, boolean fallback) { + Map fm = parsed.getFrontmatter(); + if (fm == null) return fallback; + Object value = fm.get(key); + if (value == null) return fallback; + if (value instanceof Boolean b) return b; + String s = value.toString().trim().toLowerCase(); + if (s.isEmpty()) return fallback; + return s.equals("true") || s.equals("yes") || s.equals("1") || s.equals("on"); + } + @SuppressWarnings("unchecked") private String stringFromFrontmatter(SkillFrontmatterParser.ParsedSkillMd parsed, String key, String fallback) { diff --git a/mateclaw-server/src/main/java/vip/mate/skill/installer/SkillInstaller.java b/mateclaw-server/src/main/java/vip/mate/skill/installer/SkillInstaller.java index cf6a6749..8e0e4cf1 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/installer/SkillInstaller.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/installer/SkillInstaller.java @@ -7,6 +7,7 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import vip.mate.skill.installer.model.*; import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.service.SkillFileService; import vip.mate.skill.service.SkillService; import vip.mate.skill.workspace.SkillWorkspaceEvent; import vip.mate.skill.workspace.SkillWorkspaceManager; @@ -36,6 +37,7 @@ public class SkillInstaller { private final SkillHubClient skillHubClient; private final SkillWorkspaceManager workspaceManager; private final SkillService skillService; + private final SkillFileService skillFileService; private final ObjectMapper objectMapper; private final ApplicationEventPublisher eventPublisher; @@ -146,79 +148,35 @@ public class SkillInstaller { return CompletableFuture.completedFuture(null); } - // 4. 写入 workspace 目录 - // overwrite 时先清理旧 references/ 和 scripts/,防止残留过期文件 - if (exists) { - workspaceManager.cleanWorkspaceDataDirs(skillName); - } - // 重装时 (exists=true) 覆写 SKILL.md;否则保留已有内容(向后兼容首次创建语义) + // 4. Materialize SKILL.md (overwrite on reinstall, keep on first create). workspaceManager.initWorkspace(skillName, bundle.content(), exists); - // 写入 references/ - if (bundle.references() != null) { - for (var entry : bundle.references().entrySet()) { - workspaceManager.writeWorkspaceFile(skillName, "references/" + entry.getKey(), entry.getValue()); - } - } - - // 写入 scripts/ - if (bundle.scripts() != null) { - for (var entry : bundle.scripts().entrySet()) { - workspaceManager.writeWorkspaceFile(skillName, "scripts/" + entry.getKey(), entry.getValue()); - } - } - - // cancel check: 文件已落盘,但数据库尚未写入 —— 归档已写入的目录后退出 if (task.isCancelRequested()) { workspaceManager.archiveWorkspace(skillName); task.markCancelled(); return CompletableFuture.completedFuture(null); } - // 5. 注册/更新数据库 - SkillEntity skillEntity; - if (exists) { - // 更新已有记录 - skillEntity = skillService.listSkills().stream() - .filter(s -> s.getName().equals(skillName)) - .findFirst().orElseThrow(); - skillEntity.setSkillContent(bundle.content()); - skillEntity.setDescription(bundle.description()); - skillEntity.setVersion(bundle.version()); - skillEntity.setAuthor(bundle.author()); - skillEntity.setIcon(bundle.icon()); - skillEntity.setConfigJson(buildConfigJson(bundle)); - if (Boolean.TRUE.equals(request.getEnable())) { - skillEntity.setEnabled(true); - } - skillService.updateSkill(skillEntity); - } else { - // 创建新记录 - skillEntity = new SkillEntity(); - skillEntity.setName(skillName); - skillEntity.setDescription(bundle.description()); - skillEntity.setSkillType("dynamic"); - skillEntity.setVersion(bundle.version()); - skillEntity.setAuthor(bundle.author()); - skillEntity.setIcon(bundle.icon()); - skillEntity.setSkillContent(bundle.content()); - skillEntity.setConfigJson(buildConfigJson(bundle)); - skillEntity.setEnabled(Boolean.TRUE.equals(request.getEnable())); - skillService.createSkill(skillEntity); - } + // 5. Register/update the skill row first so we have an id for the file rows. + SkillEntity skillEntity = upsertSkillRow(bundle, skillName, exists, + Boolean.TRUE.equals(request.getEnable())); - // cancel check: DB 已写入,此时取消不再回滚数据库,但标记任务为 cancelled if (task.isCancelRequested()) { task.markCancelled(); return CompletableFuture.completedFuture(null); } - // 6. 发布事件 + // 6. Persist bundle files: DB is canonical, FS is the materialized cache. + // Empty-bundle guard protects both sides from a malformed bundle + // silently wiping pre-existing scripts/references. + boolean force = Boolean.TRUE.equals(request.getForcePrune()); + persistBundleFiles(skillEntity, bundle, force, "url"); + + // 7. Publish event for runtime refresh / sibling-node materialization. eventPublisher.publishEvent(new SkillWorkspaceEvent( skillName, SkillWorkspaceEvent.Type.INSTALLED, workspaceManager.resolveConventionPath(skillName))); - // 7. 完成 task.markCompleted(InstallResult.builder() .name(skillName) .enabled(Boolean.TRUE.equals(request.getEnable())) @@ -254,28 +212,37 @@ public class SkillInstaller { "Skill '" + skillName + "' already exists. Enable overwrite to replace."); } - // 写入 workspace - if (exists) { - workspaceManager.cleanWorkspaceDataDirs(skillName); - } - workspaceManager.initWorkspace(skillName, bundle.content()); + // Materialize SKILL.md (always overwrite on reinstall path). + workspaceManager.initWorkspace(skillName, bundle.content(), exists); - if (bundle.references() != null) { - for (var entry : bundle.references().entrySet()) { - String key = entry.getKey(); - if (!key.startsWith("references/")) key = "references/" + key; - workspaceManager.writeWorkspaceFile(skillName, key, entry.getValue()); - } - } - if (bundle.scripts() != null) { - for (var entry : bundle.scripts().entrySet()) { - String key = entry.getKey(); - if (!key.startsWith("scripts/")) key = "scripts/" + key; - workspaceManager.writeWorkspaceFile(skillName, key, entry.getValue()); - } - } + // Register/update skill row first so we have an id to anchor the file rows. + SkillEntity skillEntity = upsertSkillRow(bundle, skillName, exists, enable); - // 注册/更新 DB + // DB-canonical, FS-cache. Empty-bundle guard on both sides. + persistBundleFiles(skillEntity, bundle, false, "zip"); + + eventPublisher.publishEvent(new SkillWorkspaceEvent( + skillName, SkillWorkspaceEvent.Type.INSTALLED, + workspaceManager.resolveConventionPath(skillName))); + + int filesCount = (bundle.references() != null ? bundle.references().size() : 0) + + (bundle.scripts() != null ? bundle.scripts().size() : 0) + 1; + + log.info("Skill '{}' installed from ZIP (v{}, {} files)", skillName, bundle.version(), filesCount); + + return Map.of( + "skillId", skillEntity.getId(), + "name", skillName, + "version", bundle.version() != null ? bundle.version() : "", + "filesCount", filesCount + ); + } + + /** + * Insert or update the {@code mate_skill} row from a bundle. Returns the + * persisted entity so callers have its id for downstream file writes. + */ + private SkillEntity upsertSkillRow(SkillBundle bundle, String skillName, boolean exists, boolean enable) { SkillEntity skillEntity; if (exists) { skillEntity = skillService.listSkills().stream() @@ -302,22 +269,40 @@ public class SkillInstaller { skillEntity.setEnabled(enable); skillService.createSkill(skillEntity); } + return skillEntity; + } - eventPublisher.publishEvent(new SkillWorkspaceEvent( - skillName, SkillWorkspaceEvent.Type.INSTALLED, - workspaceManager.resolveConventionPath(skillName))); + /** + * Write bundle files to both DB (canonical) and FS (cache) using the + * same prefixed-key map. Logs a single combined summary so multi-instance + * deployments can see what each node persisted vs preserved. + */ + private void persistBundleFiles(SkillEntity skillEntity, SkillBundle bundle, boolean force, String origin) { + Map combined = new LinkedHashMap<>(); + if (bundle.references() != null) { + for (var e : bundle.references().entrySet()) { + String key = e.getKey().startsWith("references/") ? e.getKey() : "references/" + e.getKey(); + combined.put(key, e.getValue()); + } + } + if (bundle.scripts() != null) { + for (var e : bundle.scripts().entrySet()) { + String key = e.getKey().startsWith("scripts/") ? e.getKey() : "scripts/" + e.getKey(); + combined.put(key, e.getValue()); + } + } - int filesCount = (bundle.references() != null ? bundle.references().size() : 0) - + (bundle.scripts() != null ? bundle.scripts().size() : 0) + 1; + var dbApply = skillFileService.applyBundleFiles(skillEntity.getId(), combined, force); + var fsApply = workspaceManager.applyBundleFiles(skillEntity.getName(), + bundle.references(), bundle.scripts(), force); - log.info("Skill '{}' installed from ZIP (v{}, {} files)", skillName, bundle.version(), filesCount); - - return Map.of( - "skillId", skillEntity.getId(), - "name", skillName, - "version", bundle.version() != null ? bundle.version() : "", - "filesCount", filesCount - ); + log.info("Persisted bundle for '{}' ({}): db(write={}, prune={}, preservedScripts={}, preservedRefs={}) " + + "fs(refs write={}, prune={}, preserved={} | scripts write={}, prune={}, preserved={})", + skillEntity.getName(), origin, + dbApply.rowsWritten(), dbApply.rowsPruned(), + dbApply.scriptsPreservedDueToEmptyBundle(), dbApply.referencesPreservedDueToEmptyBundle(), + fsApply.referencesWritten(), fsApply.referencesPruned(), fsApply.referencesPreservedDueToEmptyBundle(), + fsApply.scriptsWritten(), fsApply.scriptsPruned(), fsApply.scriptsPreservedDueToEmptyBundle()); } // ==================== 工具方法 ==================== diff --git a/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java b/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java index 29fabfe9..0f4ac2ac 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java @@ -9,15 +9,18 @@ import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; /** * Parses a ZIP-packaged Skill into a {@link SkillBundle}. *

- * Used by both the upload endpoint (MultipartFile) and the ClawHub install + * Used by both the upload endpoint (MultipartFile) and the marketplace install * path (downloaded ZIP bytes). Hardened against: *

    *
  • Zip Slip path traversal
  • @@ -25,6 +28,12 @@ import java.util.zip.ZipInputStream; *
  • Only SKILL.md / references/ / scripts/ entries are kept
  • *
* + *

Extraction is two-pass: the entire archive is buffered in memory first + * (cap-protected), then SKILL.md is located and the common parent prefix is + * stripped from every other entry. This keeps classification correct + * regardless of the order zip tools write entries — earlier single-pass logic + * silently dropped {@code scripts/*} entries that streamed before SKILL.md. + * * @author MateClaw Team */ @Slf4j @@ -35,15 +44,39 @@ public class ZipSkillFetcher { private static final String SKILL_MD = "SKILL.md"; private static final String SKILL_MD_LOWER = "skill.md"; + /** + * Lowercase file extensions that should be treated as runnable scripts + * when they appear next to SKILL.md without an explicit {@code scripts/} + * prefix. Real-world zips from third parties (e.g. the official + * tencent-meeting-mcp package) put {@code setup.sh} at the package + * root; without this fallback, those files get logged as "unclassified" + * and the skill ships with an empty scripts/ directory. + */ + private static final Set SCRIPT_EXTENSIONS = Set.of( + ".sh", ".bash", ".zsh", ".py", ".js", ".mjs", ".ts", + ".rb", ".pl", ".php", ".bat", ".cmd", ".ps1"); + + /** + * Lowercase file extensions that are documentation / data alongside + * SKILL.md and should default to {@code references/} when not nested + * under an explicit prefix. + */ + private static final Set REFERENCE_EXTENSIONS = Set.of( + ".md", ".txt", ".json", ".yaml", ".yml", ".csv", ".tsv", + ".html", ".htm", ".xml", ".toml"); + /** * Holds the in-memory result of decompressing a ZIP. Used by callers - * that want to enrich the SkillBundle with metadata (e.g. ClawHub author - * / icon) that isn't carried inside SKILL.md. + * that want to enrich the SkillBundle with metadata (e.g. marketplace + * author / icon) that isn't carried inside SKILL.md. */ public record ExtractedSkill(String skillMdContent, Map references, Map scripts) {} + /** Buffered raw zip entry, awaiting classification once SKILL.md prefix is known. */ + private record RawEntry(String name, String content) {} + /** * Parse an uploaded ZIP file into a SkillBundle. Source type is "zip" * and source URL is the original filename. @@ -93,12 +126,18 @@ public class ZipSkillFetcher { /** * Decompress a ZIP stream into in-memory SKILL.md + references + scripts. * Throws {@link IllegalArgumentException} if no SKILL.md is present. + * + *

Two-pass: the first pass buffers every text entry (subject to size + * caps) and remembers where SKILL.md lives. The second pass strips the + * SKILL.md parent prefix from each buffered entry and routes it into + * {@code references} / {@code scripts}. Anything that doesn't match + * either bucket is logged at WARN level so packaging mistakes surface + * instead of being silently dropped. */ public static ExtractedSkill extract(InputStream zipStream) throws IOException { + List raws = new ArrayList<>(); String skillMdContent = null; String skillMdPrefix = ""; - Map references = new HashMap<>(); - Map scripts = new HashMap<>(); long totalSize = 0; try (ZipInputStream zis = new ZipInputStream(zipStream, StandardCharsets.UTF_8)) { @@ -138,28 +177,20 @@ public class ZipSkillFetcher { } String content = new String(bytes, StandardCharsets.UTF_8); + String normalizedName = entryPath.toString().replace('\\', '/'); String fileName = entryPath.getFileName().toString(); + // First match wins for SKILL.md so we lock onto the shallowest one. if (skillMdContent == null && (SKILL_MD.equals(fileName) || SKILL_MD_LOWER.equals(fileName))) { skillMdContent = content; - int slashIdx = entryName.lastIndexOf('/'); - skillMdPrefix = slashIdx > 0 ? entryName.substring(0, slashIdx + 1) : ""; - log.info("[ZipSkillFetcher] Found SKILL.md at: {}", entryName); + int slashIdx = normalizedName.lastIndexOf('/'); + skillMdPrefix = slashIdx > 0 ? normalizedName.substring(0, slashIdx + 1) : ""; + log.info("[ZipSkillFetcher] Found SKILL.md at: {}", normalizedName); + } else { + raws.add(new RawEntry(normalizedName, content)); } zis.closeEntry(); - - String normalizedName = entryPath.toString().replace('\\', '/'); - String relativeName = normalizedName; - if (!skillMdPrefix.isEmpty() && normalizedName.startsWith(skillMdPrefix)) { - relativeName = normalizedName.substring(skillMdPrefix.length()); - } - - if (relativeName.startsWith("references/")) { - references.put(relativeName.substring("references/".length()), content); - } else if (relativeName.startsWith("scripts/")) { - scripts.put(relativeName.substring("scripts/".length()), content); - } } } @@ -167,6 +198,61 @@ public class ZipSkillFetcher { throw new IllegalArgumentException("ZIP does not contain SKILL.md"); } + Map references = new HashMap<>(); + Map scripts = new HashMap<>(); + + for (RawEntry raw : raws) { + String relative = raw.name(); + if (!skillMdPrefix.isEmpty() && relative.startsWith(skillMdPrefix)) { + relative = relative.substring(skillMdPrefix.length()); + } + + if (relative.startsWith("references/")) { + references.put(relative.substring("references/".length()), raw.content()); + } else if (relative.startsWith("scripts/")) { + scripts.put(relative.substring("scripts/".length()), raw.content()); + } else if (!relative.contains("/")) { + // Sibling of SKILL.md (post-prefix-strip). Some real-world + // packagers — notably the official tencent-meeting-mcp.zip — + // put setup.sh at the package root instead of under scripts/. + // Fall back to extension-based classification so those zips + // install cleanly without forcing the user to repackage. + String classified = classifyRootFile(relative); + if ("scripts".equals(classified)) { + scripts.put(relative, raw.content()); + log.info("[ZipSkillFetcher] Classified root-level entry '{}' as script by extension", relative); + } else if ("references".equals(classified)) { + references.put(relative, raw.content()); + log.info("[ZipSkillFetcher] Classified root-level entry '{}' as reference by extension", relative); + } else { + log.warn("[ZipSkillFetcher] Ignoring root-level entry with unknown extension: {}", raw.name()); + } + } else { + log.warn("[ZipSkillFetcher] Ignoring entry outside references/ or scripts/: {} (skill prefix={})", + raw.name(), skillMdPrefix.isEmpty() ? "" : skillMdPrefix); + } + } + return new ExtractedSkill(skillMdContent, references, scripts); } + + /** + * Classify a root-level file (sibling of SKILL.md, no directory prefix) + * by extension. Returns {@code "scripts"} / {@code "references"} for + * recognized extensions, {@code null} for everything else. + * + *

Only invoked for entries that are NOT already nested under + * {@code scripts/} or {@code references/}, so well-formed packages + * are unaffected. + */ + private static String classifyRootFile(String fileName) { + if (fileName == null) return null; + String lower = fileName.toLowerCase(); + int dot = lower.lastIndexOf('.'); + if (dot < 0) return null; + String ext = lower.substring(dot); + if (SCRIPT_EXTENSIONS.contains(ext)) return "scripts"; + if (REFERENCE_EXTENSIONS.contains(ext)) return "references"; + return null; + } } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/installer/model/InstallRequest.java b/mateclaw-server/src/main/java/vip/mate/skill/installer/model/InstallRequest.java index f51ec93e..47143c17 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/installer/model/InstallRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/installer/model/InstallRequest.java @@ -24,4 +24,13 @@ public class InstallRequest { /** 若同名 skill 已存在,是否覆盖 */ private Boolean overwrite = false; + + /** + * Bypass the empty-bundle prune guard. Default {@code false} keeps + * existing scripts/references when the new bundle has zero entries + * for that bucket — protects against malformed uploads. Set to + * {@code true} only when you really want to clear out a bucket via + * an intentionally empty bundle. + */ + private Boolean forcePrune = false; } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/mcp/McpSkillBridge.java b/mateclaw-server/src/main/java/vip/mate/skill/mcp/McpSkillBridge.java index 811af15c..17766a9d 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/mcp/McpSkillBridge.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/mcp/McpSkillBridge.java @@ -10,6 +10,7 @@ import vip.mate.skill.model.SkillEntity; import vip.mate.skill.runtime.model.ResolvedSkill; import vip.mate.tool.mcp.model.McpServerEntity; import vip.mate.tool.mcp.runtime.McpClientManager; +import vip.mate.tool.mcp.runtime.McpToolNameResolver; import vip.mate.tool.mcp.service.McpServerService; import java.util.ArrayList; @@ -134,8 +135,7 @@ public class McpSkillBridge { s.setId(virtualIdFor(server)); s.setName(slugify(server.getName())); s.setNameEn(displayName(server)); - s.setNameZh(server.getDescription() != null && !server.getDescription().isBlank() - ? displayName(server) : null); + s.setNameZh(displayName(server)); s.setDescription(buildDescription(server)); s.setSkillType("mcp"); s.setIcon(iconFor(server)); @@ -146,12 +146,18 @@ public class McpSkillBridge { s.setTags("mcp"); s.setSecurityScanStatus("PASSED"); // MCP servers don't go through SkillSecurityService s.setConfigJson(buildConfigJson(server)); - s.setManifestJson(serializeManifest(buildManifest(server))); + s.setManifestJson(serializeManifest(buildManifestFrom(server, readToolRawNames(server)))); return s; } private ResolvedSkill serverToResolved(McpServerEntity server) { - SkillManifest manifest = buildManifest(server); + List rawNames = readToolRawNames(server); + Map toolDisplayNames = new LinkedHashMap<>(); + for (String raw : rawNames) { + String prefixed = McpToolNameResolver.prefixedName(server.getId(), raw); + toolDisplayNames.put(prefixed, prefixed + " (" + raw + ")"); + } + SkillManifest manifest = buildManifestFrom(server, rawNames); boolean connected = "connected".equalsIgnoreCase(nullSafe(server.getLastStatus())); boolean errored = "error".equalsIgnoreCase(nullSafe(server.getLastStatus())) || (server.getLastError() != null && !server.getLastError().isBlank()); @@ -192,27 +198,33 @@ public class McpSkillBridge { .manifest(manifest) .featureStatuses(featureStatuses) .activeFeatures(active) + .toolDisplayNames(toolDisplayNames) .build(); } /** - * Auto-generate the §10.2 Q2 minimal manifest from the live MCP - * server. Tool list is the union of discovered MCP tools; one - * synthetic feature {@code default} carries them so the standard - * features-aware gate light up correctly. + * Auto-generate the minimal manifest from the MCP server's most-recent + * tool snapshot. The tool list is sourced in priority order: + *

    + *
  1. {@code mate_mcp_server.tools_cache_json} — present whenever the + * server has connected at least once. Lets the picker stay + * populated through brief disconnects.
  2. + *
  3. The runtime in-memory cache (current connection's + * {@code listTools()} result).
  4. + *
+ * + *

Tool names emitted into {@code manifest.allowedTools} go through + * {@link McpToolNameResolver#prefixedName(long, String)} so they match + * the runtime callback names registered by + * {@link McpClientManager#getAllToolCallbacks()}. Without this, a + * resolved skill's effective allowlist would carry raw names that + * don't appear in any agent's callbacks at chat time, and the LLM + * would see no MCP tools even though the bindings were saved. */ - private SkillManifest buildManifest(McpServerEntity server) { - List toolNames = new ArrayList<>(); - try { - List discovered = mcpClientManager.getServerTools(server.getId()); - for (McpSchema.Tool t : discovered) { - if (t == null) continue; - String n = t.name(); - if (n != null && !n.isBlank()) toolNames.add(n); - } - } catch (Exception e) { - log.debug("MCP bridge manifest build: getServerTools({}) failed: {}", - server.getId(), e.getMessage()); + private SkillManifest buildManifestFrom(McpServerEntity server, List rawNames) { + List toolNames = new ArrayList<>(rawNames.size()); + for (String raw : rawNames) { + toolNames.add(McpToolNameResolver.prefixedName(server.getId(), raw)); } SkillManifest.FeatureDef defaultFeature = SkillManifest.FeatureDef.builder() @@ -253,6 +265,58 @@ public class McpSkillBridge { .build(); } + /** + * Resolve the raw tool name list for a server with cache-first / live-fallback + * semantics. Returns an empty list (never null) so the manifest builder + * stays simple. + */ + private List readToolRawNames(McpServerEntity server) { + List fromCache = parseCachedToolNames(server.getToolsCacheJson()); + if (!fromCache.isEmpty()) { + return fromCache; + } + try { + List discovered = mcpClientManager.getServerTools(server.getId()); + List names = new ArrayList<>(discovered.size()); + for (McpSchema.Tool t : discovered) { + if (t == null) continue; + String n = t.name(); + if (n != null && !n.isBlank()) names.add(n); + } + return names; + } catch (Exception e) { + log.debug("MCP bridge manifest build: getServerTools({}) failed: {}", + server.getId(), e.getMessage()); + return List.of(); + } + } + + /** + * Parse the {@code tools_cache_json} column written by + * {@code McpServerService} after each successful connect. Returns an + * empty list if the column is null/blank/malformed — the bridge is + * required to keep working when the cache hasn't been populated yet + * (e.g. first-ever connect just succeeded a moment ago). + */ + private List parseCachedToolNames(String json) { + if (json == null || json.isBlank()) { + return List.of(); + } + try { + cn.hutool.json.JSONArray arr = cn.hutool.json.JSONUtil.parseArray(json); + List out = new ArrayList<>(arr.size()); + for (Object obj : arr) { + if (!(obj instanceof cn.hutool.json.JSONObject jo)) continue; + String name = jo.getStr("name"); + if (name != null && !name.isBlank()) out.add(name); + } + return out; + } catch (Exception e) { + log.debug("MCP bridge: failed to parse tools_cache_json: {}", e.getMessage()); + return List.of(); + } + } + private String slugify(String raw) { if (raw == null) return ""; return raw.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9_-]", "-"); diff --git a/mateclaw-server/src/main/java/vip/mate/skill/model/SkillEntity.java b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillEntity.java index 0199066a..44c29730 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/model/SkillEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillEntity.java @@ -97,7 +97,17 @@ public class SkillEntity { /** 标签(逗号分隔) */ private String tags; - /** RFC-023:来源对话 ID(Agent 自治合成时记录) */ + /** + * Owning workspace. The DB column has existed since the baseline schema + * (default = 1) but the field was missing from the entity, so MyBatis + * Plus silently ignored both reads and writes. Surfacing it here lets + * binding-time tenancy checks see the value; default behavior on insert + * remains "fall through to the column DEFAULT" because the field stays + * {@code null} in the no-arg create path. + */ + private Long workspaceId; + + /** 来源对话 ID(Agent 自治合成时记录) */ private String sourceConversationId; /** diff --git a/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java new file mode 100644 index 00000000..6169fa25 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java @@ -0,0 +1,53 @@ +package vip.mate.skill.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * One file inside a skill bundle (an entry under {@code scripts/} or + * {@code references/}). + *

+ * The database is the canonical store. {@code SkillFileSyncer} mirrors + * each row to the local workspace cache so {@code SkillScriptTool} and + * other directory-aware consumers see the file on disk regardless of + * which node accepted the original upload. + * + * @author MateClaw Team + */ +@Data +@TableName("mate_skill_file") +public class SkillFileEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + /** Owning skill (FK to {@code mate_skill.id}). */ + private Long skillId; + + /** + * Path relative to the skill workspace root, always starting with + * {@code scripts/} or {@code references/}. Forward slashes only. + */ + private String filePath; + + /** UTF-8 text content. Per-file size bounded by ZipSkillFetcher (1MB). */ + private String content; + + /** Length of {@link #content} in bytes — kept so listings can sort/audit without loading the blob. */ + private Integer contentSize; + + /** SHA-256 of {@link #content}; used by the syncer to skip no-op writes. */ + private String sha256; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; +} diff --git a/mateclaw-server/src/main/java/vip/mate/skill/repository/SkillFileMapper.java b/mateclaw-server/src/main/java/vip/mate/skill/repository/SkillFileMapper.java new file mode 100644 index 00000000..1adb9466 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/skill/repository/SkillFileMapper.java @@ -0,0 +1,20 @@ +package vip.mate.skill.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Delete; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import vip.mate.skill.model.SkillFileEntity; + +/** + * Mapper for {@link SkillFileEntity}. + * + * @author MateClaw Team + */ +@Mapper +public interface SkillFileMapper extends BaseMapper { + + /** Drop every file row owned by the given skill — used on hard-delete. */ + @Delete("DELETE FROM mate_skill_file WHERE skill_id = #{skillId}") + int deleteBySkillId(@Param("skillId") Long skillId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillPackageResolver.java b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillPackageResolver.java index 36b6531b..d6ee4115 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillPackageResolver.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillPackageResolver.java @@ -335,6 +335,7 @@ public class SkillPackageResolver { .enabled(Boolean.TRUE.equals(entity.getEnabled())) .icon(entity.getIcon()) .builtin(Boolean.TRUE.equals(entity.getBuiltin())) + .createTime(entity.getCreateTime()) .build(); } @@ -375,6 +376,7 @@ public class SkillPackageResolver { .enabled(Boolean.TRUE.equals(entity.getEnabled())) .icon(entity.getIcon()) .builtin(Boolean.TRUE.equals(entity.getBuiltin())) + .createTime(entity.getCreateTime()) .build(); } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillRuntimeService.java b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillRuntimeService.java index 77dd8a07..008d4553 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillRuntimeService.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillRuntimeService.java @@ -354,16 +354,20 @@ public class SkillRuntimeService { Long agentId) { List activeSkills; if (boundSkillIds != null) { - // Per-agent 过滤:从全局 enabled skills 中按 ID 过滤。RFC-090 - // §14.1 — must use the same features-aware gate as - // refreshActiveSkills() so legacy dependencyReady drift - // doesn't silently let setup-needed manifest skills through - // (or hide partially-ready features that should be visible). - List enabledSkills = skillService.listEnabledSkills(); - activeSkills = enabledSkills.stream() - .filter(s -> boundSkillIds.contains(s.getId())) - .map(packageResolver::resolve) - .filter(SkillRuntimeService::passesActiveGate) + // Per-agent filter: pick the agent's bound subset from the + // already-merged active set (real + MCP/ACP virtual). Using + // getActiveSkills() — instead of a fresh + // skillService.listEnabledSkills() walk — is what makes bound + // virtual skills surface in the prompt catalog. The earlier + // implementation only looked at mate_skill rows, so a user + // who explicitly checked an MCP/ACP card in the agent picker + // got its tools (via AgentBindingService.getEffectiveToolNames) + // but lost the corresponding `## Skills` catalog row, which + // confused the LLM when it tried to dispatch by skill name. + // Cache-backed get + same passesActiveGate semantics, so this + // is strictly additive for real skills. + activeSkills = getActiveSkills().stream() + .filter(s -> s.getId() != null && boundSkillIds.contains(s.getId())) .collect(java.util.stream.Collectors.toList()); } else { activeSkills = getActiveSkills(); @@ -391,10 +395,17 @@ public class SkillRuntimeService { int descLimit = promptDescriptionLimit(maxInputTokens); Set recentNames = usageService.recentLoadedSkillNames(agentId, 8); Set frequentNames = usageService.frequentlyLoadedSkillNames(8); + // Boost freshly installed skills for a short window so a skill the + // user *just* added is visible in the compact catalog before it has + // any usage history. Without this, qwen-turbo-style 8-entry budgets + // hide new skills behind 40+ existing ones, and the LLM tells the + // user "no such skill" minutes after they uploaded it. + java.time.LocalDateTime recencyCutoff = java.time.LocalDateTime.now().minus(NEW_SKILL_BOOST_WINDOW); List sorted = SkillCatalogSorter.sortResolved(visibleSkills, SkillCatalogSort.RECOMMENDED) .stream() .sorted(java.util.Comparator - .comparingInt((ResolvedSkill s) -> recentNames.contains(s.getName()) ? 0 : 1) + .comparingInt((ResolvedSkill s) -> isRecentlyInstalled(s, recencyCutoff) ? 0 : 1) + .thenComparingInt(s -> recentNames.contains(s.getName()) ? 0 : 1) .thenComparingInt(s -> frequentNames.contains(s.getName()) ? 0 : 1) .thenComparing(SkillCatalogSorter.resolvedComparator(SkillCatalogSort.RECOMMENDED))) .toList(); @@ -412,7 +423,12 @@ public class SkillRuntimeService { sb.append("\n\n## Skills\n"); sb.append("This is a compact catalog. If a listed skill matches the task, "); sb.append("first call `readSkillFile(skillName=, filePath=\"SKILL.md\")` and follow its instructions. "); - sb.append("If none of these skills match, call `listAvailableSkills()` to inspect the broader catalog. "); + sb.append("If none of these skills match, call `listAvailableSkills()` to inspect the broader catalog "); + sb.append("(it accepts `keyword=` and `limit=` up to 50 — use them to search by topic "); + sb.append("when the default page is truncated). "); + sb.append("If the user names a specific skill that isn't in this table, "); + sb.append("call `readSkillFile(skillName=\"\", filePath=\"SKILL.md\")` directly — "); + sb.append("the catalog above is intentionally compact and doesn't list every active skill. "); sb.append("Skills are documentation packages — calling a skill name as a tool will fail. "); sb.append("Skills with a `scripts/` directory expose `runSkillScript`; SKILL.md will name the script when needed.\n\n"); sb.append("| Skill | Status | Description |\n"); @@ -454,6 +470,28 @@ public class SkillRuntimeService { return tools == null || tools.isEmpty() || effectiveToolNames.containsAll(tools); } + /** + * Treat skills installed within this window as "new" for the prompt + * catalog ranker. Long enough that a user who installs on Friday and + * comes back Monday still sees the boost; short enough that the + * catalog reverts to usage-based ordering before the boost slot + * crowds out genuinely useful skills. + */ + public static final java.time.Duration NEW_SKILL_BOOST_WINDOW = java.time.Duration.ofDays(7); + + /** + * Returns true if the skill's row was created after {@code cutoff}. + * Builtins and virtual MCP/ACP skills typically have no createTime; + * they are not boosted (the user didn't just install them). Public so + * the user-facing {@code listAvailableSkills} catalog can apply the + * same boost as the prompt enhancement. + */ + public static boolean isRecentlyInstalled(ResolvedSkill skill, java.time.LocalDateTime cutoff) { + if (skill == null || skill.getCreateTime() == null) return false; + if (skill.isBuiltin()) return false; + return skill.getCreateTime().isAfter(cutoff); + } + private static int promptCatalogEntryLimit(Integer maxInputTokens) { int max = maxInputTokens != null && maxInputTokens > 0 ? maxInputTokens : 8192; if (max <= 8192) return 8; diff --git a/mateclaw-server/src/main/java/vip/mate/skill/runtime/model/ResolvedSkill.java b/mateclaw-server/src/main/java/vip/mate/skill/runtime/model/ResolvedSkill.java index 84f9bbaf..6df78e20 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/runtime/model/ResolvedSkill.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/runtime/model/ResolvedSkill.java @@ -6,6 +6,7 @@ import lombok.Data; import vip.mate.skill.manifest.SkillManifest; import java.nio.file.Path; +import java.time.LocalDateTime; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -72,6 +73,16 @@ public class ResolvedSkill { @Builder.Default private boolean builtin = false; + /** + * Skill row create timestamp, copied from {@code mate_skill.create_time}. + * Used by the prompt-catalog ranker to surface freshly installed skills + * before they accumulate any usage stats — without this, a brand-new + * skill stays invisible behind the recent/frequent/alphabetical sort + * and the LLM ends up replying "no such skill" right after the user + * installed it. Null for virtual MCP/ACP skills that don't own a row. + */ + private LocalDateTime createTime; + // ==================== 安全扫描状态 ==================== /** 是否被安全扫描阻断 */ @@ -130,6 +141,25 @@ public class ResolvedSkill { @Builder.Default private Set activeFeatures = Set.of(); + /** + * Per-tool display-name decoration table, keyed by the prefixed callback + * name and valued by the human-readable form (e.g. + * {@code "mcp_4_fs_a1b2c3"} → {@code "mcp_4_fs_a1b2c3 (read_file)"}). + * + *

Populated by skill source providers that have a recoverable raw + * name (currently MCP-bridged skills); other sources leave it empty, + * in which case {@link #getEffectiveAllowedToolsDisplay()} falls + * through to the prefixed names unchanged. + * + *

Held internally rather than serialized: the wire shape exposes + * the decorated set via the derived getter, which keeps the + * source-of-truth (the feature filter in + * {@link #getEffectiveAllowedTools()}) in one place. + */ + @JsonIgnore + @Builder.Default + private Map toolDisplayNames = Map.of(); + /** RFC-090 §14.1 — replacement filter for {@code dependencyReady}. */ public boolean hasAnyActiveFeature() { return activeFeatures != null && !activeFeatures.isEmpty(); @@ -200,6 +230,26 @@ public class ResolvedSkill { return out; } + /** + * Display-friendly companion to {@link #getEffectiveAllowedTools()}. + * Each prefixed callback name is replaced by its decorated form (e.g. + * {@code "mcp_4_fs_a1b2c3 (read_file)"}) when {@link #toolDisplayNames} + * carries an entry for it; names without a decoration entry are kept + * verbatim. Feature-filter semantics match the prefixed getter, so a + * tool that is hidden by a SETUP_NEEDED feature stays hidden here too. + */ + public Set getEffectiveAllowedToolsDisplay() { + Set base = getEffectiveAllowedTools(); + if (base.isEmpty() || toolDisplayNames == null || toolDisplayNames.isEmpty()) { + return base; + } + Set out = new LinkedHashSet<>(base.size()); + for (String name : base) { + out.add(toolDisplayNames.getOrDefault(name, name)); + } + return out; + } + // ==================== 综合状态 ==================== /** diff --git a/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java new file mode 100644 index 00000000..267db5a6 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java @@ -0,0 +1,178 @@ +package vip.mate.skill.service; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.skill.model.SkillFileEntity; +import vip.mate.skill.repository.SkillFileMapper; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Persistence layer for skill bundle files. + *

+ * Treated as the canonical store: every install writes the full set of + * scripts/references rows here, and {@code SkillFileSyncer} mirrors them + * to the local workspace cache on every node so script execution works + * across a multi-instance deployment that shares one database. + * + * @author MateClaw Team + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class SkillFileService { + + private final SkillFileMapper mapper; + + /** All file rows owned by a skill. */ + public List listBySkillId(Long skillId) { + if (skillId == null) return List.of(); + QueryWrapper q = new QueryWrapper<>(); + q.eq("skill_id", skillId); + return mapper.selectList(q); + } + + /** Compute SHA-256 hex of a UTF-8 string (used for idempotent diffs). */ + public static String sha256Hex(String content) { + if (content == null) content = ""; + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(digest.length * 2); + for (byte b : digest) sb.append(String.format("%02x", b)); + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 unavailable on this JVM", e); + } + } + + /** + * Replace the skill's full file set with {@code newFiles}, using + * write-then-prune semantics to mirror the on-disk applyBundleFiles. + * + *

Empty-bundle guard: if {@code newFiles} contains zero entries + * for a bucket (scripts/ or references/) and there are existing rows + * for that bucket, the rows are preserved unless {@code force=true}. + * This blocks the same data-loss scenario that tripped up the FS path. + * + * @param skillId owning skill id + * @param newFiles new full file set, keyed by path under workspace root + * (e.g. {@code "scripts/run.py"}) + * @param force bypass empty-bundle guard + */ + @Transactional + public ApplyResult applyBundleFiles(Long skillId, Map newFiles, boolean force) { + if (skillId == null) { + return new ApplyResult(0, 0, false, false); + } + + Map incoming = newFiles == null ? Map.of() : newFiles; + boolean newHasScripts = bucketHasEntries(incoming, "scripts/"); + boolean newHasRefs = bucketHasEntries(incoming, "references/"); + + List existing = listBySkillId(skillId); + boolean existingHasScripts = existing.stream().anyMatch(e -> e.getFilePath() != null && e.getFilePath().startsWith("scripts/")); + boolean existingHasRefs = existing.stream().anyMatch(e -> e.getFilePath() != null && e.getFilePath().startsWith("references/")); + + boolean preserveScripts = !newHasScripts && existingHasScripts && !force; + boolean preserveRefs = !newHasRefs && existingHasRefs && !force; + + Map existingByPath = new HashMap<>(); + for (SkillFileEntity e : existing) existingByPath.put(e.getFilePath(), e); + + Set keepPaths = new HashSet<>(); + if (preserveScripts) { + for (SkillFileEntity e : existing) { + if (e.getFilePath() != null && e.getFilePath().startsWith("scripts/")) { + keepPaths.add(e.getFilePath()); + } + } + } + if (preserveRefs) { + for (SkillFileEntity e : existing) { + if (e.getFilePath() != null && e.getFilePath().startsWith("references/")) { + keepPaths.add(e.getFilePath()); + } + } + } + keepPaths.addAll(incoming.keySet()); + + int written = 0; + LocalDateTime now = LocalDateTime.now(); + for (var entry : incoming.entrySet()) { + String path = entry.getKey(); + String content = entry.getValue() == null ? "" : entry.getValue(); + String hash = sha256Hex(content); + int size = content.getBytes(StandardCharsets.UTF_8).length; + + SkillFileEntity prior = existingByPath.get(path); + if (prior == null) { + SkillFileEntity row = new SkillFileEntity(); + row.setSkillId(skillId); + row.setFilePath(path); + row.setContent(content); + row.setContentSize(size); + row.setSha256(hash); + row.setCreateTime(now); + row.setUpdateTime(now); + mapper.insert(row); + written++; + } else if (!hash.equals(prior.getSha256())) { + prior.setContent(content); + prior.setContentSize(size); + prior.setSha256(hash); + prior.setUpdateTime(now); + mapper.updateById(prior); + written++; + } + } + + int pruned = 0; + for (SkillFileEntity e : existing) { + if (!keepPaths.contains(e.getFilePath())) { + mapper.deleteById(e.getId()); + pruned++; + } + } + + if (preserveScripts) { + log.warn("Refused to prune scripts/ for skill_id={} — new bundle is empty. Pass force=true to override.", skillId); + } + if (preserveRefs) { + log.warn("Refused to prune references/ for skill_id={} — new bundle is empty. Pass force=true to override.", skillId); + } + + return new ApplyResult(written, pruned, preserveScripts, preserveRefs); + } + + /** Drop every file row for a skill (used on hard-delete). */ + @Transactional + public int deleteAllForSkill(Long skillId) { + if (skillId == null) return 0; + return mapper.deleteBySkillId(skillId); + } + + private boolean bucketHasEntries(Map files, String prefix) { + for (String key : files.keySet()) { + if (key != null && key.startsWith(prefix)) return true; + } + return false; + } + + /** Outcome of {@link #applyBundleFiles}. */ + public record ApplyResult(int rowsWritten, + int rowsPruned, + boolean scriptsPreservedDueToEmptyBundle, + boolean referencesPreservedDueToEmptyBundle) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/skill/service/SkillService.java b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillService.java index 364bc705..f20b8612 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/service/SkillService.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillService.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import vip.mate.exception.MateClawException; import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.repository.SkillFileMapper; import vip.mate.skill.repository.SkillMapper; import vip.mate.skill.runtime.SkillCatalogSort; import vip.mate.skill.runtime.SkillCatalogSorter; @@ -42,6 +43,7 @@ import java.util.stream.Collectors; public class SkillService { private final SkillMapper skillMapper; + private final SkillFileMapper skillFileMapper; private final SkillWorkspaceManager workspaceManager; private final SkillWorkspaceProperties workspaceProperties; private final SkillSecretService skillSecretService; @@ -300,6 +302,29 @@ public class SkillService { * * 仍不允许:name / version / author / skillType / builtin —— 这些是身份字段, * 改动会破坏绑定与解析。 + * + *

The UI sends a partial body that only contains the fields the + * user edited (Identity edit → {@code nameZh/nameEn/description/tags/icon}; + * Body edit → {@code skillContent}, plus optional {@code sourceCode}). + * Every other field on the deserialized entity is {@code null}. + * + *

{@link SkillEntity} declares several + * {@code @TableField(updateStrategy = FieldStrategy.ALWAYS)} columns + * — {@code name_zh}, {@code name_en}, {@code config_json}, + * {@code source_code}, {@code skill_content}, {@code manifest_json}, + * {@code security_scan_result}. Calling + * {@code skillMapper.updateById(partial)} would tell MyBatis Plus to + * write {@code NULL} into every ALWAYS column missing from the + * partial, wiping perfectly valid content on every save. The earlier + * #45 fix only protected the resolver's scan write-back; this path + * was still exposed (and surfaced as issue #93 when a partial PUT + * also took the workspace-sync branch with a {@code null} name and + * NPE'd inside {@code sanitizeName}). + * + *

Fix: merge non-null fields from the partial onto a copy of the + * existing row, then persist the merged entity. Same shape as the + * builtin branch above, just with a wider whitelist for dynamic + * skills. */ public SkillEntity updateSkill(SkillEntity skill) { SkillEntity existing = getSkill(skill.getId()); @@ -307,7 +332,7 @@ public class SkillService { if (Boolean.TRUE.equals(existing.getBuiltin())) { // Functional fields existing.setEnabled(skill.getEnabled() != null ? skill.getEnabled() : existing.getEnabled()); - existing.setConfigJson(skill.getConfigJson()); + if (skill.getConfigJson() != null) existing.setConfigJson(skill.getConfigJson()); existing.setDescription(skill.getDescription() != null ? skill.getDescription() : existing.getDescription()); if (skill.getSkillContent() != null) { existing.setSkillContent(skill.getSkillContent()); @@ -335,20 +360,36 @@ public class SkillService { return existing; } - // 非内置技能:允许修改所有字段,但不允许改为 builtin - skill.setBuiltin(false); - skillMapper.updateById(skill); - log.info("Updated skill: {}", skill.getName()); + // 非内置技能:merge non-null fields from the partial onto the + // existing row. name / skillType / builtin stay locked because + // they're identity fields whose change would orphan bindings + // and break the resolver. + if (skill.getDescription() != null) existing.setDescription(skill.getDescription()); + if (skill.getIcon() != null) existing.setIcon(skill.getIcon()); + if (skill.getVersion() != null && !skill.getVersion().isBlank()) existing.setVersion(skill.getVersion()); + if (skill.getAuthor() != null) existing.setAuthor(skill.getAuthor()); + if (skill.getEnabled() != null) existing.setEnabled(skill.getEnabled()); + if (skill.getTags() != null) existing.setTags(skill.getTags()); + if (skill.getNameZh() != null) existing.setNameZh(skill.getNameZh()); + if (skill.getNameEn() != null) existing.setNameEn(skill.getNameEn()); + if (skill.getConfigJson() != null) existing.setConfigJson(skill.getConfigJson()); + if (skill.getSourceCode() != null) existing.setSourceCode(skill.getSourceCode()); + if (skill.getSkillContent() != null) existing.setSkillContent(skill.getSkillContent()); + if (skill.getManifestJson() != null) existing.setManifestJson(skill.getManifestJson()); + existing.setBuiltin(false); + + skillMapper.updateById(existing); + log.info("Updated skill: {}", existing.getName()); // 若 skillContent 变更且约定工作区存在,同步 SKILL.md - syncSkillContentToWorkspace(skill); + syncSkillContentToWorkspace(existing); // 刷新 runtime cache if (runtimeService != null) { runtimeService.refreshActiveSkills(); } - return skill; + return existing; } /** @@ -401,6 +442,10 @@ public class SkillService { "内置技能不可硬删除: " + skill.getName()); } skillMapper.hardDeleteById(id); // bypass the logical-delete flag + int filesDropped = skillFileMapper.deleteBySkillId(id); + if (filesDropped > 0) { + log.info("Hard-deleted {} bundle file row(s) for skill {}", filesDropped, skill.getName()); + } log.info("Hard-deleted skill (physical delete + purge): {}", skill.getName()); // RFC-091 settings bridge — purge any per-skill secrets so a diff --git a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java new file mode 100644 index 00000000..9acafb7a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java @@ -0,0 +1,209 @@ +package vip.mate.skill.workspace; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.model.SkillFileEntity; +import vip.mate.skill.service.SkillFileService; +import vip.mate.skill.service.SkillService; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Mirrors canonical {@code mate_skill_file} rows down to each node's local + * workspace cache so {@code scripts/} and {@code references/} files exist + * on disk wherever the skill might run. + * + *

Also runs a one-time backfill: for any skill that has on-disk files + * but no DB rows (typically pre-V112 installs), the local files are read + * up into the DB so the canonical store catches up to reality. Backfill + * is content-hash idempotent and safe to invoke repeatedly. + * + *

Triggered: + *

    + *
  • At startup, after the bundled-skill syncer (see + * {@link SkillWorkspaceBootstrapRunner}).
  • + *
  • On-demand via the admin endpoint {@code POST /api/v1/skills/{id}/sync-files}.
  • + *
+ * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class SkillFileSyncer { + + private final SkillService skillService; + private final SkillFileService skillFileService; + private final SkillWorkspaceManager workspaceManager; + + /** Aggregate counters for one full sync pass. */ + public record SyncReport(int skillsConsidered, + int skillsBackfilled, + int filesMaterialized, + int filesAlreadyCurrent, + int filesBackfilledFromDisk) {} + + /** Sync every active skill once. Idempotent. */ + public SyncReport syncAll() { + List skills = skillService.listSkills(); + int considered = 0; + int backfilled = 0; + int materialized = 0; + int current = 0; + int diskBackfilled = 0; + + for (SkillEntity skill : skills) { + if (skill.getId() == null || skill.getName() == null) continue; + considered++; + var per = syncOne(skill); + materialized += per.filesMaterialized(); + current += per.filesAlreadyCurrent(); + diskBackfilled += per.filesBackfilledFromDisk(); + if (per.didBackfillFromDisk()) backfilled++; + } + + if (considered > 0) { + log.info("SkillFileSyncer pass: skills={}, materialized={}, current={}, " + + "backfilledFromDisk(skills={}, files={})", + considered, materialized, current, backfilled, diskBackfilled); + } + return new SyncReport(considered, backfilled, materialized, current, diskBackfilled); + } + + /** Per-skill sync outcome. */ + public record PerSkillReport(int filesMaterialized, + int filesAlreadyCurrent, + int filesBackfilledFromDisk, + boolean didBackfillFromDisk) {} + + /** + * Sync a single skill: backfill DB from FS if DB is empty and FS has + * files, then materialize DB rows down to FS so any missing/stale files + * are restored. + */ + public PerSkillReport syncOne(SkillEntity skill) { + Path workspaceDir = workspaceManager.resolveConventionPath(skill.getName()); + List dbFiles = skillFileService.listBySkillId(skill.getId()); + + boolean didBackfill = false; + int backfilled = 0; + if (dbFiles.isEmpty()) { + backfilled = backfillFromDiskIfNeeded(skill, workspaceDir); + if (backfilled > 0) { + didBackfill = true; + dbFiles = skillFileService.listBySkillId(skill.getId()); + } + } + + int materialized = 0; + int alreadyCurrent = 0; + for (SkillFileEntity row : dbFiles) { + switch (materializeOne(workspaceDir, row)) { + case WROTE -> materialized++; + case CURRENT -> alreadyCurrent++; + case SKIPPED -> { + /* unsafe path / IO failure already logged */ + } + } + } + + return new PerSkillReport(materialized, alreadyCurrent, backfilled, didBackfill); + } + + private enum MaterializeOutcome { WROTE, CURRENT, SKIPPED } + + private MaterializeOutcome materializeOne(Path workspaceDir, SkillFileEntity row) { + String relative = row.getFilePath(); + if (relative == null || relative.isBlank()) return MaterializeOutcome.SKIPPED; + if (!relative.startsWith("references/") && !relative.startsWith("scripts/")) { + log.warn("Skipping skill_file row {} — path outside scripts/ or references/: {}", + row.getId(), relative); + return MaterializeOutcome.SKIPPED; + } + if (relative.contains("..")) { + log.warn("Skipping skill_file row {} — suspicious path: {}", row.getId(), relative); + return MaterializeOutcome.SKIPPED; + } + + Path target = workspaceDir.resolve(relative).normalize(); + if (!target.startsWith(workspaceDir.normalize())) { + log.warn("Skipping skill_file row {} — escapes workspace: {}", row.getId(), relative); + return MaterializeOutcome.SKIPPED; + } + + try { + String content = row.getContent() == null ? "" : row.getContent(); + if (Files.exists(target)) { + String onDisk = Files.readString(target, StandardCharsets.UTF_8); + if (SkillFileService.sha256Hex(onDisk).equals(row.getSha256())) { + return MaterializeOutcome.CURRENT; + } + } + Files.createDirectories(target.getParent()); + Files.writeString(target, content, StandardCharsets.UTF_8); + return MaterializeOutcome.WROTE; + } catch (IOException e) { + log.warn("Failed to materialize skill_file {} → {}: {}", row.getId(), target, e.getMessage()); + return MaterializeOutcome.SKIPPED; + } + } + + /** + * One-time ingestion of pre-V112 on-disk files into the canonical + * {@code mate_skill_file} table. Only runs when the skill has zero + * file rows; subsequent installs go through the installer's normal + * write-to-both-stores path. + */ + private int backfillFromDiskIfNeeded(SkillEntity skill, Path workspaceDir) { + if (!Files.exists(workspaceDir) || !Files.isDirectory(workspaceDir)) return 0; + + List roots = new ArrayList<>(2); + Path scripts = workspaceDir.resolve("scripts"); + Path references = workspaceDir.resolve("references"); + if (Files.isDirectory(scripts)) roots.add(scripts); + if (Files.isDirectory(references)) roots.add(references); + if (roots.isEmpty()) return 0; + + java.util.Map ingested = new java.util.LinkedHashMap<>(); + Set seen = new HashSet<>(); + for (Path root : roots) { + String prefix = workspaceDir.relativize(root).toString().replace('\\', '/') + "/"; + try (var stream = Files.walk(root)) { + List files = stream.filter(Files::isRegularFile).toList(); + for (Path f : files) { + String relative = workspaceDir.relativize(f).toString().replace('\\', '/'); + if (!relative.startsWith(prefix)) continue; + if (!seen.add(relative)) continue; + try { + String content = Files.readString(f, StandardCharsets.UTF_8); + ingested.put(relative, content); + } catch (IOException e) { + log.warn("Backfill skipped {} (read failed: {})", f, e.getMessage()); + } + } + } catch (IOException e) { + log.warn("Backfill walk failed for {}: {}", root, e.getMessage()); + } + } + + if (ingested.isEmpty()) return 0; + skillFileService.applyBundleFiles(skill.getId(), ingested, false); + log.info("Backfilled {} bundle file(s) into mate_skill_file for skill '{}' (id={})", + ingested.size(), skill.getName(), skill.getId()); + + // Touch the workspace event so other observers (e.g. runtime cache) refresh. + // Use a synthetic event type — INSTALLED is the closest existing match. + skill.setUpdateTime(LocalDateTime.now()); + return ingested.size(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceBootstrapRunner.java b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceBootstrapRunner.java index d60c7695..029cda4a 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceBootstrapRunner.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceBootstrapRunner.java @@ -10,36 +10,51 @@ import org.springframework.stereotype.Component; import java.util.List; /** - * Skill 工作区启动初始化 - *

- * 1. 确保 workspace root 目录存在 - * 2. 将 classpath 下预置技能同步到 workspace - * - 首次:创建并同步 - * - 后续:比对 SKILL.md frontmatter 中的 version 字段, - * bundled version 更高时归档旧版本并覆盖升级 - *

- * Order(195) — 在 DatabaseBootstrapRunner(200) 之前执行。 + * Skill workspace bootstrap. + *

    + *
  1. Ensure the workspace root exists.
  2. + *
  3. Sync classpath-bundled skills into the workspace + * (first install creates them; later starts upgrade only when the + * bundled SKILL.md frontmatter version is strictly newer).
  4. + *
  5. Materialize {@code mate_skill_file} rows down to each node's + * local cache so multi-instance deployments share the same + * scripts/references regardless of which node accepted the upload. + * Also backfills any pre-V112 on-disk-only skill files into the + * canonical store.
  6. + *
+ * + *

Order(210) — runs after {@code DatabaseBootstrapRunner}(200) so the + * skill rows the syncer needs to read are already loaded. * * @author MateClaw Team */ @Slf4j @Component -@Order(195) +@Order(210) @RequiredArgsConstructor public class SkillWorkspaceBootstrapRunner implements ApplicationRunner { private final SkillWorkspaceManager workspaceManager; private final BundledSkillSyncer bundledSkillSyncer; + private final SkillFileSyncer skillFileSyncer; @Override public void run(ApplicationArguments args) { var root = workspaceManager.getWorkspaceRoot(); log.info("Skill workspace root ready: {}", root); - // 同步 classpath 下预置技能到 workspace List synced = bundledSkillSyncer.sync(); if (!synced.isEmpty()) { log.info("Synced {} bundled skill(s) to workspace: {}", synced.size(), synced); } + + // Pull canonical bundle files from DB → local cache (and one-time + // backfill of pre-V112 disk-only skills back into the DB). + var report = skillFileSyncer.syncAll(); + log.info("Skill file sync: skills={}, materialized={}, current={}, " + + "diskBackfilled(skills={}, files={})", + report.skillsConsidered(), report.filesMaterialized(), + report.filesAlreadyCurrent(), + report.skillsBackfilled(), report.filesBackfilledFromDisk()); } } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceManager.java b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceManager.java index ffed5c50..16414841 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceManager.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillWorkspaceManager.java @@ -239,6 +239,158 @@ public class SkillWorkspaceManager { cleanDirectoryContents(workspaceDir.resolve("scripts")); } + /** + * Outcome of {@link #applyBundleFiles}, exposing per-bucket counters so + * the installer can log a meaningful summary and the admin UI can show + * what actually changed. + */ + public record ApplyBundleResult( + int referencesWritten, + int referencesPruned, + boolean referencesPreservedDueToEmptyBundle, + int scriptsWritten, + int scriptsPruned, + boolean scriptsPreservedDueToEmptyBundle + ) {} + + /** + * Apply a bundle's references/ + scripts/ to the workspace using + * write-then-prune semantics: + *

    + *
  1. Write every entry from the bundle (overwrites same paths).
  2. + *
  3. Delete any pre-existing file under references/ or scripts/ that + * is NOT in the bundle.
  4. + *
+ * + *

Empty-bundle safety: if the bundle has zero entries for a bucket + * AND the workspace already has files in that bucket, the bucket is + * left untouched (no pruning) unless {@code force=true}. This protects + * against malformed uploads, network truncation, and parser bugs that + * would otherwise wipe a user's scripts on reinstall — the same class + * of regression that an earlier patch fixed for SKILL.md. + * + * @param skillName workspace owner + * @param references new bundle's references map (key = path under references/) + * @param scripts new bundle's scripts map (key = path under scripts/) + * @param force bypass the empty-bundle guard (admin-only switch) + * @return per-bucket apply summary (never null) + */ + public ApplyBundleResult applyBundleFiles(String skillName, + Map references, + Map scripts, + boolean force) { + Path workspaceDir = resolveConventionPath(skillName); + try { + Files.createDirectories(workspaceDir.resolve("references")); + Files.createDirectories(workspaceDir.resolve("scripts")); + } catch (IOException e) { + log.warn("Failed to ensure data dirs for skill '{}': {}", skillName, e.getMessage()); + } + + int refsWritten = applyBucket(skillName, "references/", references); + int scriptsWritten = applyBucket(skillName, "scripts/", scripts); + + var refsPrune = pruneBucket(workspaceDir.resolve("references"), + normalizeKeys(references), force, skillName, "references"); + var scriptsPrune = pruneBucket(workspaceDir.resolve("scripts"), + normalizeKeys(scripts), force, skillName, "scripts"); + + return new ApplyBundleResult( + refsWritten, refsPrune.deleted(), refsPrune.preservedDueToEmpty(), + scriptsWritten, scriptsPrune.deleted(), scriptsPrune.preservedDueToEmpty() + ); + } + + private int applyBucket(String skillName, String bucketPrefix, Map entries) { + if (entries == null || entries.isEmpty()) return 0; + int written = 0; + for (var e : entries.entrySet()) { + String key = e.getKey(); + String relative = key.startsWith(bucketPrefix) ? key : (bucketPrefix + key); + try { + writeWorkspaceFile(skillName, relative, e.getValue()); + written++; + } catch (RuntimeException ex) { + log.warn("Failed to write {} for skill '{}': {}", relative, skillName, ex.getMessage()); + } + } + return written; + } + + /** Strip a leading "/" prefix so the key matches the path relative to the bucket dir. */ + private Set normalizeKeys(Map entries) { + if (entries == null || entries.isEmpty()) return Collections.emptySet(); + Set out = new HashSet<>(entries.size() * 2); + for (String key : entries.keySet()) { + String k = key.replace('\\', '/'); + int firstSlash = k.indexOf('/'); + if (firstSlash > 0 && (k.startsWith("references/") || k.startsWith("scripts/"))) { + out.add(k.substring(firstSlash + 1)); + } else { + out.add(k); + } + } + return out; + } + + private record PruneOutcome(int deleted, boolean preservedDueToEmpty) {} + + private PruneOutcome pruneBucket(Path bucketDir, Set keep, boolean force, + String skillName, String bucketLabel) { + if (!Files.exists(bucketDir) || !Files.isDirectory(bucketDir)) { + return new PruneOutcome(0, false); + } + + // Empty-bundle guard: if the new bundle has nothing for this bucket + // and there's at least one file on disk, refuse to prune unless the + // caller explicitly asked for it. Logged so the operator can see why + // their "clean install" didn't actually clean. + if (keep.isEmpty() && !force) { + try (var stream = Files.walk(bucketDir)) { + boolean hasAny = stream.filter(Files::isRegularFile).findFirst().isPresent(); + if (hasAny) { + log.warn("Refusing to prune {}/{}/ — new bundle is empty and would wipe existing files. " + + "Pass force=true to override.", skillName, bucketLabel); + return new PruneOutcome(0, true); + } + } catch (IOException e) { + log.warn("Failed to inspect {}/{}/: {}", skillName, bucketLabel, e.getMessage()); + return new PruneOutcome(0, false); + } + } + + int deleted = 0; + try (var stream = Files.walk(bucketDir)) { + List files = stream.filter(Files::isRegularFile).toList(); + for (Path file : files) { + String relative = bucketDir.relativize(file).toString().replace('\\', '/'); + if (!keep.contains(relative)) { + try { + Files.delete(file); + deleted++; + } catch (IOException e) { + log.warn("Failed to prune {}/{}/{}: {}", skillName, bucketLabel, relative, e.getMessage()); + } + } + } + // Best-effort: tidy up emptied subdirs (leave the bucket root in place). + try (var dirs = Files.walk(bucketDir)) { + dirs.sorted(java.util.Comparator.reverseOrder()) + .filter(p -> Files.isDirectory(p) && !p.equals(bucketDir)) + .forEach(p -> { + try (var children = Files.list(p)) { + if (children.findAny().isEmpty()) Files.delete(p); + } catch (IOException ignored) { + /* leave non-empty / locked dirs in place */ + } + }); + } + } catch (IOException e) { + log.warn("Failed to prune {}/{}/: {}", skillName, bucketLabel, e.getMessage()); + } + return new PruneOutcome(deleted, false); + } + /** * 验证写入路径安全性,防止路径逃逸 * diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttTransport.java b/mateclaw-server/src/main/java/vip/mate/stt/SttTransport.java new file mode 100644 index 00000000..01d5bc1e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttTransport.java @@ -0,0 +1,46 @@ +package vip.mate.stt; + +/** + * Issue #76: protocol-family abstraction for STT. + * + *

The original {@link SttProvider} bundled "which vendor is this" with + * "how does its wire protocol work", forcing every new vendor to ship a + * dedicated Java class even when the wire protocol is identical to an + * existing one. {@code SttTransport} is the protocol-only half: it knows how + * to send a request and parse a response, but doesn't care whether the + * endpoint is OpenAI cloud, FunASR self-hosted, SiliconFlow, Groq, or + * Together — anything that speaks the same protocol can plug in. + * + *

Two transports cover ~99% of the market today: + *

    + *
  • OpenAI Whisper compatible HTTP multipart (this transport)
  • + *
  • DashScope realtime WebSocket (kept inline in + * {@code DashScopeSttProvider} for now — its own transport class + * can be carved out the same way when a second WebSocket-based + * vendor lands)
  • + *
+ * + *

Identity (display name, baseUrl defaults, language bias, ...) is + * declared by a future {@code SttProviderProfile} layer (Phase 2 of the + * refactor). Phase 1 keeps {@link SttProvider} as the public SPI but + * delegates the wire work to a transport so swapping the credential row + * doesn't require changing the provider class. + */ +public interface SttTransport { + + /** + * Stable id of the protocol family this transport speaks. Profiles + * pick a transport by matching against this — e.g. + * {@code "openai_compatible_audio"} for any OpenAI Whisper-shaped + * endpoint. + */ + String apiMode(); + + /** + * Run a transcription against the resolved endpoint. Returns a typed + * success/failure result; transport implementations must NOT throw — + * caller relies on the failure path to keep the {@link SttProvider} + * fallback chain alive. + */ + SttResult transcribe(SttRequest request, SttTransportConfig config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/SttTransportConfig.java b/mateclaw-server/src/main/java/vip/mate/stt/SttTransportConfig.java new file mode 100644 index 00000000..7e276764 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/SttTransportConfig.java @@ -0,0 +1,19 @@ +package vip.mate.stt; + +/** + * Issue #76: resolved endpoint config passed to an {@link SttTransport}. + * + *

Decoupling the transport from {@code ModelProviderService} lookups makes + * tests trivial (no Spring context) and lets the same transport serve any + * credential row — OpenAI cloud, FunASR self-hosted, SiliconFlow, Groq, etc. + * + * @param baseUrl fully-qualified provider base URL ({@code https://api.openai.com} + * or {@code http://10.0.0.5:9999/v1}). Trailing slash optional; + * transports normalize it. + * @param apiKey bearer token. May be blank when the provider doesn't require + * authentication (some self-hosted FunASR deployments). + * @param model the model id sent in the multipart "model" field + * (whisper-1 / paraformer-large / FunAudioLLM-Whisper / ...). + */ +public record SttTransportConfig(String baseUrl, String apiKey, String model) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java b/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java index 1ac7a75e..3257b56b 100644 --- a/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/stt/provider/OpenAiSttProvider.java @@ -1,23 +1,40 @@ package vip.mate.stt.provider; -import cn.hutool.http.HttpRequest; -import cn.hutool.http.HttpResponse; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelProviderEntity; import vip.mate.llm.service.ModelProviderService; -import vip.mate.stt.AudioMimeTypes; import vip.mate.stt.SttProvider; import vip.mate.stt.SttRequest; import vip.mate.stt.SttResult; +import vip.mate.stt.SttTransportConfig; +import vip.mate.stt.transport.OpenAiCompatibleSttTransport; import vip.mate.system.model.SystemSettingsDTO; /** - * OpenAI STT Provider — Whisper / gpt-4o-mini-transcribe - *

- * 复用模型管理中的 OpenAI API Key。 + * OpenAI Whisper / OpenAI-compatible STT provider — thin wrapper. + * + *

Issue #76: this used to bake the {@code id="openai"} credential row + the + * {@code https://api.openai.com} base URL + Whisper-1 directly into the + * transport call, so the only way to point STT at FunASR / SiliconFlow / Groq + * was to hand-edit the OpenAI provider row's baseUrl (lossy + side-effects on + * chat). After this refactor: + * + *

    + *
  • Wire protocol lives in {@link OpenAiCompatibleSttTransport}.
  • + *
  • Credential row is selected by {@code SystemSettingsDTO.sttOpenAiCompatProviderId} + * (defaults to {@code "openai"} for backwards compatibility).
  • + *
  • Model is selected by {@code SystemSettingsDTO.sttOpenAiCompatModel} + * (defaults to {@code "whisper-1"}).
  • + *
+ * + *

The provider id stays {@code "openai"} because settings UI / fallback + * registry / per-language ordering all key off it. Phase 2 of the refactor + * will replace this single provider with a profile-driven registry; until + * then, swapping the credential row is the path forward for new vendors. */ @Slf4j @Component @@ -25,12 +42,13 @@ import vip.mate.system.model.SystemSettingsDTO; public class OpenAiSttProvider implements SttProvider { private final ModelProviderService modelProviderService; - private final ObjectMapper objectMapper; + private final OpenAiCompatibleSttTransport transport; - private static final String DEFAULT_MODEL = "whisper-1"; + private static final String LEGACY_DEFAULT_PROVIDER_ID = "openai"; + private static final String LEGACY_DEFAULT_MODEL = "whisper-1"; @Override public String id() { return "openai"; } - @Override public String label() { return "OpenAI Whisper"; } + @Override public String label() { return "OpenAI / OpenAI-compatible (Whisper)"; } @Override public boolean requiresCredential() { return true; } @Override public int autoDetectOrder() { return 100; } @@ -56,7 +74,8 @@ public class OpenAiSttProvider implements SttProvider { @Override public boolean isAvailable(SystemSettingsDTO config) { try { - return modelProviderService.isProviderConfigured("openai"); + String providerId = resolveProviderId(config); + return modelProviderService.isProviderConfigured(providerId); } catch (Exception e) { log.warn("[OpenAI STT] availability check failed: {}", e.getMessage()); return false; @@ -65,40 +84,39 @@ public class OpenAiSttProvider implements SttProvider { @Override public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + String providerId = resolveProviderId(config); + ModelProviderEntity provider; try { - String apiKey = modelProviderService.getProviderConfig("openai").getApiKey(); - String baseUrl = modelProviderService.getProviderConfig("openai").getBaseUrl(); - if (apiKey == null) return SttResult.failure("OpenAI API Key 未配置"); - - String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/audio/transcriptions"; - String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL; - // AudioMimeTypes ensures the filename extension matches the - // actual bytes (audio.wav, audio.mp3, etc.), which Hutool then - // uses to infer the multipart Content-Type. Don't pass - // contentType to .form() explicitly — Hutool has no - // form(String,byte[],String,String) overload, and the wrong - // dispatch crashes with ClassCastException on byte[] → Object[]. - String fileName = AudioMimeTypes.resolveFileName(request.getFileName(), request.getContentType()); - - HttpResponse response = HttpRequest.post(url) - .header("Authorization", "Bearer " + apiKey) - .form("model", model) - .form("file", request.getAudioData(), fileName) - .timeout(60_000) - .execute(); - - if (response.getStatus() == 200) { - JsonNode result = objectMapper.readTree(response.body()); - String text = result.path("text").asText(""); - log.info("[OpenAI STT] Transcribed {} chars (model={})", text.length(), model); - return SttResult.success(text); - } else { - log.warn("[OpenAI STT] Failed: HTTP {} - {}", response.getStatus(), response.body()); - return SttResult.failure("OpenAI STT 失败: HTTP " + response.getStatus()); - } - } catch (Exception e) { - log.error("[OpenAI STT] Error: {}", e.getMessage(), e); - return SttResult.failure("OpenAI STT 异常: " + e.getMessage()); + provider = modelProviderService.getProviderConfig(providerId); + } catch (MateClawException e) { + return SttResult.failure("STT 凭证 provider 未找到: " + providerId); } + + String apiKey = provider.getApiKey(); + String baseUrl = StringUtils.hasText(provider.getBaseUrl()) + ? provider.getBaseUrl() + : "https://api.openai.com"; + + // Allow blank apiKey for self-hosted / no-auth setups (FunASR is the + // typical case). The transport will only attach the Authorization + // header when apiKey is present. + boolean requiresKey = Boolean.TRUE.equals(provider.getRequireApiKey()); + if (requiresKey && (apiKey == null || apiKey.isBlank())) { + return SttResult.failure("STT 凭证 provider 未配置 API Key: " + providerId); + } + + String model = resolveModel(config); + SttTransportConfig transportConfig = new SttTransportConfig(baseUrl, apiKey, model); + return transport.transcribe(request, transportConfig); + } + + private String resolveProviderId(SystemSettingsDTO config) { + String configured = config != null ? config.getSttOpenAiCompatProviderId() : null; + return StringUtils.hasText(configured) ? configured.trim() : LEGACY_DEFAULT_PROVIDER_ID; + } + + private String resolveModel(SystemSettingsDTO config) { + String configured = config != null ? config.getSttOpenAiCompatModel() : null; + return StringUtils.hasText(configured) ? configured.trim() : LEGACY_DEFAULT_MODEL; } } diff --git a/mateclaw-server/src/main/java/vip/mate/stt/transport/OpenAiCompatibleSttTransport.java b/mateclaw-server/src/main/java/vip/mate/stt/transport/OpenAiCompatibleSttTransport.java new file mode 100644 index 00000000..6e28414b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/stt/transport/OpenAiCompatibleSttTransport.java @@ -0,0 +1,128 @@ +package vip.mate.stt.transport; + +import cn.hutool.http.HttpRequest; +import cn.hutool.http.HttpResponse; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.stt.AudioMimeTypes; +import vip.mate.stt.SttRequest; +import vip.mate.stt.SttResult; +import vip.mate.stt.SttTransport; +import vip.mate.stt.SttTransportConfig; + +/** + * Issue #76: protocol family transport for the OpenAI Whisper-shaped HTTP + * audio endpoint. Identical request format covers OpenAI itself, FunASR with + * the openai-compat shim, SiliconFlow, Groq Whisper, Together, Volcano, + * and roughly every other paid + self-hosted ASR vendor available today. + * + *

Wire shape: + *

    + *
  • {@code POST {baseUrl}/v1/audio/transcriptions} + * (or {@code {baseUrl}/audio/transcriptions} when baseUrl already + * carries a {@code /vN} suffix)
  • + *
  • multipart/form-data with {@code model} field + {@code file} field + * carrying the audio bytes named after the detected mime type + * (Hutool infers the multipart Content-Type from the extension — + * {@link AudioMimeTypes#resolveFileName} is what makes that work).
  • + *
  • Optional {@code Authorization: Bearer } when the caller + * supplies one. Self-hosted FunASR commonly skips auth entirely.
  • + *
+ * + *

Response: {@code { "text": "..." }} — the only field we read. + * + *

The transport intentionally does NOT touch {@code ModelProviderService} + * or {@code SystemSettingsDTO}: the caller resolves credentials and hands + * them in via {@link SttTransportConfig}. This keeps the transport reusable + * across any number of credential rows and trivially unit-testable. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class OpenAiCompatibleSttTransport implements SttTransport { + + public static final String API_MODE = "openai_compatible_audio"; + + private final ObjectMapper objectMapper; + + @Override + public String apiMode() { + return API_MODE; + } + + @Override + public SttResult transcribe(SttRequest request, SttTransportConfig config) { + try { + String baseUrl = normalizeBaseUrl(config.baseUrl()); + if (baseUrl == null) { + return SttResult.failure("STT 端点 base URL 未配置"); + } + String url = baseUrl + resolveAudioPath(baseUrl); + String model = effectiveModel(request, config); + // AudioMimeTypes ensures the filename extension matches the + // actual bytes (audio.wav, audio.mp3, etc.), which Hutool then + // uses to infer the multipart Content-Type. Don't pass + // contentType to .form() explicitly — Hutool has no + // form(String,byte[],String,String) overload, and the wrong + // dispatch crashes with ClassCastException on byte[] → Object[]. + String fileName = AudioMimeTypes.resolveFileName(request.getFileName(), request.getContentType()); + + HttpRequest http = HttpRequest.post(url) + .form("model", model) + .form("file", request.getAudioData(), fileName) + .timeout(60_000); + String apiKey = config.apiKey(); + if (apiKey != null && !apiKey.isBlank()) { + http.header("Authorization", "Bearer " + apiKey.trim()); + } + + HttpResponse response = http.execute(); + if (response.getStatus() == 200) { + JsonNode result = objectMapper.readTree(response.body()); + String text = result.path("text").asText(""); + log.info("[OpenAI-compat STT] Transcribed {} chars (model={}, baseUrl={})", + text.length(), model, baseUrl); + return SttResult.success(text); + } + log.warn("[OpenAI-compat STT] Failed: HTTP {} - {}", response.getStatus(), response.body()); + return SttResult.failure("STT 失败: HTTP " + response.getStatus()); + } catch (Exception e) { + log.error("[OpenAI-compat STT] Error: {}", e.getMessage(), e); + return SttResult.failure("STT 异常: " + e.getMessage()); + } + } + + /** + * Pick the audio path to append. If baseUrl already ends in a {@code /vN} + * version segment (lmstudio-style), append only {@code /audio/transcriptions}. + * Otherwise append {@code /v1/audio/transcriptions}. Mirrors the resolver + * pattern used by the chat-models probe so user-set baseUrls behave + * consistently across endpoints. + */ + static String resolveAudioPath(String baseUrl) { + if (baseUrl != null && baseUrl.matches(".*/v\\d{1,2}$")) { + return "/audio/transcriptions"; + } + return "/v1/audio/transcriptions"; + } + + static String normalizeBaseUrl(String raw) { + if (raw == null) return null; + String trimmed = raw.trim(); + if (trimmed.isEmpty()) return null; + return trimmed.endsWith("/") ? trimmed.substring(0, trimmed.length() - 1) : trimmed; + } + + private static String effectiveModel(SttRequest request, SttTransportConfig config) { + if (request.getModel() != null && !request.getModel().isBlank()) { + return request.getModel(); + } + if (config.model() != null && !config.model().isBlank()) { + return config.model(); + } + return "whisper-1"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java b/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java index 9f055aea..741f5b8d 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java +++ b/mateclaw-server/src/main/java/vip/mate/system/controller/SystemSettingController.java @@ -41,8 +41,39 @@ public class SystemSettingController { return R.ok(systemSettingService.saveLanguage(request.getLanguage())); } + /** + * Dedicated endpoint for the multimodal sidecar configuration. + *

+ * Separated from the bulk {@code PUT /settings} because the bulk endpoint + * now guards sidecar keys with null checks (so unrelated settings pages + * can't clobber them via partial payloads). This endpoint always writes + * both fields, so passing {@code null} for either explicitly clears that + * sidecar — preserving the "clear via UI" UX without leaking the + * write-on-null semantics into every other settings save. + */ + @Operation(summary = "更新多模态 sidecar 配置") + @PutMapping("/sidecar") + public R saveSidecar(@RequestBody SidecarRequest request) { + return R.ok(systemSettingService.updateSidecarSettings( + request.getDefaultVisionModelId(), + request.getDefaultVideoModelId())); + } + @Data public static class LanguageRequest { private String language; } + + /** + * Body for {@code PUT /settings/sidecar}. Both fields are nullable; + * {@code null} means "explicit clear". Field absence in the JSON + * payload also deserializes to null, which is the same outcome — the + * sidecar UI is the only caller of this endpoint and always sends both + * fields, so the absent-vs-null distinction doesn't matter here. + */ + @Data + public static class SidecarRequest { + private Long defaultVisionModelId; + private Long defaultVideoModelId; + } } diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index 25b21b8e..42f012a5 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -108,6 +108,20 @@ public class SystemSettingsDTO { /** 首选 STT provider: auto / openai / dashscope */ private String sttProvider; private Boolean sttFallbackEnabled; + /** + * Issue #76: which {@code mate_model_provider} row should the OpenAI STT + * provider read its baseUrl + apiKey from. Defaults to "openai" so existing + * deployments keep working; swap to a custom OpenAI-compatible provider row + * (FunASR / SiliconFlow / Groq / Together / Volcano / etc.) to point STT + * at any compatible endpoint without a code change. + */ + private String sttOpenAiCompatProviderId; + /** + * Issue #76: model id sent in the multipart "model" field. Defaults to + * whisper-1; override with paraformer-large / FunAudioLLM-Whisper / etc. + * when the configured provider exposes a different identifier. + */ + private String sttOpenAiCompatModel; // ===== 音乐生成配置 ===== private Boolean musicEnabled; @@ -120,4 +134,22 @@ public class SystemSettingsDTO { /** 首选 3D provider: auto / hunyuan-3d */ private String model3dProvider; private Boolean model3dFallbackEnabled; + + // ===== Multimodal sidecar routing ===== + /** + * Default vision-capable model id used to caption image attachments when the + * agent's primary model lacks the VISION modality. References mate_model_config.id; + * provider+model_name pairs are not unique so we store the surrogate key. + * null / non-existent / disabled rows are treated as "not configured" — the + * runtime then leaves the attachment out and asks the user to pick a model. + */ + private Long defaultVisionModelId; + + /** + * Default video-capable model id used when the agent's primary model lacks + * the VIDEO modality. Same semantics as defaultVisionModelId. v1 routing does + * not yet implement video sidecar; this is reserved for the next iteration so + * the configuration surface is stable. + */ + private Long defaultVideoModelId; } diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index 17bcdae0..1a22ca6a 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -49,6 +49,9 @@ public class SystemSettingService { private static final String STT_ENABLED_KEY = "sttEnabled"; private static final String STT_PROVIDER_KEY = "sttProvider"; private static final String STT_FALLBACK_ENABLED_KEY = "sttFallbackEnabled"; + // Issue #76: let the OpenAI STT provider point at any OpenAI-compat endpoint. + private static final String STT_OPENAI_COMPAT_PROVIDER_ID_KEY = "sttOpenAiCompatProviderId"; + private static final String STT_OPENAI_COMPAT_MODEL_KEY = "sttOpenAiCompatModel"; // 音乐生成配置 keys private static final String MUSIC_ENABLED_KEY = "musicEnabled"; @@ -60,6 +63,10 @@ public class SystemSettingService { private static final String MODEL3D_PROVIDER_KEY = "model3dProvider"; private static final String MODEL3D_FALLBACK_ENABLED_KEY = "model3dFallbackEnabled"; + // Multimodal sidecar routing keys (id values; references mate_model_config.id) + private static final String DEFAULT_VISION_MODEL_KEY = "default.vision_model"; + private static final String DEFAULT_VIDEO_MODEL_KEY = "default.video_model"; + private static final String ZHIPU_API_KEY_KEY = "zhipuApiKey"; private static final String ZHIPU_BASE_URL_KEY = "zhipuBaseUrl"; private static final String FAL_API_KEY_KEY = "falApiKey"; @@ -134,6 +141,10 @@ public class SystemSettingService { dto.setSttEnabled(Boolean.parseBoolean(getValue(STT_ENABLED_KEY, "false"))); dto.setSttProvider(getValue(STT_PROVIDER_KEY, "auto")); dto.setSttFallbackEnabled(Boolean.parseBoolean(getValue(STT_FALLBACK_ENABLED_KEY, "true"))); + // Issue #76: default to "openai" so upgrades behave identically to the + // old hard-coded path; users can swap to any OpenAI-compat provider row. + dto.setSttOpenAiCompatProviderId(getValue(STT_OPENAI_COMPAT_PROVIDER_ID_KEY, "openai")); + dto.setSttOpenAiCompatModel(getValue(STT_OPENAI_COMPAT_MODEL_KEY, "whisper-1")); // 音乐生成配置 dto.setMusicEnabled(Boolean.parseBoolean(getValue(MUSIC_ENABLED_KEY, "false"))); @@ -144,9 +155,22 @@ public class SystemSettingService { dto.setModel3dEnabled(Boolean.parseBoolean(getValue(MODEL3D_ENABLED_KEY, "false"))); dto.setModel3dProvider(getValue(MODEL3D_PROVIDER_KEY, "auto")); dto.setModel3dFallbackEnabled(Boolean.parseBoolean(getValue(MODEL3D_FALLBACK_ENABLED_KEY, "true"))); + + // Multimodal sidecar routing — empty string means "not configured" + dto.setDefaultVisionModelId(parseIdOrNull(getValue(DEFAULT_VISION_MODEL_KEY, ""))); + dto.setDefaultVideoModelId(parseIdOrNull(getValue(DEFAULT_VIDEO_MODEL_KEY, ""))); return dto; } + private Long parseIdOrNull(String value) { + if (value == null || value.isBlank()) return null; + try { + return Long.parseLong(value.trim()); + } catch (NumberFormatException e) { + return null; + } + } + /** * 获取全部配置(内部使用,包含明文 API Key)— 供 VideoGenerationService 等后端服务使用 */ @@ -295,6 +319,15 @@ public class SystemSettingService { if (dto.getSttFallbackEnabled() != null) { saveValue(STT_FALLBACK_ENABLED_KEY, String.valueOf(dto.getSttFallbackEnabled()), "STT Provider 级 Fallback"); } + // Issue #76: persist the OpenAI-compatible STT routing target. + if (dto.getSttOpenAiCompatProviderId() != null) { + saveValue(STT_OPENAI_COMPAT_PROVIDER_ID_KEY, dto.getSttOpenAiCompatProviderId(), + "OpenAI-compat STT 凭证来源 provider id"); + } + if (dto.getSttOpenAiCompatModel() != null) { + saveValue(STT_OPENAI_COMPAT_MODEL_KEY, dto.getSttOpenAiCompatModel(), + "OpenAI-compat STT 模型名"); + } // 音乐生成配置 if (dto.getMusicEnabled() != null) { @@ -317,6 +350,49 @@ public class SystemSettingService { if (dto.getModel3dFallbackEnabled() != null) { saveValue(MODEL3D_FALLBACK_ENABLED_KEY, String.valueOf(dto.getModel3dFallbackEnabled()), "3D Provider 级 Fallback"); } + + // Multimodal sidecar routing — guarded with null check, matching the + // pattern used for music / 3D / image / video / tts / stt blocks + // above. The bulk PUT /settings is used by every settings page (System, + // Music, Video, Image, Stt, Tts, Model3D), each sending a partial + // payload that omits sidecar fields. Without this guard, saving any + // unrelated setting would silently write "" into the sidecar keys + // (Long? defaultVisionModelId deserializes to null when absent), which + // wiped users' configured vision/video models the moment they touched + // an unrelated settings page. Explicit clearing via the sidecar UI now + // routes through {@link #updateSidecarSettings} instead. + if (dto.getDefaultVisionModelId() != null) { + saveValue(DEFAULT_VISION_MODEL_KEY, + String.valueOf(dto.getDefaultVisionModelId()), + "Default vision-capable model id (mate_model_config.id) for sidecar routing"); + } + if (dto.getDefaultVideoModelId() != null) { + saveValue(DEFAULT_VIDEO_MODEL_KEY, + String.valueOf(dto.getDefaultVideoModelId()), + "Default video-capable model id (mate_model_config.id) for sidecar routing"); + } + return getSettings(); + } + + /** + * Dedicated update path for the multimodal sidecar configuration. + *

+ * This endpoint is the ONLY place vision/video model ids can be written + * unconditionally — null is treated as an explicit "clear" and writes + * an empty string (parse-back returns null). The bulk + * {@link #saveSettings} now guards both keys with non-null checks so + * unrelated settings pages can't accidentally clobber sidecar config. + *

+ * Both fields are always written so a single API call can independently + * assign / clear either modality. + */ + public SystemSettingsDTO updateSidecarSettings(Long visionModelId, Long videoModelId) { + saveValue(DEFAULT_VISION_MODEL_KEY, + visionModelId == null ? "" : String.valueOf(visionModelId), + "Default vision-capable model id (mate_model_config.id) for sidecar routing"); + saveValue(DEFAULT_VIDEO_MODEL_KEY, + videoModelId == null ? "" : String.valueOf(videoModelId), + "Default video-capable model id (mate_model_config.id) for sidecar routing"); return getSettings(); } diff --git a/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java index 77a028f6..54647533 100644 --- a/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java +++ b/mateclaw-server/src/main/java/vip/mate/task/AsyncTaskService.java @@ -6,11 +6,13 @@ import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Service; import vip.mate.channel.web.ChatStreamTracker; import vip.mate.task.model.AsyncTaskEntity; import vip.mate.task.model.AsyncTaskInfo; import vip.mate.task.repository.AsyncTaskMapper; +import vip.mate.workspace.conversation.event.ConversationDeletedEvent; import jakarta.annotation.PreDestroy; import java.time.LocalDateTime; @@ -48,6 +50,33 @@ public class AsyncTaskService implements ApplicationRunner { /** 活跃轮询任务,key = taskId */ private final ConcurrentHashMap> activePolls = new ConcurrentHashMap<>(); + /** Reverse mapping taskId → conversationId so a {@link ConversationDeletedEvent} + * listener can cancel every poller belonging to the deleted conversation + * without scanning DB (the {@code mate_async_task} rows are gone by the + * time the after-commit event fires). Populated in {@link #startPolling}; + * cleared in {@link #cancelPolling}. */ + private final ConcurrentHashMap pollTaskToConv = new ConcurrentHashMap<>(); + + /** Conversations whose deletion has fanned out to this service. Workers + * consult {@link #isConversationCanceled} before persisting anything tied + * to a conversation — the music virtual-thread worker, image/video poll + * completion handlers, and any future provider-level callback are + * asynchronous and may finish AFTER the conversation row + attachment + * directory have already been wiped. Without this gate they would + * recreate the directory + a dangling {@code mate_message} row. + *

+ * Value = expiry epoch-ms. Entries older than {@link #CANCEL_RETENTION_MS} + * are reaped on each event and on each lookup so the map cannot grow + * without bound. The retention window is comfortably longer than + * {@link #MAX_POLL_DURATION_MINUTES} and the music worker's ~120s upstream + * HTTP timeout, so any in-flight worker for a deleted conversation will + * still see the cancel flag when it tries to write back. */ + private final ConcurrentHashMap canceledConversations = new ConcurrentHashMap<>(); + + /** 30 minutes — covers MAX_POLL_DURATION_MINUTES (15) + music worker's + * ~2 min upstream blocking call with comfortable headroom. */ + private static final long CANCEL_RETENTION_MS = 30L * 60 * 1000; + /** 每用户最多并行任务数 */ private static final int MAX_ACTIVE_TASKS_PER_USER = 3; @@ -181,6 +210,9 @@ public class AsyncTaskService implements ApplicationRunner { }, 3, POLL_INTERVAL_SECONDS, TimeUnit.SECONDS); activePolls.put(taskId, future); + if (task.getConversationId() != null) { + pollTaskToConv.put(taskId, task.getConversationId()); + } log.info("[AsyncTask] Started polling for task {} (interval={}s, timeout={}min)", taskId, POLL_INTERVAL_SECONDS, MAX_POLL_DURATION_MINUTES); } @@ -190,6 +222,49 @@ public class AsyncTaskService implements ApplicationRunner { if (future != null) { future.cancel(false); } + pollTaskToConv.remove(taskId); + } + + // ==================== Conversation-deleted fan-out ==================== + + /** + * Returns true if this conversation was deleted recently enough that any + * still-running async worker (music virtual-thread, image/video poll + * completion, …) must abort before writing a file or persisting a + * message — see {@link #canceledConversations}. + *

+ * Sweeps stale entries on read so the map stays small. + */ + public boolean isConversationCanceled(String conversationId) { + if (conversationId == null) return false; + sweepCanceled(); + return canceledConversations.containsKey(conversationId); + } + + @EventListener + public void onConversationDeleted(ConversationDeletedEvent event) { + String convId = event.conversationId(); + if (convId == null) return; + + canceledConversations.put(convId, System.currentTimeMillis() + CANCEL_RETENTION_MS); + + int cancelled = 0; + for (Map.Entry entry : pollTaskToConv.entrySet()) { + if (convId.equals(entry.getValue())) { + cancelPolling(entry.getKey()); + cancelled++; + } + } + if (cancelled > 0) { + log.info("[AsyncTask] Cancelled {} active poller(s) for deleted conversation {}", + cancelled, convId); + } + sweepCanceled(); + } + + private void sweepCanceled() { + long now = System.currentTimeMillis(); + canceledConversations.entrySet().removeIf(e -> e.getValue() < now); } // ==================== 状态更新 ==================== diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DelegateAgentTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DelegateAgentTool.java index 261e3bf9..4fcdd294 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DelegateAgentTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DelegateAgentTool.java @@ -71,12 +71,20 @@ public class DelegateAgentTool { static final int INHERITED_CONTEXT_MAX_MESSAGES = 10; static final int INHERITED_CONTEXT_PER_MESSAGE_CHARS = 1000; /** - * Per-child timeout — raised from 60 s to 120 s so that slow LLM models - * (kimi-code observed p99 ≈ 91 s) can complete before the parent gives up. - * The previous 60 s limit was structurally impossible to satisfy once any - * child called an LLM-backed tool. + * Wall-clock budget for one delegateParallel batch — applies to all children + * together, not per child (they run concurrently on virtual threads). + * + *

Configurable via {@code mateclaw.delegation.parallel-timeout-seconds}; + * default 300 s (5 minutes). Earlier defaults (60 s → 120 s) were + * structurally too tight for thinking models: a single LLM turn against + * Kimi / GLM / MiniMax routinely takes 90–290 s when the child must + * produce multi-section structured output, so the parent gave up while the + * children were still happily streaming. 300 s matches the per-prompt + * ceiling used by ACP delegation and keeps headroom for one tool-call + * round trip on top of a single LLM turn. */ - private static final int PARALLEL_TIMEOUT_SECONDS = 120; + @Value("${mateclaw.delegation.parallel-timeout-seconds:300}") + private int parallelTimeoutSeconds; /** * Default deny list for child agents. Names are matched against the @@ -413,9 +421,9 @@ public class DelegateAgentTool { List results = new ArrayList<>(); try { CompletableFuture.allOf(futures.values().toArray(new CompletableFuture[0])) - .get(PARALLEL_TIMEOUT_SECONDS, TimeUnit.SECONDS); + .get(parallelTimeoutSeconds, TimeUnit.SECONDS); } catch (TimeoutException e) { - log.warn("Parallel delegation timed out ({}s), collecting completed results", PARALLEL_TIMEOUT_SECONDS); + log.warn("Parallel delegation timed out ({}s), collecting completed results", parallelTimeoutSeconds); } catch (Exception e) { log.error("Parallel delegation error: {}", e.getMessage()); } @@ -444,7 +452,7 @@ public class DelegateAgentTool { } f.cancel(true); // Use ofTimeout so outcome="timeout" is explicit and distinct from "error". - results.add(ChildResult.ofTimeout(idx, agentName, PARALLEL_TIMEOUT_SECONDS)); + results.add(ChildResult.ofTimeout(idx, agentName, parallelTimeoutSeconds)); } } @@ -548,7 +556,7 @@ public class DelegateAgentTool { .append(",trim 后 0 字符)。请勿将此误报为超时或失败——子 Agent 已正常完成,只是本次无输出。\n"); } case "timeout" -> - sb.append("❌ 超时(").append(PARALLEL_TIMEOUT_SECONDS).append("s 内未返回)\n"); + sb.append("❌ 超时(").append(parallelTimeoutSeconds).append("s 内未返回)\n"); default -> sb.append("❌ 失败:").append(r.error).append("\n"); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java index 8587b89b..32e07c99 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocumentExtractTool.java @@ -19,13 +19,15 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; /** - * 文档文本提取工具 - * 支持 PDF、DOCX、XLSX、PPTX 等 Office 文档的文本提取 - * 实现 fallback 链:系统命令 -> Java 实现 -> 结构化错误 + * Document text extraction tool. + * Supports PDF, DOCX, XLSX, PPTX with format-specific fallback chains. * - * 实现策略: - * - PDF: pdftotext -> pypdf/pdfplumber (Java 实现) - * - DOCX: textutil/pandoc -> ZIP XML 解析 + * Strategy by format: + * - PDF: pdftotext -> pdfplumber/pypdf -> pdfbox -> OCR (scanned) -> Tika + * - DOCX: textutil / pandoc / libreoffice -> ZIP+XML -> Tika + * - XLSX/PPTX: Tika directly (POI-based; correctly resolves the shared-strings + * indirection table and walks SmartArt / chart / grouped-shape + * text that a naive ZIP+XML scan misses). */ @Slf4j @Component @@ -45,12 +47,11 @@ public class DocumentExtractTool { - Excel (.xlsx, .xls) - 提取为文本表格 - PowerPoint (.pptx, .ppt) - 提取策略(默认自动选择最优方式): - 1. 优先使用系统命令(pdftotext, textutil, pandoc 等) - 2. 系统命令不可用时使用纯 Java 实现 - 3. PDF 扫描版进入 OCR - 4. 全部失败前用 Apache Tika 兜底(覆盖 SmartArt、共享字符串表等盲区) - 5. 返回详细的提取过程和元数据 + 提取策略(按格式分链): + - PDF: pdftotext → pdfplumber/pypdf → pdfbox → OCR(扫描版) → Tika + - DOCX: textutil / pandoc / libreoffice → ZIP-XML → Tika + - XLSX/PPTX: 直接走 Tika(基于 POI,正确解析 sharedStrings 表与 SmartArt / 图表文本) + - 返回详细的提取过程和元数据 参数 options 可包含: - pages: 指定页码范围(如 "1-5" 或 "1,3,5") @@ -212,13 +213,12 @@ public class DocumentExtractTool { long t0 = System.currentTimeMillis(); String content = tryPdftotext(path, options); if (content != null && !content.isBlank()) { - if (!needsOcr(content, realPageCount)) { + ExtractionQuality q = classifyExtraction(content, realPageCount); + if (!q.needsOcr()) { attempts.add("pdftotext: 成功 (" + (System.currentTimeMillis() - t0) + "ms)"); return new ExtractedContent(content, "pdftotext", realPageCount > 0 ? realPageCount : estimatePages(content)); } - double perPage = realPageCount > 0 ? (double) content.strip().length() / realPageCount : 0; - attempts.add("pdftotext: 文本过少 (总 " + content.strip().length() + " 字符, " - + realPageCount + " 页, 每页 " + String.format("%.0f", perPage) + " 字符),可能是扫描版"); + attempts.add("pdftotext: 触发 OCR (" + describeTrigger(q, content.strip().length(), realPageCount) + ")"); bestContent = content; bestMethod = "pdftotext"; } else { @@ -229,11 +229,12 @@ public class DocumentExtractTool { long t1 = System.currentTimeMillis(); content = tryPythonPdfExtractor(path, options); if (content != null && !content.isBlank()) { - if (!needsOcr(content, realPageCount)) { + ExtractionQuality q = classifyExtraction(content, realPageCount); + if (!q.needsOcr()) { attempts.add("python_pdf: 成功 (" + (System.currentTimeMillis() - t1) + "ms)"); return new ExtractedContent(content, "python_pdfplumber", realPageCount > 0 ? realPageCount : estimatePages(content)); } - attempts.add("python_pdf: 文本过少"); + attempts.add("python_pdf: 触发 OCR (" + describeTrigger(q, content.strip().length(), realPageCount) + ")"); if (bestContent == null || content.strip().length() > bestContent.strip().length()) { bestContent = content; bestMethod = "python_pdfplumber"; @@ -246,11 +247,12 @@ public class DocumentExtractTool { long t2 = System.currentTimeMillis(); content = extractPdfWithJava(path); if (content != null && !content.isBlank()) { - if (!needsOcr(content, realPageCount)) { + ExtractionQuality q = classifyExtraction(content, realPageCount); + if (!q.needsOcr()) { attempts.add("java_pdf: 成功 (" + (System.currentTimeMillis() - t2) + "ms)"); return new ExtractedContent(content, "java_pdfbox", realPageCount > 0 ? realPageCount : estimatePages(content)); } - attempts.add("java_pdf: 文本过少"); + attempts.add("java_pdf: 触发 OCR (" + describeTrigger(q, content.strip().length(), realPageCount) + ")"); if (bestContent == null || content.strip().length() > bestContent.strip().length()) { bestContent = content; bestMethod = "java_pdfbox"; @@ -324,22 +326,99 @@ public class DocumentExtractTool { return 0; // 未知页数 } + /** Fraction below which extracted text is judged unreadable and an OCR pass is forced. */ + static final double READABLE_RATIO_THRESHOLD = 0.5; + + /** Outcome of {@link #classifyExtraction}; {@link #trigger()} is {@code null} when usable. */ + record ExtractionQuality(String trigger, double readableRatio, double charsPerPage) { + boolean needsOcr() { return trigger != null; } + } + /** - * 判断提取到的文本是否太少、需要尝试 OCR。 - * 使用真实页数(来自 getPdfPageCount)计算字符密度,不再依赖 estimatePages 反推。 - * 页数未知(0)时,只看总字符数。 + * Classify the quality of a text extraction pass. + *

+ * Three failure modes can fire an OCR retry: + *

    + *
  • {@code empty} / {@code too_short}: nothing extracted, typical of image-only PDFs.
  • + *
  • {@code low_readable_ratio}: extractor returned plenty of characters but most of + * them are control bytes / high-Latin junk — typical of CID-encoded fonts without + * a {@code ToUnicode} CMap, where the engine dumps glyph indices as bytes.
  • + *
  • {@code low_char_density}: per-page char count is far below what a real text PDF + * would yield, typical of scanned PDFs with a thin OCR layer applied upstream.
  • + *
*/ - private boolean needsOcr(String text, int realPageCount) { - if (text == null || text.isBlank()) return true; - String stripped = text.strip(); - if (stripped.length() < 20) return true; - if (realPageCount <= 0) { - // 页数未知时回退到总字符数判定(保守阈值) - return stripped.length() < 100; + static ExtractionQuality classifyExtraction(String text, int realPageCount) { + if (text == null || text.isBlank()) { + return new ExtractionQuality("empty", 0.0, 0.0); } - double perPage = (double) stripped.length() / realPageCount; - // 正常文本 PDF 每页至少数百字符;每页不到 30 字符大概率是扫描版 - return perPage < 30; + String stripped = text.strip(); + if (stripped.length() < 20) { + return new ExtractionQuality("too_short", 0.0, 0.0); + } + double ratio = readableRatio(stripped); + double perPage = realPageCount > 0 + ? (double) stripped.length() / realPageCount + : stripped.length(); + if (ratio < READABLE_RATIO_THRESHOLD) { + return new ExtractionQuality("low_readable_ratio", ratio, perPage); + } + if (realPageCount <= 0) { + // Page count unknown — fall back to a conservative total-length cutoff. + if (stripped.length() < 100) { + return new ExtractionQuality("too_short", ratio, perPage); + } + } else if (perPage < 30) { + return new ExtractionQuality("low_char_density", ratio, perPage); + } + return new ExtractionQuality(null, ratio, perPage); + } + + /** + * Fraction of code points that are obviously readable: ASCII printable, tab/newline, + * CJK Unified Ideographs (+ ext A), CJK punctuation, halfwidth/fullwidth forms, + * hiragana/katakana, hangul syllables. Returns 0 for empty input. + *

+ * The threshold {@link #READABLE_RATIO_THRESHOLD} separates real-world noisy + * extraction (well above 0.7 even with OCR errors) from font-encoding garbage, + * which typically lands below 0.1 because the bytes fall outside every script range. + */ + static double readableRatio(String text) { + if (text == null || text.isEmpty()) return 0.0; + int total = 0, good = 0; + for (int i = 0; i < text.length(); ) { + int cp = text.codePointAt(i); + i += Character.charCount(cp); + total++; + if (isReadable(cp)) good++; + } + return total == 0 ? 0.0 : (double) good / total; + } + + /** Compact one-line summary of why an extraction was rejected, for the attempts log. */ + private static String describeTrigger(ExtractionQuality q, int totalChars, int realPageCount) { + return switch (q.trigger()) { + case "low_readable_ratio" -> String.format( + "readable=%.2f<%.2f, %d 字符多为非可读字节,可能是字体编码异常", + q.readableRatio(), READABLE_RATIO_THRESHOLD, totalChars); + case "low_char_density" -> String.format( + "每页 %.0f 字符(总 %d, %d 页),可能是扫描版", + q.charsPerPage(), totalChars, realPageCount); + case "too_short" -> "总 " + totalChars + " 字符,文本过少"; + case "empty" -> "提取结果为空"; + default -> "trigger=" + q.trigger(); + }; + } + + private static boolean isReadable(int cp) { + if (cp == 9 || cp == 10 || cp == 13) return true; + if (cp >= 0x20 && cp <= 0x7E) return true; // ASCII printable + if (cp >= 0x3000 && cp <= 0x303F) return true; // CJK punctuation + if (cp >= 0x3040 && cp <= 0x30FF) return true; // hiragana / katakana + if (cp >= 0x3400 && cp <= 0x4DBF) return true; // CJK ext A + if (cp >= 0x4E00 && cp <= 0x9FFF) return true; // CJK unified + if (cp >= 0xAC00 && cp <= 0xD7AF) return true; // hangul syllables + if (cp >= 0xFF00 && cp <= 0xFFEF) return true; // halfwidth / fullwidth + return false; } /** OCR 结果(含成功/失败页数统计) */ @@ -743,90 +822,51 @@ public class DocumentExtractTool { // ==================== XLSX 提取 ==================== private ExtractedContent extractXlsx(Path path, String options, List attempts) throws Exception { - StringBuilder text = new StringBuilder(); - - try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(path))) { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (entry.getName().startsWith("xl/worksheets/sheet") && entry.getName().endsWith(".xml")) { - String xml = new String(zis.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); - text.append("--- ").append(entry.getName()).append(" ---\n"); - text.append(extractTextFromXlsxXml(xml)).append("\n"); - } - } + long t = System.currentTimeMillis(); + String text = TikaExtractor.extract(path); + long elapsed = System.currentTimeMillis() - t; + if (text != null && !text.isBlank()) { + attempts.add("tika: 成功 (" + elapsed + "ms)"); + return new ExtractedContent(text, "tika", 0); } - - // Our ZIP-XML extractor only reads tags and skips the shared-strings table, - // so cells full of text labels look "empty". When that happens, fall through to - // Tika which knows how to resolve the shared-strings indirection. - if (text.toString().replaceAll("---.*?---", "").strip().isEmpty()) { - String fallback = TikaExtractor.extract(path); - if (fallback != null && !fallback.isBlank()) { - attempts.add("tika: 成功(ZIP-XML 仅有数字 / 共享字符串未解析)"); - return new ExtractedContent(fallback, "tika", 0); - } - } - - attempts.add("java_zip_xml: 成功"); - return new ExtractedContent(text.toString(), "java_zip_xml", 0); - } - - private String extractTextFromXlsxXml(String xml) { - StringBuilder text = new StringBuilder(); - int start = 0; - while ((start = xml.indexOf("", start)) != -1) { - int end = xml.indexOf("", start); - if (end == -1) break; - String value = xml.substring(start + 3, end); - text.append(value).append("\t"); - start = end + 4; - } - return text.toString(); + attempts.add("tika: 失败或不可用 (" + elapsed + "ms)"); + throw new Exception("XLSX 提取失败:Tika 无法解析(文件可能损坏、加密或非标准格式)"); } // ==================== PPTX 提取 ==================== private ExtractedContent extractPptx(Path path, String options, List attempts) throws Exception { - StringBuilder text = new StringBuilder(); - int slideNum = 1; - - try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(path))) { - ZipEntry entry; - while ((entry = zis.getNextEntry()) != null) { - if (entry.getName().startsWith("ppt/slides/slide") && entry.getName().endsWith(".xml")) { - String xml = new String(zis.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); - text.append("--- Slide ").append(slideNum++).append(" ---\n"); - text.append(extractTextFromPptxXml(xml)).append("\n\n"); - } - } + long t = System.currentTimeMillis(); + String text = TikaExtractor.extract(path); + long elapsed = System.currentTimeMillis() - t; + if (text != null && !text.isBlank()) { + attempts.add("tika: 成功 (" + elapsed + "ms)"); + int slides = countPptxSlides(path); + return new ExtractedContent(text, "tika", slides); } - - // Slide layouts with text inside SmartArt / charts / grouped shapes don't surface - // through the simple grep — Tika walks the full DrawingML graph and pulls - // them out. Only invoke when our walker produced nothing useful. - if (text.toString().replaceAll("---.*?---", "").strip().isEmpty()) { - String fallback = TikaExtractor.extract(path); - if (fallback != null && !fallback.isBlank()) { - attempts.add("tika: 成功(ZIP-XML 未抓到正文,可能是 SmartArt / 图表)"); - return new ExtractedContent(fallback, "tika", Math.max(0, slideNum - 1)); - } - } - - attempts.add("java_zip_xml: 成功"); - return new ExtractedContent(text.toString(), "java_zip_xml", Math.max(0, slideNum - 1)); + attempts.add("tika: 失败或不可用 (" + elapsed + "ms)"); + throw new Exception("PPTX 提取失败:Tika 无法解析(文件可能损坏、加密或非标准格式)"); } - private String extractTextFromPptxXml(String xml) { - StringBuilder text = new StringBuilder(); - int start = 0; - while ((start = xml.indexOf("", start)) != -1) { - int end = xml.indexOf("", start); - if (end == -1) break; - String txt = xml.substring(start + 5, end); - text.append(txt).append(" "); - start = end + 6; + /** + * Cheap slide count for the result metadata. Counts {@code ppt/slides/slideN.xml} + * entries in the OOXML zip without parsing the slide content. Returns 0 if the + * file isn't a readable zip. + */ + private int countPptxSlides(Path path) { + int count = 0; + try (ZipInputStream zis = new ZipInputStream(Files.newInputStream(path))) { + ZipEntry e; + while ((e = zis.getNextEntry()) != null) { + String name = e.getName(); + if (name.startsWith("ppt/slides/slide") && name.endsWith(".xml")) { + count++; + } + } + } catch (IOException ignored) { + // Slide count is best-effort metadata; never fail the extract on this. } - return text.toString().trim(); + return count; } // ==================== 工具方法 ==================== diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocxRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocxRenderTool.java index 63975cee..ce3f8f90 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocxRenderTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/DocxRenderTool.java @@ -5,14 +5,14 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.ai.tool.annotation.Tool; import org.springframework.ai.tool.annotation.ToolParam; import org.springframework.stereotype.Component; +import vip.mate.tool.document.FilenameSanitizer; import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; import vip.mate.tool.document.MarkdownDocxRenderer; -import vip.mate.tool.guard.WorkspacePathGuard; +import vip.mate.tool.document.MarkdownInputResolver; +import vip.mate.tool.document.MarkdownInputResolver.Resolved; +import vip.mate.tool.document.MarkdownInputResolver.ResolveException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; /** @@ -39,8 +39,10 @@ public class DocxRenderTool { private final GeneratedFileCache cache; @Tool(description = """ - Render a new .docx file from Markdown text and return a one-time download URL. - Use for creating NEW documents: reports, memos, contracts, letters, resumes. + Render a new .docx (Microsoft Word) file from Markdown text and return a + one-time download URL. Use for creating EDITABLE Word documents the user + will continue to revise — reports, memos, contracts, letters, resumes. + Supports: headings (# ## ###), bold (**text**), bullet lists (- item), numbered lists (1. item), tables (| col | col |), plain paragraphs, images (![alt](path/to/file.png|jpg|gif|bmp|svg)) — SVG is rasterized @@ -50,6 +52,11 @@ public class DocxRenderTool { disk) — passing huge markdown as a tool argument burns LLM tokens needlessly. Do NOT use for: + - **Anything the user asked for in PDF / .pdf format — use `renderPdf` / + `renderPdfFromFile` instead. PDF is a separate non-editable deliverable + format; don't silently substitute docx for it.** + - Spreadsheets / workbooks — use `renderXlsx` / `renderXlsxFromFile`. + - Slide decks / presentations — use `renderPptx` / `renderPptxFromFile`. - Editing an existing .docx file (use run_skill_script with unpack/edit/pack) - Adding tracked changes or comments (use run_skill_script) - GB/T 9704 official documents (use writeGongwen tool, BmacClaw only) @@ -69,26 +76,15 @@ public class DocxRenderTool { return "错误:markdown 参数为空,无法生成文档。"; } - String safeName = sanitizeFilename(filename); - String displayName = safeName + ".docx"; - String size = (pageSize == null || pageSize.isBlank()) ? "A4" : pageSize.trim(); + String displayName = FilenameSanitizer.sanitize(filename, "document", ".docx") + ".docx"; + String size = resolveSize(pageSize); try { long t0 = System.currentTimeMillis(); byte[] bytes = renderer.render(markdown, size); - String id = cache.put(bytes, displayName, DOCX_MIME); - long elapsed = System.currentTimeMillis() - t0; - log.info("[DocxRender] generated {} ({} bytes, {}ms, id={})", - displayName, bytes.length, elapsed, id); - - String url = "/api/v1/files/generated/" + id; - // Explicit instruction to suppress LLM hallucinating an absolute host. - // DeepSeek/Claude have been observed prepending placeholder domains - // (e.g. https://ai-tools-system.com) when echoing the URL back to the user, - // breaking the download link. Repeat the path verbatim with no host. - return "文档已生成:[" + displayName + "](" + url + ")(链接 10 分钟内有效)。\n" - + "重要:回答用户时**必须**使用上述相对路径 `" + url + "`," - + "**不要**添加任何 https://、http:// 域名前缀,前端会自动拼接当前主机。"; + log.info("[DocxRender] generated {} ({} bytes, {}ms)", + displayName, bytes.length, System.currentTimeMillis() - t0); + return GeneratedFileLink.resultZh(bytes, displayName, DOCX_MIME, cache, "文档"); } catch (Exception e) { log.error("[DocxRender] render failed for {}: {}", displayName, e.getMessage(), e); return "渲染失败:" + e.getMessage(); @@ -107,7 +103,13 @@ public class DocxRenderTool { * rendered from disk in one IO call. Token cost ≈ 50 (just the path). */ @Tool(description = """ - Render a .docx file from a markdown FILE on disk and return a one-time download URL. + Render a .docx (Microsoft Word) file from a markdown FILE on disk and return + a one-time download URL. Use this for EDITABLE Word documents only. + + **If the user asked for PDF / .pdf in any wording, use `renderPdfFromFile` + instead. Do not silently substitute docx for PDF.** Same for spreadsheets + (`renderXlsxFromFile`) and slide decks (`renderPptxFromFile`). + Use this instead of `renderDocx` when the markdown body is large (>5 KB) — the LLM does not need to repeat its own previous output as a tool argument. @@ -132,56 +134,25 @@ public class DocxRenderTool { @ToolParam(description = "Page size: A4 or LETTER (default: A4)", required = false) String pageSize) { - if (filePath == null || filePath.isBlank()) { - return "Error: filePath parameter is empty."; - } - - Path resolved; + Resolved input; try { - resolved = WorkspacePathGuard.validatePath(filePath); - } catch (Exception e) { - return "Error: path validation failed — " + e.getMessage(); - } - if (!Files.exists(resolved)) { - return "Error: file not found at " + resolved; - } - if (!Files.isRegularFile(resolved) || !Files.isReadable(resolved)) { - return "Error: path is not a readable regular file " + resolved; + input = MarkdownInputResolver.readSingle(filePath); + } catch (ResolveException e) { + return "Error: " + e.getMessage(); } - String markdown; - long mdBytes; - try { - mdBytes = Files.size(resolved); - markdown = Files.readString(resolved, StandardCharsets.UTF_8); - } catch (Exception e) { - log.error("[DocxRender] read markdown failed for {}: {}", resolved, e.getMessage(), e); - return "Error: failed to read markdown — " + e.getMessage(); - } - if (markdown.isBlank()) { - return "Error: markdown file is empty " + resolved; - } - - String safeName = sanitizeFilename(filename); - String displayName = safeName + ".docx"; - String size = (pageSize == null || pageSize.isBlank()) ? "A4" : pageSize.trim(); + String displayName = FilenameSanitizer.sanitize(filename, "document", ".docx") + ".docx"; + String size = resolveSize(pageSize); try { long t0 = System.currentTimeMillis(); - byte[] bytes = renderer.render(markdown, size); - String id = cache.put(bytes, displayName, DOCX_MIME); - long elapsed = System.currentTimeMillis() - t0; - log.info("[DocxRender] generated {} ({} bytes from {} bytes md, {}ms, id={})", - displayName, bytes.length, mdBytes, elapsed, id); - - String url = "/api/v1/files/generated/" + id; - return "Document generated: [" + displayName + "](" + url + ") (link valid for 10 minutes).\n" - + "IMPORTANT: when replying to the user you **must** use the relative path `" - + url + "` verbatim. Do **not** prepend any https://, http:// or domain — " - + "the frontend will resolve the current host automatically."; + byte[] bytes = renderer.render(input.markdown(), size); + log.info("[DocxRender] generated {} ({} bytes from {} bytes md, {}ms)", + displayName, bytes.length, input.totalBytes(), System.currentTimeMillis() - t0); + return GeneratedFileLink.resultEn(bytes, displayName, DOCX_MIME, cache, "Document", 1); } catch (Exception e) { log.error("[DocxRender] render failed for {} (source: {}): {}", - displayName, resolved, e.getMessage(), e); + displayName, input.sources().get(0), e.getMessage(), e); return "Render failed: " + e.getMessage(); } } @@ -222,91 +193,32 @@ public class DocxRenderTool { @ToolParam(description = "Page size: A4 or LETTER (default: A4)", required = false) String pageSize) { - if (filePaths == null || filePaths.isEmpty()) { - return "Error: filePaths is empty."; + Resolved input; + try { + input = MarkdownInputResolver.readManyJoined(filePaths); + } catch (ResolveException e) { + return "Error: " + e.getMessage(); } - StringBuilder combined = new StringBuilder(); - long totalBytes = 0; - List resolvedPaths = new ArrayList<>(); - for (int idx = 0; idx < filePaths.size(); idx++) { - String raw = filePaths.get(idx); - if (raw == null || raw.isBlank()) { - return "Error: filePaths[" + idx + "] is empty."; - } - Path resolved; - try { - resolved = WorkspacePathGuard.validatePath(raw); - } catch (Exception e) { - return "Error: filePaths[" + idx + "] validation failed — " + e.getMessage(); - } - if (!Files.exists(resolved)) { - return "Error: filePaths[" + idx + "] not found at " + resolved; - } - if (!Files.isRegularFile(resolved) || !Files.isReadable(resolved)) { - return "Error: filePaths[" + idx + "] is not a readable regular file " + resolved; - } - String content; - try { - totalBytes += Files.size(resolved); - content = Files.readString(resolved, StandardCharsets.UTF_8); - } catch (Exception e) { - log.error("[DocxRender] read failed for {}: {}", resolved, e.getMessage(), e); - return "Error: read failed for " + resolved + " — " + e.getMessage(); - } - if (content.isBlank()) { - return "Error: filePaths[" + idx + "] is blank " + resolved; - } - if (combined.length() > 0) combined.append("\n\n"); - combined.append(content); - resolvedPaths.add(resolved.toString()); - } - - String safeName = sanitizeFilename(filename); - String displayName = safeName + ".docx"; - String size = (pageSize == null || pageSize.isBlank()) ? "A4" : pageSize.trim(); + String displayName = FilenameSanitizer.sanitize(filename, "document", ".docx") + ".docx"; + String size = resolveSize(pageSize); try { long t0 = System.currentTimeMillis(); - byte[] bytes = renderer.render(combined.toString(), size); - String id = cache.put(bytes, displayName, DOCX_MIME); - long elapsed = System.currentTimeMillis() - t0; - log.info("[DocxRender] generated {} ({} bytes from {} files / {} bytes md, {}ms, id={})", - displayName, bytes.length, resolvedPaths.size(), totalBytes, elapsed, id); - - String url = "/api/v1/files/generated/" + id; - return "Document generated from " + resolvedPaths.size() + " files: [" - + displayName + "](" + url + ") (link valid for 10 minutes).\n" - + "IMPORTANT: when replying to the user you **must** use the relative path `" - + url + "` verbatim. Do **not** prepend any https://, http:// or domain — " - + "the frontend will resolve the current host automatically."; + byte[] bytes = renderer.render(input.markdown(), size); + log.info("[DocxRender] generated {} ({} bytes from {} files / {} bytes md, {}ms)", + displayName, bytes.length, input.fileCount(), input.totalBytes(), + System.currentTimeMillis() - t0); + return GeneratedFileLink.resultEn(bytes, displayName, DOCX_MIME, cache, + "Document", input.fileCount()); } catch (Exception e) { log.error("[DocxRender] render failed for {} (sources: {}): {}", - displayName, resolvedPaths, e.getMessage(), e); + displayName, input.sources(), e.getMessage(), e); return "Render failed: " + e.getMessage(); } } - /** - * Strip path separators and other unsafe characters from a user-supplied - * filename. Falls back to a generic name when nothing usable remains. - */ - private String sanitizeFilename(String name) { - if (name == null) return "document"; - String trimmed = name.trim(); - if (trimmed.toLowerCase().endsWith(".docx")) { - trimmed = trimmed.substring(0, trimmed.length() - 5); - } - StringBuilder sb = new StringBuilder(trimmed.length()); - for (char c : trimmed.toCharArray()) { - if (c == '/' || c == '\\' || c == ':' || c == '*' || c == '?' - || c == '"' || c == '<' || c == '>' || c == '|' || c < 0x20) { - sb.append('_'); - } else { - sb.append(c); - } - } - String cleaned = sb.toString().strip(); - return cleaned.isEmpty() ? "document" : cleaned; + private static String resolveSize(String pageSize) { + return (pageSize == null || pageSize.isBlank()) ? "A4" : pageSize.trim(); } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java new file mode 100644 index 00000000..2051f2e0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/HtmlImageRenderTool.java @@ -0,0 +1,189 @@ +package vip.mate.tool.builtin; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.BrowserType; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; +import com.microsoft.playwright.options.ScreenshotType; +import com.microsoft.playwright.options.WaitUntilState; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.browser.BrowserLauncher; +import vip.mate.tool.document.FilenameSanitizer; +import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; +import vip.mate.tool.guard.WorkspacePathGuard; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * Render arbitrary HTML to a PNG and return a one-time download URL. + * + *

Bridges the gap between HTML-producing skills (architecture diagrams, + * infographics, dashboards) and IM channels whose native message types only + * accept rasterised images. The PNG is stashed in {@link GeneratedFileCache} + * with an {@code image/png} MIME so the per-channel sniff layer + * ({@code WeComChannelAdapter}, {@code DingTalkChannelAdapter}, …) uploads it + * as a native image attachment rather than a fallback file. + */ +@Slf4j +@Component +public class HtmlImageRenderTool { + + private static final String PNG_MIME = "image/png"; + private static final int DEFAULT_VIEWPORT_WIDTH = 1440; + private static final int DEFAULT_VIEWPORT_HEIGHT = 900; + private static final int MAX_VIEWPORT_DIMENSION = 4096; + private static final int SET_CONTENT_TIMEOUT_MS = 15_000; + + private final GeneratedFileCache cache; + + private volatile Playwright sharedPlaywright; + private final Object playwrightLock = new Object(); + + public HtmlImageRenderTool(GeneratedFileCache cache) { + this.cache = cache; + } + + @Tool(description = """ + Render HTML to a PNG image and return a one-time download URL. + + Use this whenever the user wants an HTML artifact (architecture + diagram, infographic, dashboard, mockup, ...) delivered as an + *image* — especially when the chat is happening on an IM channel + (WeCom / 企业微信, DingTalk, Feishu, Telegram, Discord) where users + cannot click through a raw HTML link. + + The returned URL is `/api/v1/files/generated/` with MIME + `image/png`. Channel adapters detect this MIME and upload the + bytes as a native image message, so the recipient sees an inline + picture rather than a file attachment. + + Typical workflow when paired with an HTML-producing skill: + 1. write_file(filePath="diagram.html", content="...") + 2. render_html_image(filePath="diagram.html", filename="diagram") + 3. return the markdown link to the user + + Or directly, without going through disk: + 1. render_html_image(html="...", filename="diagram") + + Exactly one of `filePath` or `html` must be supplied. The link is + valid for 10 minutes. + """) + public String render_html_image( + @ToolParam(description = "Path to an HTML file on disk (workspace-relative or absolute). Mutually exclusive with `html`.", required = false) + String filePath, + @ToolParam(description = "Inline HTML source. Mutually exclusive with `filePath`.", required = false) + String html, + @ToolParam(description = "Output filename without extension, e.g. 'architecture'") + String filename, + @ToolParam(description = "Viewport width in px (default 1440, max 4096)", required = false) + Integer width, + @ToolParam(description = "Viewport height in px (default 900, max 4096). Ignored when fullPage=true except as initial layout hint.", required = false) + Integer height, + @ToolParam(description = "Capture full scrollable page (default true). Set false to only capture the viewport.", required = false) + Boolean fullPage) { + + String source; + try { + source = resolveHtml(filePath, html); + } catch (IllegalArgumentException e) { + return "Error: " + e.getMessage(); + } catch (Exception e) { + log.error("[HtmlImageRender] failed to load HTML: {}", e.getMessage(), e); + return "Error: failed to load HTML — " + e.getMessage(); + } + + int vw = clampViewport(width, DEFAULT_VIEWPORT_WIDTH); + int vh = clampViewport(height, DEFAULT_VIEWPORT_HEIGHT); + boolean full = fullPage == null || fullPage; + String displayName = FilenameSanitizer.sanitize(filename, "image", ".png") + ".png"; + + byte[] pngBytes; + try { + pngBytes = renderToPng(source, vw, vh, full); + } catch (Exception e) { + log.error("[HtmlImageRender] render failed for {}: {}", displayName, e.getMessage(), e); + String hint = e.getMessage() != null && e.getMessage().contains("Executable doesn't exist") + ? " Hint: run `mvn exec:java -e -Dexec.mainClass=\"com.microsoft.playwright.CLI\" -Dexec.args=\"install chromium\"` to install the bundled browser." + : ""; + return "Render failed: " + e.getMessage() + hint; + } + + log.info("[HtmlImageRender] rendered {} ({} bytes, viewport={}x{}, fullPage={})", + displayName, pngBytes.length, vw, vh, full); + return GeneratedFileLink.resultZh(pngBytes, displayName, PNG_MIME, cache, "图片"); + } + + private String resolveHtml(String filePath, String inlineHtml) throws Exception { + boolean hasPath = filePath != null && !filePath.isBlank(); + boolean hasInline = inlineHtml != null && !inlineHtml.isBlank(); + if (hasPath == hasInline) { + throw new IllegalArgumentException( + "Provide exactly one of `filePath` or `html` (not both, not neither)."); + } + if (hasPath) { + Path path = WorkspacePathGuard.validatePath(filePath); + if (!Files.exists(path)) { + throw new IllegalArgumentException("HTML file not found: " + filePath); + } + if (Files.isDirectory(path)) { + throw new IllegalArgumentException("Path is a directory, not a file: " + filePath); + } + return Files.readString(path, StandardCharsets.UTF_8); + } + return inlineHtml; + } + + private byte[] renderToPng(String source, int viewportWidth, int viewportHeight, boolean fullPage) { + Playwright pw = getOrCreatePlaywright(); + BrowserType.LaunchOptions opts = new BrowserType.LaunchOptions() + .setHeadless(true) + .setArgs(BrowserLauncher.chromiumLaunchArgs()); + Browser browser = pw.chromium().launch(opts); + try { + BrowserContext ctx = browser.newContext(new Browser.NewContextOptions() + .setViewportSize(viewportWidth, viewportHeight) + .setDeviceScaleFactor(2.0)); + try { + Page page = ctx.newPage(); + page.setContent(source, new Page.SetContentOptions() + .setWaitUntil(WaitUntilState.NETWORKIDLE) + .setTimeout(SET_CONTENT_TIMEOUT_MS)); + return page.screenshot(new Page.ScreenshotOptions() + .setFullPage(fullPage) + .setType(ScreenshotType.PNG)); + } finally { + try { ctx.close(); } catch (Exception ignored) {} + } + } finally { + try { browser.close(); } catch (Exception ignored) {} + } + } + + /** + * Lazily create one Playwright instance per JVM. Playwright.create() + * spawns a Node.js child process and costs ~1–2 s; keeping the instance + * around means subsequent screenshots only pay the browser-launch cost. + */ + private Playwright getOrCreatePlaywright() { + Playwright local = sharedPlaywright; + if (local != null) return local; + synchronized (playwrightLock) { + if (sharedPlaywright == null) { + sharedPlaywright = Playwright.create(); + } + return sharedPlaywright; + } + } + + private static int clampViewport(Integer requested, int fallback) { + if (requested == null || requested <= 0) return fallback; + return Math.min(requested, MAX_VIEWPORT_DIMENSION); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java index a15c4538..7041bf78 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/ImageGenerateTool.java @@ -13,6 +13,7 @@ import vip.mate.task.AsyncTaskService; import vip.mate.task.model.AsyncTaskInfo; import vip.mate.tool.image.*; +import java.util.ArrayList; import java.util.List; import java.util.StringJoiner; @@ -30,19 +31,23 @@ public class ImageGenerateTool { private final ImageProviderRegistry providerRegistry; private final SystemSettingService systemSettingService; private final AsyncTaskService asyncTaskService; + private final ImageReferenceLoader imageReferenceLoader; @vip.mate.tool.ConcurrencyUnsafe("creates async tasks and persists generated artifacts; provider rate limits also forbid parallel calls") - @Tool(description = "Image generation tool. Supports actions: generate (default), list (show available providers), " - + "status (check task status). Some providers are async (30s-2min), results auto-displayed in conversation.") + @Tool(description = "Image generation tool. Supports actions: generate (default — text-to-image, OR image-edit when " + + "image/images parameters are set), list (show available providers/models), status (check task status). " + + "Reference images may be local paths, http(s) URLs, data: URLs, or msg:: for an attachment " + + "from an earlier conversation message. Async providers take 30s-2min; results auto-display in the conversation.") public String image_generate( @ToolParam(description = "Action type: generate, list, status. Default: generate", required = false) String action, @ToolParam(description = "Image content description, be detailed (required for generate)", required = false) String prompt, + @ToolParam(description = "Single reference image for edit mode. Path / http(s) URL / data: URL / msg:[:]", required = false) String image, + @ToolParam(description = "Multiple reference images for edit mode (provider caps the count). Same formats as 'image'.", required = false) List images, @ToolParam(description = "Image size: 1024x1024 / 1024x1792 / 1792x1024", required = false) String size, @ToolParam(description = "Aspect ratio: 1:1 / 16:9 / 9:16, default 1:1", required = false) String aspectRatio, @ToolParam(description = "Generation count (1-4), default 1", required = false) Integer count, @ToolParam(description = "Model name (optional)", required = false) String model, @ToolParam(description = "Task ID to check status (for status action)", required = false) String taskId, - // RFC-063r §2.5: ToolContext is hidden from the LLM by JsonSchemaGenerator. @Nullable ToolContext ctx ) { String normalizedAction = (action == null || action.isBlank()) ? "generate" : action.trim().toLowerCase(); @@ -50,7 +55,7 @@ public class ImageGenerateTool { return switch (normalizedAction) { case "list" -> handleListAction(); case "status" -> handleStatusAction(taskId, ctx); - default -> handleGenerateAction(prompt, size, aspectRatio, count, model, ctx); + default -> handleGenerateAction(prompt, image, images, size, aspectRatio, count, model, ctx); }; } @@ -120,7 +125,8 @@ public class ImageGenerateTool { // ==================== action=generate ==================== - private String handleGenerateAction(String prompt, String size, String aspectRatio, + private String handleGenerateAction(String prompt, String image, List images, + String size, String aspectRatio, Integer count, String model, @Nullable ToolContext ctx) { String conversationId = ToolExecutionContext.conversationId(ctx); String username = ToolExecutionContext.username(ctx); @@ -133,12 +139,33 @@ public class ImageGenerateTool { return "错误:prompt 为必填参数,请描述你想要生成的图片内容"; } + // Combine the singular and plural forms — the agent picks whichever is + // ergonomic. Order: image (first) then images[]. + List referenceInputs = new ArrayList<>(); + if (image != null && !image.isBlank()) { + referenceInputs.add(image); + } + if (images != null) { + for (String s : images) { + if (s != null && !s.isBlank()) referenceInputs.add(s); + } + } + + List inputImages; + try { + inputImages = imageReferenceLoader.loadAll(referenceInputs, conversationId); + } catch (Exception e) { + log.warn("[ImageGenerateTool] Failed to load reference images: {}", e.getMessage()); + return "错误:无法加载参考图片:" + e.getMessage(); + } + ImageGenerationRequest request = ImageGenerationRequest.builder() .prompt(prompt) .size(size) .aspectRatio(aspectRatio != null ? aspectRatio : "1:1") .count(count != null ? count : 1) .model(model) + .inputImages(inputImages) .build(); ImageGenerationResult result = imageGenerationService.submitGeneration( diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/PdfRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/PdfRenderTool.java new file mode 100644 index 00000000..0c8f42b0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/PdfRenderTool.java @@ -0,0 +1,172 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.document.FilenameSanitizer; +import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; +import vip.mate.tool.document.MarkdownInputResolver; +import vip.mate.tool.document.MarkdownInputResolver.Resolved; +import vip.mate.tool.document.MarkdownInputResolver.ResolveException; +import vip.mate.tool.document.pdf.MarkdownPdfRenderer; +import vip.mate.tool.document.pdf.PdfProperties; + +import java.util.Locale; + +/** + * Render a brand-new .pdf from Markdown. Two backends sit behind this tool: + * a LibreOffice subprocess (preferred when {@code soffice} is available, best + * Chinese typography) and an in-process OpenHTMLtoPDF path (always available, + * supports cover / page header / page footer driven by YAML frontmatter). + * The orchestrator picks one per call; see {@link MarkdownPdfRenderer}. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class PdfRenderTool { + + private static final String PDF_MIME = "application/pdf"; + + private final MarkdownPdfRenderer renderer; + private final GeneratedFileCache cache; + + @Tool(description = """ + Render a NEW .pdf file from Markdown and return a one-time download URL. + + **MUST use this tool (NOT renderDocx / renderDocxFromFile) whenever the user + says any of: "PDF", ".pdf", "导出 PDF", "生成 pdf", "另存为 PDF", "出一份 PDF", + "save as PDF", "export to PDF".** PDF is a final, non-editable deliverable + format; if the user asked for it explicitly, do not silently substitute docx. + + **Do NOT bypass this tool by shelling out to `chrome --headless --print-to-pdf`, + `wkhtmltopdf`, `weasyprint`, or any markdown-to-PDF Python skill. Those produce + a PDF on local disk that is NOT registered in mateclaw's download cache, so + the user has no clickable download link and the file leaks into the workspace. + Always use this tool instead — it returns a `/api/v1/files/generated/` URL + the user can download from chat.** + + Use for FINAL deliverables — reports, white-papers, contracts, briefings — + where the recipient should not edit the document. + + Markdown convention: + - Standard subset: headings (# ## ###), bold, italic, lists, tables, + blockquotes, code blocks, links. + - Optional YAML frontmatter at the top of the markdown drives cover + page and page header / footer: + + --- + title: 季度业务回顾 + subtitle: Q1 2026 + header: 内部资料 - 仅限分发 + footer: Mate Inc. © 2026 + --- + + # 第一章 + ... + + - Without frontmatter, the first `# H1` heading is used as the cover + title and pages are numbered automatically with no header / footer. + + For markdown bodies larger than ~5 KB, prefer renderPdfFromFile. + + Returns a markdown link the user can click to download the file. + The link is valid for 10 minutes. + """) + public String renderPdf( + @ToolParam(description = "Document content in Markdown format (optional YAML frontmatter for cover / header / footer)") + String markdown, + @ToolParam(description = "Output filename without extension, e.g. 'q1-review'") + String filename, + @ToolParam(description = "Page size: A4 or LETTER (default: A4)", required = false) + String pageSize, + @ToolParam(description = "Engine: 'auto' (default), 'html' (force in-process), or 'libreoffice' (force soffice)", required = false) + String engine) { + + if (markdown == null || markdown.isBlank()) { + return "错误:markdown 参数为空,无法生成 PDF。"; + } + + String displayName = FilenameSanitizer.sanitize(filename, "document", ".pdf") + ".pdf"; + String size = resolveSize(pageSize); + PdfProperties.Engine eng = resolveEngine(engine); + + try { + MarkdownPdfRenderer.Result result = renderer.render(markdown, size, eng); + log.info("[PdfRender] generated {} ({} bytes via {})", + displayName, result.bytes().length, result.backend()); + return GeneratedFileLink.resultZh(result.bytes(), displayName, PDF_MIME, cache, "PDF"); + } catch (Exception e) { + log.error("[PdfRender] render failed for {}: {}", displayName, e.getMessage(), e); + return "渲染失败:" + e.getMessage(); + } + } + + @Tool(description = """ + Render a .pdf from a markdown FILE on disk and return a one-time download URL. + + **MUST use this tool (NOT renderDocxFromFile) whenever the user asks for a + PDF / .pdf / 导出 PDF / 生成 pdf and the markdown body is already on disk.** + Do not silently substitute docx when the user explicitly requested PDF. + + Use this instead of `renderPdf` when the markdown body is large (>5 KB) — the + LLM does not need to repeat its own previous output as a tool argument. + + Typical workflow: + 1. write_file(path="report.md", content="---\\ntitle: ...\\n---\\n# ...") + 2. renderPdfFromFile(filePath="report.md", filename="q1-review") + 3. return the download link to the user + + The markdown file is read with UTF-8. Path resolution honors the workspace + boundary (same rules as read_file / write_file). + + Same supported markdown subset and frontmatter convention as renderPdf. + """) + public String renderPdfFromFile( + @ToolParam(description = "Absolute or workspace-relative path to a markdown file") + String filePath, + @ToolParam(description = "Output filename without extension, e.g. 'q1-review'") + String filename, + @ToolParam(description = "Page size: A4 or LETTER (default: A4)", required = false) + String pageSize, + @ToolParam(description = "Engine: 'auto' (default), 'html', or 'libreoffice'", required = false) + String engine) { + + Resolved input; + try { + input = MarkdownInputResolver.readSingle(filePath); + } catch (ResolveException e) { + return "Error: " + e.getMessage(); + } + + String displayName = FilenameSanitizer.sanitize(filename, "document", ".pdf") + ".pdf"; + String size = resolveSize(pageSize); + PdfProperties.Engine eng = resolveEngine(engine); + + try { + MarkdownPdfRenderer.Result result = renderer.render(input.markdown(), size, eng); + log.info("[PdfRender] generated {} ({} bytes via {} from {} bytes md)", + displayName, result.bytes().length, result.backend(), input.totalBytes()); + return GeneratedFileLink.resultEn(result.bytes(), displayName, PDF_MIME, cache, "Document", 1); + } catch (Exception e) { + log.error("[PdfRender] render failed for {} (source: {}): {}", + displayName, input.sources().get(0), e.getMessage(), e); + return "Render failed: " + e.getMessage(); + } + } + + private static String resolveSize(String pageSize) { + return (pageSize == null || pageSize.isBlank()) ? "A4" : pageSize.trim(); + } + + private static PdfProperties.Engine resolveEngine(String engine) { + if (engine == null || engine.isBlank()) return PdfProperties.Engine.AUTO; + try { + return PdfProperties.Engine.valueOf(engine.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + return PdfProperties.Engine.AUTO; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/PptxRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/PptxRenderTool.java new file mode 100644 index 00000000..013c90f1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/PptxRenderTool.java @@ -0,0 +1,149 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.document.FilenameSanitizer; +import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; +import vip.mate.tool.document.MarkdownInputResolver; +import vip.mate.tool.document.MarkdownInputResolver.Resolved; +import vip.mate.tool.document.MarkdownInputResolver.ResolveException; +import vip.mate.tool.document.MarkdownPptxRenderer; + +/** + * Render a brand-new .pptx deck from Markdown, in-process via Apache POI. + * The LLM produces a Marp-style markdown body where {@code ---} separates + * slides, {@code # / ## / ###} is the slide title, and {@code - item} are + * bullets. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class PptxRenderTool { + + private static final String PPTX_MIME = + "application/vnd.openxmlformats-officedocument.presentationml.presentation"; + + private final MarkdownPptxRenderer renderer; + private final GeneratedFileCache cache; + + @Tool(description = """ + Render a NEW .pptx slide deck from Markdown and return a one-time download URL. + Use for creating presentations: pitch decks, project plans, talks, briefings. + + Markdown convention (Marp-style): + - `---` on its own line separates slides. + - The first `# / ## / ###` of a slide becomes its title. + - Lines starting with `-` or `*` become bullet points. + - Other non-blank lines become plain paragraphs. + - `` HTML comments become speaker notes. + + Example: + # My Presentation + + By Author Name + + --- + + ## Topic 1 + + - Point one + - Point two + - Point three + + + + --- + + ## Conclusion + + Thanks! + + For markdown bodies larger than ~5 KB, prefer renderPptxFromFile (read + from disk) — passing huge markdown as a tool argument burns LLM tokens. + + Returns a markdown link the user can click to download the file. + The link is valid for 10 minutes. + """) + public String renderPptx( + @ToolParam(description = "Slide content in Marp-style Markdown ('---' between slides)") + String markdown, + @ToolParam(description = "Output filename without extension, e.g. 'pitch-deck'") + String filename, + @ToolParam(description = "Aspect ratio: '16:9' (default, widescreen) or '4:3' (legacy)", required = false) + String aspectRatio) { + + if (markdown == null || markdown.isBlank()) { + return "错误:markdown 参数为空,无法生成演示文稿。"; + } + + String displayName = FilenameSanitizer.sanitize(filename, "presentation", ".pptx") + ".pptx"; + String ratio = resolveRatio(aspectRatio); + + try { + long t0 = System.currentTimeMillis(); + byte[] bytes = renderer.render(markdown, ratio); + log.info("[PptxRender] generated {} ({} bytes, {}ms)", + displayName, bytes.length, System.currentTimeMillis() - t0); + return GeneratedFileLink.resultZh(bytes, displayName, PPTX_MIME, cache, "演示文稿"); + } catch (Exception e) { + log.error("[PptxRender] render failed for {}: {}", displayName, e.getMessage(), e); + return "渲染失败:" + e.getMessage(); + } + } + + @Tool(description = """ + Render a .pptx deck from a markdown FILE on disk and return a one-time download URL. + Use this instead of `renderPptx` when the markdown body is large (>5 KB) — the + LLM does not need to repeat its own previous output as a tool argument. + + Typical workflow: + 1. write_file(path="deck.md", content="# Title\\n\\n---\\n\\n## Topic\\n\\n- ...") + 2. renderPptxFromFile(filePath="deck.md", filename="pitch-deck") + 3. return the download link to the user + + The markdown file is read with UTF-8. Path resolution honors the workspace + boundary (same rules as read_file / write_file). + + Same supported Marp-style markdown subset as renderPptx (`---` slide breaks, + `# / ##` titles, `-` / `*` bullets, `` speaker notes). + """) + public String renderPptxFromFile( + @ToolParam(description = "Absolute or workspace-relative path to a markdown file") + String filePath, + @ToolParam(description = "Output filename without extension, e.g. 'pitch-deck'") + String filename, + @ToolParam(description = "Aspect ratio: '16:9' (default) or '4:3'", required = false) + String aspectRatio) { + + Resolved input; + try { + input = MarkdownInputResolver.readSingle(filePath); + } catch (ResolveException e) { + return "Error: " + e.getMessage(); + } + + String displayName = FilenameSanitizer.sanitize(filename, "presentation", ".pptx") + ".pptx"; + String ratio = resolveRatio(aspectRatio); + + try { + long t0 = System.currentTimeMillis(); + byte[] bytes = renderer.render(input.markdown(), ratio); + log.info("[PptxRender] generated {} ({} bytes from {} bytes md, {}ms)", + displayName, bytes.length, input.totalBytes(), + System.currentTimeMillis() - t0); + return GeneratedFileLink.resultEn(bytes, displayName, PPTX_MIME, cache, "Presentation", 1); + } catch (Exception e) { + log.error("[PptxRender] render failed for {} (source: {}): {}", + displayName, input.sources().get(0), e.getMessage(), e); + return "Render failed: " + e.getMessage(); + } + } + + private static String resolveRatio(String aspectRatio) { + return (aspectRatio == null || aspectRatio.isBlank()) ? "16:9" : aspectRatio.trim(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillFileTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillFileTool.java index a6167a8c..c2f14b36 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillFileTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillFileTool.java @@ -265,7 +265,7 @@ public class SkillFileTool { } @Tool(description = """ - List all currently available Skills (documentation packages). + List currently available Skills (documentation packages). IMPORTANT: Skills are NOT directly callable as tools. Each name returned here is a `skillName` argument, not a tool name. To use @@ -273,6 +273,17 @@ public class SkillFileTool { first to read its instructions, then follow what SKILL.md tells you. Calling a skill name as a tool will fail with "Tool not found". + Search strategy when looking for a specific skill: + - The default page is 20 of N — if "Showing: 20 of " appears + and you don't see what you're after, retry with `keyword=` + (matched against name + description, case-insensitive) or raise `limit` + up to 50. + - If the user mentions an exact skill name (e.g. "tencent-meeting-mcp"), + skip this tool and go straight to + `readSkillFile(skillName="", filePath="SKILL.md")` — + that bypasses the catalog truncation entirely and either returns + the skill's instructions or a clear "skill not found" error. + Note: this returns Skills (vendor-installable docs), not Agents. For Agents, use `listAvailableAgents`. @@ -280,7 +291,7 @@ public class SkillFileTool { """) public String listAvailableSkills( @JsonProperty(required = false) - @JsonPropertyDescription("Optional keyword matched against skill name or description") + @JsonPropertyDescription("Optional keyword matched against skill name or description (case-insensitive). Use this when a specific skill name was mentioned but didn't appear in the default page.") String keyword, @JsonProperty(required = false) @@ -299,6 +310,16 @@ public class SkillFileTool { int safeLimit = limit == null || limit <= 0 ? 20 : Math.min(limit, 50); String kw = keyword == null ? "" : keyword.trim().toLowerCase(); + // Push freshly installed skills to the top of the truncated page so + // a user who just installed something can still find it without + // remembering to pass keyword=. Same window the prompt catalog uses. + java.time.LocalDateTime recencyCutoff = java.time.LocalDateTime.now() + .minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW); + // sortResolved gives the RECOMMENDED ordering; the secondary sort + // below uses the JDK's stable sort to lift recently-installed skills + // to the top while preserving RECOMMENDED order among same-recency + // entries — no need to thread the (package-private) recommended + // comparator back through here. List activeSkills = SkillCatalogSorter.sortResolved( runtimeService.getActiveSkills().stream() .filter(s -> SkillCatalogSorter.sourceMatches(s, source)) @@ -307,7 +328,10 @@ public class SkillFileTool { || containsIgnoreCase(s.getName(), kw) || containsIgnoreCase(s.getDescription(), kw)) .toList(), - SkillCatalogSort.RECOMMENDED); + SkillCatalogSort.RECOMMENDED).stream() + .sorted(java.util.Comparator.comparingInt((ResolvedSkill s) -> + SkillRuntimeService.isRecentlyInstalled(s, recencyCutoff) ? 0 : 1)) + .toList(); if (activeSkills.isEmpty()) { return "No skills are currently available."; @@ -338,8 +362,17 @@ public class SkillFileTool { } sb.append(" |\n"); } - sb.append("\nShowing: ").append(Math.min(safeLimit, activeSkills.size())) + int shown = Math.min(safeLimit, activeSkills.size()); + sb.append("\nShowing: ").append(shown) .append(" of ").append(activeSkills.size()).append(" skill(s)."); + if (shown < activeSkills.size()) { + // Surface the truncation hint so the LLM knows how to widen the + // search instead of concluding the missing skill doesn't exist. + sb.append(" Result truncated — retry with `keyword=` ") + .append("to search the full catalog, or `limit=50` to see more rows. ") + .append("If the user gave an exact skill name, prefer ") + .append("`readSkillFile(skillName=\"\", filePath=\"SKILL.md\")` directly."); + } return sb.toString(); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillScriptTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillScriptTool.java index f6901bd0..378dc165 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillScriptTool.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/SkillScriptTool.java @@ -13,7 +13,6 @@ import vip.mate.skill.runtime.model.ResolvedSkill; import vip.mate.skill.secret.SkillSecretService; import java.nio.file.Path; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; @@ -40,7 +39,10 @@ public class SkillScriptTool { Parameters: - skillName: Name of the skill - scriptPath: Relative path to script under scripts/ directory (e.g., "scripts/run.py") - - args: Optional comma-separated arguments to pass to the script + - args: Optional list of script arguments. Each element is passed as a separate + CLI argument exactly as written — no shell interpretation, no splitting. + For a JSON payload, wrap it as a single-element list, e.g. + ["{\\"date\\":\\"2026-05-12\\",\\"topic\\":\\"meeting\\"}"]. Returns: JSON with exitCode, stdout, stderr @@ -57,33 +59,33 @@ public class SkillScriptTool { String scriptPath, @JsonProperty(required = false) - @JsonPropertyDescription("Optional comma-separated script arguments") - String args + @JsonPropertyDescription("Optional list of script arguments. Each element is passed as one CLI arg verbatim. Wrap a JSON payload as a single-element list.") + List args ) { log.info("Executing skill script: skill={}, script={}, args={}", skillName, scriptPath, args); - // 查找 active skill + // Look up active skill. ResolvedSkill skill = runtimeService.findActiveSkill(skillName); if (skill == null) { return formatError("Skill '" + skillName + "' not found or not enabled"); } - // 必须是目录型 skill + // Must be a directory-backed skill. if (skill.getSkillDir() == null) { return formatError("Skill '" + skillName + "' is database-based, no script execution available"); } - // 验证脚本路径(必须在 scripts/ 下) + // Validate script path (must live under scripts/). Path resolvedPath = accessPolicy.validateScriptPath(skill.getSkillDir(), scriptPath); if (resolvedPath == null) { return formatError("Invalid or unsafe script path: " + scriptPath); } - // 解析参数 - List argList = null; - if (args != null && !args.isBlank()) { - argList = Arrays.asList(args.split(",")); - } + // Pass args straight through. No splitting — arbitrary delimiters + // (notably commas inside JSON payloads) used to shatter a single + // logical argument into multiple positional args, which broke any + // skill expecting a JSON-encoded payload. + List argList = (args == null || args.isEmpty()) ? null : args; // RFC-091 settings bridge — pull this skill's stored secrets // (e.g. AIRTABLE_API_KEY) and inject them as env vars for the diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/XlsxRenderTool.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/XlsxRenderTool.java new file mode 100644 index 00000000..c1f728e1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/XlsxRenderTool.java @@ -0,0 +1,133 @@ +package vip.mate.tool.builtin; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.stereotype.Component; +import vip.mate.tool.document.FilenameSanitizer; +import vip.mate.tool.document.GeneratedFileCache; +import vip.mate.tool.document.GeneratedFileLink; +import vip.mate.tool.document.MarkdownInputResolver; +import vip.mate.tool.document.MarkdownInputResolver.Resolved; +import vip.mate.tool.document.MarkdownInputResolver.ResolveException; +import vip.mate.tool.document.MarkdownXlsxRenderer; + +/** + * Render a brand-new .xlsx workbook from a Markdown body, in-process via + * Apache POI. Mirrors {@link DocxRenderTool}'s shape: the LLM produces a + * Markdown body where each {@code # Heading} starts a sheet and the pipe-style + * table beneath it becomes the sheet content. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class XlsxRenderTool { + + private static final String XLSX_MIME = + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + + private final MarkdownXlsxRenderer renderer; + private final GeneratedFileCache cache; + + @Tool(description = """ + Render a NEW .xlsx workbook from Markdown and return a one-time download URL. + Use for creating spreadsheets: financial reports, data tables, comparison + matrices, plans, schedules. + + Markdown convention: + - Each `# Sheet Name` starts a new sheet. + - The pipe-style table under the heading becomes the sheet body. + - The first table row is rendered as the header (bold, light-grey fill, + frozen). Numeric cells are auto-detected and stored as numbers so + Excel can sort / sum them; non-numeric cells stay as strings. + - Sub-headings (## / ###) and free-form prose are ignored — xlsx is + tabular and there is nowhere sensible to put them. + + Example: + # Q1 Sales + | Region | Revenue | Growth | + | --- | --- | --- | + | North | 12000 | 0.15 | + | South | 8500 | 0.08 | + + # Q2 Sales + | Region | Revenue | + | --- | --- | + | North | 14000 | + + For markdown bodies larger than ~5 KB, prefer renderXlsxFromFile (read + from disk) — passing huge markdown as a tool argument burns LLM tokens. + + Returns a markdown link the user can click to download the file. + The link is valid for 10 minutes. + """) + public String renderXlsx( + @ToolParam(description = "Workbook content in Markdown format (sheets as `# Heading`, tables as `| ... |`)") + String markdown, + @ToolParam(description = "Output filename without extension, e.g. 'q1-sales'") + String filename) { + + if (markdown == null || markdown.isBlank()) { + return "错误:markdown 参数为空,无法生成工作簿。"; + } + + String displayName = FilenameSanitizer.sanitize(filename, "workbook", ".xlsx") + ".xlsx"; + + try { + long t0 = System.currentTimeMillis(); + byte[] bytes = renderer.render(markdown); + log.info("[XlsxRender] generated {} ({} bytes, {}ms)", + displayName, bytes.length, System.currentTimeMillis() - t0); + return GeneratedFileLink.resultZh(bytes, displayName, XLSX_MIME, cache, "工作簿"); + } catch (Exception e) { + log.error("[XlsxRender] render failed for {}: {}", displayName, e.getMessage(), e); + return "渲染失败:" + e.getMessage(); + } + } + + @Tool(description = """ + Render a .xlsx workbook from a markdown FILE on disk and return a one-time download URL. + Use this instead of `renderXlsx` when the markdown body is large (>5 KB) — the + LLM does not need to repeat its own previous output as a tool argument. + + Typical workflow: + 1. write_file(path="report.md", content="# Q1\\n| ... |\\n...") + 2. renderXlsxFromFile(filePath="report.md", filename="quarterly-report") + 3. return the download link to the user + + The markdown file is read with UTF-8. Path resolution honors the workspace + boundary (same rules as read_file / write_file). + + Same supported markdown subset as renderXlsx (`# Heading` per sheet, + pipe-style tables; numeric cells auto-detected). + """) + public String renderXlsxFromFile( + @ToolParam(description = "Absolute or workspace-relative path to a markdown file") + String filePath, + @ToolParam(description = "Output filename without extension, e.g. 'quarterly-report'") + String filename) { + + Resolved input; + try { + input = MarkdownInputResolver.readSingle(filePath); + } catch (ResolveException e) { + return "Error: " + e.getMessage(); + } + + String displayName = FilenameSanitizer.sanitize(filename, "workbook", ".xlsx") + ".xlsx"; + + try { + long t0 = System.currentTimeMillis(); + byte[] bytes = renderer.render(input.markdown()); + log.info("[XlsxRender] generated {} ({} bytes from {} bytes md, {}ms)", + displayName, bytes.length, input.totalBytes(), + System.currentTimeMillis() - t0); + return GeneratedFileLink.resultEn(bytes, displayName, XLSX_MIME, cache, "Workbook", 1); + } catch (Exception e) { + log.error("[XlsxRender] render failed for {} (source: {}): {}", + displayName, input.sources().get(0), e.getMessage(), e); + return "Render failed: " + e.getMessage(); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/controller/ToolController.java b/mateclaw-server/src/main/java/vip/mate/tool/controller/ToolController.java index 93e3ff4c..e2074d3f 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/controller/ToolController.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/controller/ToolController.java @@ -5,7 +5,9 @@ import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.*; import vip.mate.common.result.R; +import vip.mate.tool.model.AvailableToolDTO; import vip.mate.tool.model.ToolEntity; +import vip.mate.tool.service.AvailableToolService; import vip.mate.tool.service.ToolService; import java.util.List; @@ -22,6 +24,7 @@ import java.util.List; public class ToolController { private final ToolService toolService; + private final AvailableToolService availableToolService; @Operation(summary = "获取工具列表") @GetMapping @@ -35,6 +38,12 @@ public class ToolController { return R.ok(toolService.listEnabledTools()); } + @Operation(summary = "获取员工可绑定的全部原子工具(含 MCP)") + @GetMapping("/available") + public R> listAvailable() { + return R.ok(availableToolService.listAvailable()); + } + @Operation(summary = "获取工具详情") @GetMapping("/{id}") public R get(@PathVariable Long id) { diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/FilenameSanitizer.java b/mateclaw-server/src/main/java/vip/mate/tool/document/FilenameSanitizer.java new file mode 100644 index 00000000..d7248d4f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/FilenameSanitizer.java @@ -0,0 +1,42 @@ +package vip.mate.tool.document; + +import java.util.Locale; + +/** + * Strip path separators and other characters that are illegal in download + * filenames from an LLM-supplied name. The LLM is allowed to suffix the + * extension itself (e.g. "report.docx") — {@link #sanitize} drops a known + * extension before sanitizing so callers can re-append it consistently. + */ +public final class FilenameSanitizer { + + private FilenameSanitizer() {} + + /** + * @param name candidate name from the LLM (may be null / blank / contain ext) + * @param fallback name to use when {@code name} is null, blank, or sanitizes to empty + * @param dropExt optional trailing extension to strip case-insensitively + * before sanitizing (e.g. {@code ".docx"}); pass {@code null} + * to skip + * @return a non-blank base name with no path separators or shell metacharacters + */ + public static String sanitize(String name, String fallback, String dropExt) { + if (name == null) return fallback; + String trimmed = name.trim(); + if (dropExt != null && !dropExt.isEmpty() + && trimmed.toLowerCase(Locale.ROOT).endsWith(dropExt.toLowerCase(Locale.ROOT))) { + trimmed = trimmed.substring(0, trimmed.length() - dropExt.length()); + } + StringBuilder sb = new StringBuilder(trimmed.length()); + for (char c : trimmed.toCharArray()) { + if (c == '/' || c == '\\' || c == ':' || c == '*' || c == '?' + || c == '"' || c == '<' || c == '>' || c == '|' || c < 0x20) { + sb.append('_'); + } else { + sb.append(c); + } + } + String cleaned = sb.toString().strip(); + return cleaned.isEmpty() ? fallback : cleaned; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java index e7bd8547..35ad1756 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileCache.java @@ -7,6 +7,8 @@ import java.time.Duration; import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * In-memory cache of bytes produced by tools (e.g. {@code DocxRenderTool}) and @@ -23,6 +25,22 @@ public class GeneratedFileCache { public static final Duration TTL = Duration.ofMinutes(10); + /** + * URL pattern for in-memory generated files served by + * {@code GeneratedFileController}. Public so channel adapters and graph + * nodes share a single source of truth. + */ + public static final Pattern GENERATED_URL_PATTERN = + Pattern.compile("/api/v1/files/generated/([a-zA-Z0-9-]+)"); + + /** + * User-visible warning swapped in for a cache-miss URL. Identical + * wording to the channel-side fallback so users see one consistent + * message regardless of which surface (web, IM, etc.) renders it. + */ + public static final String MISSING_REFERENCE_NOTICE = + "⚠️ 文件未真正生成(模型未调用文档生成工具),请重新发送请求"; + private final ConcurrentHashMap entries = new ConcurrentHashMap<>(); public record Entry(byte[] bytes, String filename, String mimeType, long expireAt) { @@ -66,4 +84,35 @@ public class GeneratedFileCache { long now = System.currentTimeMillis(); entries.entrySet().removeIf(e -> e.getValue().expireAt() <= now); } + + /** + * Replace any {@code /api/v1/files/generated/{id}} URL in {@code text} + * whose id is NOT present (or has expired) in this cache with + * {@link #MISSING_REFERENCE_NOTICE}. URLs whose ids ARE in the cache are + * left intact so downstream channel adapters can still rewrite them + * into native attachments. + * + *

Cache misses are nearly always LLM hallucinations — the model + * emitted a UUID-shaped string without ever calling a render tool. + * Without this scrub, every channel that receives the answer (Web, + * Slack, DingTalk, Telegram, …) would render a clickable link that + * 404s, and IM clients save the 404 HTML body as a {@code .docx} + * which users then report as "corrupted file". + */ + public String scrubMissingReferences(String text) { + if (text == null || text.isEmpty()) return text; + Matcher m = GENERATED_URL_PATTERN.matcher(text); + if (!m.find()) return text; + StringBuilder out = new StringBuilder(); + m.reset(); + while (m.find()) { + String id = m.group(1); + Entry entry = entries.get(id); + boolean live = entry != null && !entry.expired(); + String replacement = live ? m.group(0) : MISSING_REFERENCE_NOTICE; + m.appendReplacement(out, Matcher.quoteReplacement(replacement)); + } + m.appendTail(out); + return out.toString(); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java new file mode 100644 index 00000000..06540941 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/GeneratedFileLink.java @@ -0,0 +1,58 @@ +package vip.mate.tool.document; + +/** + * Stash freshly-rendered bytes into the {@link GeneratedFileCache} and format + * the markdown link the tool returns to the LLM. + * + *

Two locales are exposed because mateclaw's existing convention has the + * inline render tools speak Chinese and the file-driven render tools speak + * English. Each variant carries the "do NOT prepend a host" instruction + * because some models hallucinate a placeholder domain in front of the + * relative URL when echoing it back. + */ +public final class GeneratedFileLink { + + private GeneratedFileLink() {} + + /** + * Chinese-language tool result for inline render entry points + * ({@code renderDocx} / {@code renderXlsx} / {@code renderPptx}). + * + * @param typeLabel "文档" / "工作簿" / "演示文稿" + */ + public static String resultZh(byte[] bytes, String displayName, String mimeType, + GeneratedFileCache cache, String typeLabel) { + String url = stash(bytes, displayName, mimeType, cache); + return typeLabel + "已生成:[" + displayName + "](" + url + ")(链接 10 分钟内有效)。\n" + + "重要:回答用户时**必须**使用上述相对路径 `" + url + "`," + + "**不要**添加任何 https://、http:// 域名前缀,前端会自动拼接当前主机。"; + } + + /** + * English-language tool result for file-driven render entry points + * ({@code renderDocxFromFile} / {@code renderDocxFromFiles} / etc.). + * + * @param typeLabel "Document" / "Workbook" / "Presentation" + * @param sourceFileCount number of source markdown files combined into the + * artifact; values {@code > 1} produce a "from N files" + * prefix, {@code 1} produces the plain "generated" prefix + */ + public static String resultEn(byte[] bytes, String displayName, String mimeType, + GeneratedFileCache cache, String typeLabel, + int sourceFileCount) { + String url = stash(bytes, displayName, mimeType, cache); + String prefix = sourceFileCount > 1 + ? typeLabel + " generated from " + sourceFileCount + " files" + : typeLabel + " generated"; + return prefix + ": [" + displayName + "](" + url + ") (link valid for 10 minutes).\n" + + "IMPORTANT: when replying to the user you **must** use the relative path `" + + url + "` verbatim. Do **not** prepend any https://, http:// or domain — " + + "the frontend will resolve the current host automatically."; + } + + private static String stash(byte[] bytes, String displayName, String mimeType, + GeneratedFileCache cache) { + String id = cache.put(bytes, displayName, mimeType); + return "/api/v1/files/generated/" + id; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownInputResolver.java b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownInputResolver.java new file mode 100644 index 00000000..a7a1caad --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownInputResolver.java @@ -0,0 +1,116 @@ +package vip.mate.tool.document; + +import vip.mate.tool.guard.WorkspacePathGuard; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +/** + * Read one or more markdown files from the workspace, returning a single + * resolved record that the document-render tools can hand straight to a + * markdown-to-bytes renderer. + * + *

All path validation goes through {@link WorkspacePathGuard} so the LLM + * cannot escape the workspace boundary by passing {@code ../}-prefixed paths. + * Errors are signalled via {@link ResolveException} carrying a short message + * the tool layer surfaces verbatim to the model. + */ +public final class MarkdownInputResolver { + + private MarkdownInputResolver() {} + + public record Resolved(String markdown, List sources, long totalBytes) { + public int fileCount() { + return sources.size(); + } + } + + public static class ResolveException extends Exception { + public ResolveException(String message) { super(message); } + } + + /** Read a single markdown file. */ + public static Resolved readSingle(String filePath) throws ResolveException { + if (filePath == null || filePath.isBlank()) { + throw new ResolveException("filePath parameter is empty."); + } + Path resolved = validate(filePath, -1); + long size; + String content; + try { + size = Files.size(resolved); + content = Files.readString(resolved, StandardCharsets.UTF_8); + } catch (Exception e) { + throw new ResolveException("failed to read markdown — " + e.getMessage()); + } + if (content.isBlank()) { + throw new ResolveException("markdown file is empty " + resolved); + } + return new Resolved(content, List.of(resolved), size); + } + + /** + * Read multiple markdown files in order and join them with one blank line + * between each. Used by the multi-chapter docx renderer so a long report + * can live in {@code cover.md} / {@code ch1.md} / {@code ch2.md} and still + * compile to a single document. + */ + public static Resolved readManyJoined(List filePaths) throws ResolveException { + if (filePaths == null || filePaths.isEmpty()) { + throw new ResolveException("filePaths is empty."); + } + StringBuilder combined = new StringBuilder(); + long totalBytes = 0; + List resolvedPaths = new ArrayList<>(filePaths.size()); + for (int idx = 0; idx < filePaths.size(); idx++) { + String raw = filePaths.get(idx); + if (raw == null || raw.isBlank()) { + throw new ResolveException("filePaths[" + idx + "] is empty."); + } + Path resolved = validate(raw, idx); + String content; + try { + totalBytes += Files.size(resolved); + content = Files.readString(resolved, StandardCharsets.UTF_8); + } catch (Exception e) { + throw new ResolveException( + "filePaths[" + idx + "] read failed — " + e.getMessage()); + } + if (content.isBlank()) { + throw new ResolveException("filePaths[" + idx + "] is blank " + resolved); + } + if (combined.length() > 0) combined.append("\n\n"); + combined.append(content); + resolvedPaths.add(resolved); + } + return new Resolved(combined.toString(), List.copyOf(resolvedPaths), totalBytes); + } + + /** + * Resolve and validate a single path. {@code idx >= 0} formats errors as + * {@code filePaths[idx]: ...} for the multi-file caller; {@code idx < 0} + * uses the bare message form for the single-file caller. + */ + private static Path validate(String raw, int idx) throws ResolveException { + Path resolved; + try { + resolved = WorkspacePathGuard.validatePath(raw); + } catch (Exception e) { + throw new ResolveException(prefix(idx) + "path validation failed — " + e.getMessage()); + } + if (!Files.exists(resolved)) { + throw new ResolveException(prefix(idx) + "file not found at " + resolved); + } + if (!Files.isRegularFile(resolved) || !Files.isReadable(resolved)) { + throw new ResolveException(prefix(idx) + "path is not a readable regular file " + resolved); + } + return resolved; + } + + private static String prefix(int idx) { + return idx < 0 ? "" : "filePaths[" + idx + "] "; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownPptxRenderer.java b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownPptxRenderer.java new file mode 100644 index 00000000..4efc3a53 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownPptxRenderer.java @@ -0,0 +1,206 @@ +package vip.mate.tool.document; + +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.xslf.usermodel.XMLSlideShow; +import org.apache.poi.xslf.usermodel.XSLFSlide; +import org.apache.poi.xslf.usermodel.XSLFTextBox; +import org.apache.poi.xslf.usermodel.XSLFTextParagraph; +import org.apache.poi.xslf.usermodel.XSLFTextRun; +import org.springframework.stereotype.Component; + +import java.awt.Dimension; +import java.awt.Rectangle; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Render a Markdown string into a PowerPoint .pptx byte array using Apache POI. + * + *

Convention (Marp-compatible subset): + *

    + *
  • {@code ---} on its own line separates slides.
  • + *
  • The first {@code # / ## / ###} of a slide becomes the slide title.
  • + *
  • Lines starting with {@code - } or {@code * } become bullets.
  • + *
  • Other non-blank lines become plain paragraphs.
  • + *
  • {@code } HTML comments become speaker notes.
  • + *
+ * + *

Page size: 16:9 widescreen by default (960pt x 540pt). Pass + * {@code "4:3"} or {@code "STANDARD"} to {@link #render(String, String)} for + * legacy 4:3 (720pt x 540pt). + */ +@Slf4j +@Component +public class MarkdownPptxRenderer { + + /** {@code ---} alone on a line separates slides (Marp / commonmark thematic break). */ + private static final Pattern SLIDE_BREAK = Pattern.compile("^-{3,}\\s*$"); + + /** {@code # / ## / ###} title at the start of a slide. */ + private static final Pattern HEADING = Pattern.compile("^(#{1,3})\\s+(.+)$"); + + /** Bullet item: {@code - foo} or {@code * foo}. */ + private static final Pattern BULLET = Pattern.compile("^\\s*[-*]\\s+(.*)$"); + + /** Speaker note marker: {@code }. */ + private static final Pattern SPEAKER_NOTE = Pattern.compile("^\\s*$"); + + private static final double TITLE_FONT_SIZE = 32.0; + private static final double BULLET_FONT_SIZE = 20.0; + private static final double PARAGRAPH_FONT_SIZE = 18.0; + + public byte[] render(String markdown, String aspectRatio) throws IOException { + if (markdown == null) markdown = ""; + + try (XMLSlideShow ppt = new XMLSlideShow(); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + + ppt.setPageSize(resolvePageSize(aspectRatio)); + + List slides = parseSlides(markdown); + if (slides.isEmpty()) { + // Always produce at least one slide so the file is openable. + slides.add(new SlideSpec(null, List.of(), null)); + } + + int width = (int) ppt.getPageSize().getWidth(); + int height = (int) ppt.getPageSize().getHeight(); + for (SlideSpec spec : slides) { + writeSlide(ppt, spec, width, height); + } + + ppt.write(baos); + return baos.toByteArray(); + } + } + + private record SlideSpec(String title, List body, String speakerNote) {} + + private record BodyLine(boolean bullet, String text) {} + + private List parseSlides(String markdown) { + List result = new ArrayList<>(); + String[] lines = markdown.split("\\R", -1); + + String currentTitle = null; + List currentBody = new ArrayList<>(); + StringBuilder currentNote = new StringBuilder(); + + for (String rawLine : lines) { + String line = rawLine.strip(); + if (SLIDE_BREAK.matcher(line).matches()) { + if (currentTitle != null || !currentBody.isEmpty() || currentNote.length() > 0) { + result.add(new SlideSpec( + currentTitle, currentBody, + currentNote.length() == 0 ? null : currentNote.toString().strip())); + } + currentTitle = null; + currentBody = new ArrayList<>(); + currentNote = new StringBuilder(); + continue; + } + + var noteMatch = SPEAKER_NOTE.matcher(line); + if (noteMatch.matches()) { + if (currentNote.length() > 0) currentNote.append('\n'); + currentNote.append(noteMatch.group(1)); + continue; + } + + if (line.isEmpty()) { + if (!currentBody.isEmpty()) { + currentBody.add(new BodyLine(false, "")); + } + continue; + } + + var headingMatch = HEADING.matcher(line); + if (headingMatch.matches() && currentTitle == null && currentBody.isEmpty()) { + currentTitle = headingMatch.group(2).strip(); + continue; + } + + var bulletMatch = BULLET.matcher(line); + if (bulletMatch.matches()) { + currentBody.add(new BodyLine(true, bulletMatch.group(1).strip())); + continue; + } + + currentBody.add(new BodyLine(false, line)); + } + + if (currentTitle != null || !currentBody.isEmpty() || currentNote.length() > 0) { + result.add(new SlideSpec( + currentTitle, currentBody, + currentNote.length() == 0 ? null : currentNote.toString().strip())); + } + return result; + } + + private void writeSlide(XMLSlideShow ppt, SlideSpec spec, int slideW, int slideH) { + XSLFSlide slide = ppt.createSlide(); + + int margin = 48; + int titleY = 36; + int titleH = spec.title() != null ? 80 : 0; + int bodyY = titleY + (titleH > 0 ? titleH + 12 : 0); + int bodyH = slideH - bodyY - margin; + + if (spec.title() != null) { + XSLFTextBox titleBox = slide.createTextBox(); + titleBox.setAnchor(new Rectangle(margin, titleY, slideW - margin * 2, titleH)); + // POI creates text boxes with one empty paragraph + run; reuse it for the title. + XSLFTextParagraph titleP = titleBox.getTextParagraphs().get(0); + XSLFTextRun titleR = titleP.getTextRuns().isEmpty() + ? titleP.addNewTextRun() + : titleP.getTextRuns().get(0); + titleR.setText(spec.title()); + titleR.setFontSize(TITLE_FONT_SIZE); + titleR.setBold(true); + } + + if (!spec.body().isEmpty()) { + XSLFTextBox bodyBox = slide.createTextBox(); + bodyBox.setAnchor(new Rectangle(margin + 12, bodyY, slideW - margin * 2 - 12, bodyH)); + // Drop the default empty paragraph so our first body line lines up at the top. + bodyBox.clearText(); + + for (BodyLine bl : spec.body()) { + XSLFTextParagraph p = bodyBox.addNewTextParagraph(); + if (bl.bullet()) { + p.setBullet(true); + p.setIndentLevel(0); + } + XSLFTextRun r = p.addNewTextRun(); + r.setText(bl.text()); + r.setFontSize(bl.bullet() ? BULLET_FONT_SIZE : PARAGRAPH_FONT_SIZE); + } + } + + if (spec.speakerNote() != null && !spec.speakerNote().isBlank()) { + try { + slide.getNotes().getPlaceholder(0).setText(spec.speakerNote()); + } catch (Exception e) { + log.debug("Failed to attach speaker note: {}", e.getMessage()); + } + } + } + + /** + * Resolve a user-supplied aspect-ratio string to a POI {@link Dimension} + * in points. The default (and value for any unrecognized input) is 16:9. + */ + private Dimension resolvePageSize(String aspectRatio) { + if (aspectRatio == null) return new Dimension(960, 540); + String normalized = aspectRatio.trim().toUpperCase(Locale.ROOT); + return switch (normalized) { + case "4:3", "STANDARD" -> new Dimension(720, 540); + case "16:9", "WIDE", "WIDESCREEN", "" -> new Dimension(960, 540); + default -> new Dimension(960, 540); + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownXlsxRenderer.java b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownXlsxRenderer.java new file mode 100644 index 00000000..d103ce44 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/MarkdownXlsxRenderer.java @@ -0,0 +1,234 @@ +package vip.mate.tool.document; + +import lombok.extern.slf4j.Slf4j; +import org.apache.poi.ss.usermodel.BorderStyle; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellStyle; +import org.apache.poi.ss.usermodel.FillPatternType; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.HorizontalAlignment; +import org.apache.poi.ss.usermodel.IndexedColors; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.springframework.stereotype.Component; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.regex.Pattern; + +/** + * Render a Markdown string into an Excel .xlsx byte array using Apache POI. + * + *

Convention: each ATX H1 ({@code # Sheet Name}) starts a new sheet. The + * pipe-style table that follows becomes the sheet body. The first table row + * is treated as the header (bold, light-grey fill, frozen). Numeric-looking + * cells are stored as numbers; everything else is stored as a string. + * + *

Markdown without an explicit {@code # heading} produces a single sheet + * named {@code Sheet1}. Markdown without any {@code | table |} rows produces + * an empty workbook with one blank sheet (rendering still succeeds). + */ +@Slf4j +@Component +public class MarkdownXlsxRenderer { + + /** Detects the markdown table separator row, e.g. {@code | --- | :---: |}. */ + private static final Pattern TABLE_SEPARATOR = + Pattern.compile("^\\s*\\|?\\s*:?-{3,}:?\\s*(\\|\\s*:?-{3,}:?\\s*)+\\|?\\s*$"); + + /** Detects a sheet boundary {@code # Sheet Name}. ## / ### are NOT boundaries. */ + private static final Pattern SHEET_BOUNDARY = Pattern.compile("^#\\s+(.+)$"); + + /** Cells that look like numbers (optional sign, digits, optional decimal). */ + private static final Pattern NUMERIC = Pattern.compile("^-?\\d+(\\.\\d+)?$"); + + public byte[] render(String markdown) throws IOException { + if (markdown == null) markdown = ""; + + try (XSSFWorkbook wb = new XSSFWorkbook(); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + + CellStyle headerStyle = buildHeaderStyle(wb); + + List sheets = parseSheets(markdown); + if (sheets.isEmpty()) { + // Always produce a non-empty workbook so the file is openable. + wb.createSheet("Sheet1"); + } else { + // Track names lowercased — Excel sheet uniqueness is + // case-insensitive ("Sales" and "sales" collide). + Set usedLower = new HashSet<>(sheets.size()); + int seq = 1; + for (SheetSpec spec : sheets) { + String safe = sanitizeSheetName(spec.name(), seq++); + String unique = uniqueSheetName(safe, usedLower); + Sheet sheet = wb.createSheet(unique); + writeSheetBody(sheet, spec.rows(), headerStyle); + } + } + + wb.write(baos); + return baos.toByteArray(); + } + } + + private record SheetSpec(String name, List> rows) {} + + private List parseSheets(String markdown) { + List sheets = new ArrayList<>(); + String currentName = null; + List> currentRows = new ArrayList<>(); + + for (String rawLine : markdown.split("\\R", -1)) { + String line = rawLine.strip(); + if (line.isEmpty()) continue; + + var sheetMatch = SHEET_BOUNDARY.matcher(line); + if (sheetMatch.matches()) { + if (currentName != null || !currentRows.isEmpty()) { + sheets.add(new SheetSpec(currentName, currentRows)); + } + currentName = sheetMatch.group(1).strip(); + currentRows = new ArrayList<>(); + continue; + } + + if (TABLE_SEPARATOR.matcher(line).matches()) { + continue; + } + + // Strict markdown-table detection: a row must be wrapped in pipes, + // otherwise prose lines like "A | B 是数据库主键" or file paths like + // "src/main/java/Foo|Bar" would be silently swallowed into the sheet. + // GFM technically allows pipe-less leading/trailing pipes for tables, + // but the rendered LLM output overwhelmingly uses the wrapped form, + // and being strict avoids false positives that pollute the workbook. + if (line.startsWith("|") && line.endsWith("|") && line.length() >= 2) { + List cells = splitTableRow(line); + if (!cells.isEmpty()) { + currentRows.add(cells); + } + } + // Other content (paragraphs, sub-headings) is intentionally ignored — + // xlsx is tabular and there is nowhere sensible to render free prose. + } + + if (currentName != null || !currentRows.isEmpty()) { + sheets.add(new SheetSpec(currentName, currentRows)); + } + return sheets; + } + + private List splitTableRow(String line) { + String trimmed = line.strip(); + if (trimmed.startsWith("|")) trimmed = trimmed.substring(1); + if (trimmed.endsWith("|")) trimmed = trimmed.substring(0, trimmed.length() - 1); + String[] parts = trimmed.split("\\|", -1); + List cells = new ArrayList<>(parts.length); + for (String p : parts) cells.add(p.strip()); + return cells; + } + + private void writeSheetBody(Sheet sheet, List> rows, CellStyle headerStyle) { + if (rows.isEmpty()) return; + + int maxCols = 0; + for (int r = 0; r < rows.size(); r++) { + List rowData = rows.get(r); + Row row = sheet.createRow(r); + for (int c = 0; c < rowData.size(); c++) { + Cell cell = row.createCell(c); + String value = rowData.get(c); + if (NUMERIC.matcher(value).matches()) { + cell.setCellValue(Double.parseDouble(value)); + } else { + cell.setCellValue(value); + } + if (r == 0) cell.setCellStyle(headerStyle); + } + if (rowData.size() > maxCols) maxCols = rowData.size(); + } + + // Freeze the header row and auto-size columns. autoSizeColumn is O(n*m) + // but agent-generated workbooks are small, so the cost is negligible. + sheet.createFreezePane(0, 1); + for (int c = 0; c < maxCols; c++) { + try { + sheet.autoSizeColumn(c); + } catch (Exception e) { + log.debug("autoSizeColumn({}) failed (likely missing fonts on a headless host): {}", + c, e.getMessage()); + } + } + } + + private CellStyle buildHeaderStyle(XSSFWorkbook wb) { + CellStyle style = wb.createCellStyle(); + Font font = wb.createFont(); + font.setBold(true); + style.setFont(font); + style.setFillForegroundColor(IndexedColors.GREY_25_PERCENT.getIndex()); + style.setFillPattern(FillPatternType.SOLID_FOREGROUND); + style.setAlignment(HorizontalAlignment.LEFT); + style.setBorderBottom(BorderStyle.THIN); + return style; + } + + /** + * Resolve duplicate sheet names by appending {@code (2)}, {@code (3)}… + * within the 31-char Excel limit. POI throws on collision, which would + * otherwise abort the entire render when an LLM emits two sheets with the + * same heading or two long headings whose first 31 chars happen to match. + * + *

Excel sheet uniqueness is case-INsensitive, so {@code "Sales"} and + * {@code "sales"} collide. We track names lowercased while still passing + * the original casing into {@link Sheet#createSheet(String)} — so the + * displayed tab keeps the user's casing. + */ + private String uniqueSheetName(String candidate, Set usedLower) { + if (usedLower.add(candidate.toLowerCase(Locale.ROOT))) return candidate; + for (int i = 2; i < 1000; i++) { + String suffix = " (" + i + ")"; + int maxBase = 31 - suffix.length(); + String base = candidate.length() > maxBase + ? candidate.substring(0, maxBase) + : candidate; + String trial = base + suffix; + if (usedLower.add(trial.toLowerCase(Locale.ROOT))) return trial; + } + // Pathological: 1000 collisions. Fall back to a guaranteed-unique tag + // built from nanoTime so the render still succeeds. + String fallback = ("Sheet_" + System.nanoTime()); + if (fallback.length() > 31) fallback = fallback.substring(0, 31); + usedLower.add(fallback.toLowerCase(Locale.ROOT)); + return fallback; + } + + /** + * Excel sheet names are limited to 31 chars and cannot contain {@code : / \ ? * [ ]}, + * cannot be blank, and must be unique. Uniqueness is enforced separately by + * {@link #uniqueSheetName(String, Set)} so this method stays single-shot. + */ + private String sanitizeSheetName(String raw, int seq) { + if (raw == null || raw.isBlank()) return "Sheet" + seq; + StringBuilder sb = new StringBuilder(raw.length()); + for (char ch : raw.toCharArray()) { + if (ch == ':' || ch == '/' || ch == '\\' || ch == '?' + || ch == '*' || ch == '[' || ch == ']') { + sb.append('_'); + } else { + sb.append(ch); + } + } + String cleaned = sb.toString().strip(); + if (cleaned.isEmpty()) cleaned = "Sheet" + seq; + if (cleaned.length() > 31) cleaned = cleaned.substring(0, 31); + return cleaned; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/CjkFontResolver.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/CjkFontResolver.java new file mode 100644 index 00000000..335dbfc7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/CjkFontResolver.java @@ -0,0 +1,112 @@ +package vip.mate.tool.document.pdf; + +import lombok.extern.slf4j.Slf4j; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * Locate a font file capable of rendering CJK text for {@link OpenHtmlToPdfBackend}. + * + *

OpenHTMLtoPDF renders any glyph the registered font does not cover as a + * blank {@code .notdef} box, so a CJK-capable font is mandatory whenever the + * markdown contains Chinese, Japanese, or Korean text. We try in this order: + *

    + *
  1. An explicit {@code mateclaw.pdf.font-path} configuration value.
  2. + *
  3. A short list of OS-default paths that ship with macOS / Windows / common + * Linux distributions. The first existing file wins.
  4. + *
  5. {@link Optional#empty()} — the renderer falls back to PDFBox's built-in + * Latin-only fonts, which renders Chinese as boxes; logged as a warning.
  6. + *
+ */ +@Slf4j +public final class CjkFontResolver { + + // .ttf candidates are listed FIRST because OpenPDF 2.0.5 (used by the + // FlyingSaucer PDF backend) cannot reliably read Apple-style .ttc font + // collections — it loads them without throwing, but the resulting + // BaseFont has an empty cmap and reports `charExists` as false even for + // ASCII. The PDF then renders as a blank page. .ttf collections do not + // share that limitation, so we try them first and only fall through to + // .ttc when nothing else is available. The runtime charExists check in + // FlyingSaucerPdfBackend will reject any candidate that loads but + // cannot actually render glyphs. + + private static final String USER_HOME = System.getProperty("user.home", ""); + + private static final List CANDIDATES_MACOS = List.of( + // Popular open-source CJK .ttf fonts that users commonly install + USER_HOME + "/Library/Fonts/HarmonyOS_SansSC_Regular.ttf", + "/Library/Fonts/HarmonyOS_SansSC_Regular.ttf", + USER_HOME + "/Library/Fonts/SourceHanSansSC-Regular.otf", + "/Library/Fonts/SourceHanSansSC-Regular.otf", + USER_HOME + "/Library/Fonts/NotoSansSC-Regular.ttf", + "/Library/Fonts/NotoSansSC-Regular.ttf", + USER_HOME + "/Library/Fonts/Arial Unicode.ttf", + "/Library/Fonts/Arial Unicode.ttf", + // .ttc fallbacks — known to be lossy under OpenPDF on macOS, + // but listed so the resolver can still warn about them. + "/System/Library/Fonts/PingFang.ttc", + "/System/Library/Fonts/STHeiti Light.ttc", + "/System/Library/Fonts/STHeiti Medium.ttc", + "/Library/Fonts/Songti.ttc"); + + private static final List CANDIDATES_WINDOWS = List.of( + // Plain .ttf first, .ttc / .otf later + "C:/Windows/Fonts/msyh.ttf", + "C:/Windows/Fonts/simhei.ttf", // 黑体 + "C:/Windows/Fonts/simsun.ttf", + "C:/Windows/Fonts/HarmonyOS_SansSC_Regular.ttf", + "C:/Windows/Fonts/NotoSansSC-Regular.ttf", + // Collections last + "C:/Windows/Fonts/msyh.ttc", // 微软雅黑 + "C:/Windows/Fonts/simsun.ttc"); // 宋体 + + private static final List CANDIDATES_LINUX = List.of( + // Plain .ttf / .otf first + "/usr/share/fonts/opentype/noto/NotoSansCJKsc-Regular.otf", + "/usr/share/fonts/truetype/noto/NotoSansSC-Regular.ttf", + "/usr/share/fonts/truetype/harmonyos-sans/HarmonyOS_SansSC_Regular.ttf", + "/usr/share/fonts/truetype/source-han-sans/SourceHanSansSC-Regular.otf", + // Collections last + "/usr/share/fonts/opentype/noto/NotoSansCJK-Regular.ttc", + "/usr/share/fonts/truetype/wqy/wqy-microhei.ttc", + "/usr/share/fonts/truetype/wqy/wqy-zenhei.ttc", + "/usr/share/fonts/truetype/arphic/uming.ttc", + "/usr/share/fonts/truetype/arphic/ukai.ttc"); + + private CjkFontResolver() {} + + public static Optional resolve(String configuredPath) { + if (configuredPath != null && !configuredPath.isBlank()) { + Path explicit = Paths.get(configuredPath.trim()); + if (Files.isRegularFile(explicit)) { + log.debug("[CjkFont] using configured font: {}", explicit); + return Optional.of(explicit); + } + log.warn("[CjkFont] configured font path does not exist: {}", explicit); + } + + for (String candidate : candidatesForCurrentOs()) { + Path p = Paths.get(candidate); + if (Files.isRegularFile(p)) { + log.debug("[CjkFont] auto-detected system font: {}", p); + return Optional.of(p); + } + } + log.warn("[CjkFont] no CJK font found on this host; PDF Chinese characters " + + "will render as blank boxes. Set mateclaw.pdf.font-path to override."); + return Optional.empty(); + } + + private static List candidatesForCurrentOs() { + String osName = System.getProperty("os.name", "").toLowerCase(Locale.ROOT); + if (osName.contains("mac")) return CANDIDATES_MACOS; + if (osName.contains("win")) return CANDIDATES_WINDOWS; + return CANDIDATES_LINUX; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/FlyingSaucerPdfBackend.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/FlyingSaucerPdfBackend.java new file mode 100644 index 00000000..7f9cbae0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/FlyingSaucerPdfBackend.java @@ -0,0 +1,377 @@ +package vip.mate.tool.document.pdf; + +import com.lowagie.text.pdf.BaseFont; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.commonmark.ext.autolink.AutolinkExtension; +import org.commonmark.ext.front.matter.YamlFrontMatterExtension; +import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension; +import org.commonmark.ext.gfm.tables.TablesExtension; +import org.commonmark.node.Node; +import org.commonmark.parser.Parser; +import org.commonmark.renderer.html.HtmlRenderer; +import org.springframework.stereotype.Component; +import org.xhtmlrenderer.pdf.ITextFontResolver; +import org.xhtmlrenderer.pdf.ITextRenderer; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +/** + * In-process PDF rendering: markdown → flexmark XHTML → Flying Saucer (XHTMLRenderer) + * → OpenPDF. + * + *

This backend is always available and is the only one that supports cover + * pages, page headers, and page footers (driven by YAML frontmatter; see + * {@link PdfFrontmatter}). It uses CSS3 paged-media features that Flying Saucer + * implements: {@code @page}, {@code counter(page)}, {@code counter(pages)}, + * {@code @top-center}, {@code @bottom-center}, and {@code page-break-before}. + * + *

Flying Saucer requires strict XHTML, so flexmark's HTML output is wrapped + * in an XHTML envelope. Self-closing void elements ({@code
}, {@code


}, + * {@code }) are normalised by flexmark when generating the body, so we do + * not need a post-processor. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class FlyingSaucerPdfBackend implements PdfBackend { + + private final PdfProperties properties; + + @Override + public String name() { return "flying-saucer"; } + + @Override + public byte[] render(PdfRenderRequest request) throws Exception { + String bodyHtml = renderMarkdownToHtml(request.markdown()); + + try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + ITextRenderer renderer = new ITextRenderer(); + // Register the CJK font BEFORE building the HTML, because the CSS we + // emit references the font's actual family name (read from the font + // file). Aliases via ITextFontResolver's 5-arg overload proved + // unreliable on .ttc collections: the API accepts the override but + // the lookup map silently misses it, leaving the body to fall back + // to Times-Roman and Chinese to render as .notdef boxes. + String cjkFamily = registerCjkFont(renderer.getFontResolver()); + String fullHtml = wrapHtml(bodyHtml, request, cjkFamily); + log.debug("[FlyingSaucerPdf] HTML length={}, body length={}, cjkFamily={}", + fullHtml.length(), bodyHtml.length(), cjkFamily); + try { + renderer.setDocumentFromString(fullHtml); + renderer.layout(); + renderer.createPDF(baos); + } catch (Throwable t) { + log.error("[FlyingSaucerPdf] ITextRenderer failed: {}: {}", + t.getClass().getName(), t.getMessage(), t); + throw t; + } + return baos.toByteArray(); + } + } + + private String renderMarkdownToHtml(String markdown) { + List extensions = List.of( + TablesExtension.create(), + StrikethroughExtension.create(), + AutolinkExtension.create(), + YamlFrontMatterExtension.create()); + Parser parser = Parser.builder().extensions(extensions).build(); + // Flying Saucer requires strict XHTML, so void elements (
,
, + // ) must be self-closed. The xhtml renderer flavour does this. + HtmlRenderer renderer = HtmlRenderer.builder() + .extensions(extensions) + .build(); + Node document = parser.parse(markdown); + return renderer.render(document); + } + + /** + * Register the resolved CJK font with Flying Saucer and return the + * font's actual {@code font-family} name so the inline stylesheet can + * reference it. Returns {@code null} if no font was found or the + * registration failed — callers must tolerate Chinese rendering as + * blank boxes in that case. + * + *

Why we read the real family name instead of using the 5-arg + * {@code addFont(... fontFamilyNameOverride ...)} overload: that override + * succeeds in the call but does not get added to the renderer's + * {@code _fontFamilies} lookup map for {@code .ttc} collections, so the + * CSS declaration {@code font-family: "CJK"} still misses and the body + * falls back to Times-Roman. Reading the font's intrinsic family name + * via OpenPDF's {@link BaseFont#getFamilyFontName()} sidesteps that + * map entirely. + */ + private String registerCjkFont(ITextFontResolver fonts) { + Optional fontPath = CjkFontResolver.resolve(properties.fontPath()); + if (fontPath.isEmpty()) { + log.error("[FlyingSaucerPdf] No CJK font registered. Chinese characters " + + "in this PDF will render as blank boxes. Set mateclaw.pdf.font-path " + + "to the absolute path of a CJK-capable .ttf / .ttc / .otf file."); + return null; + } + // BaseFont.IDENTITY_H + EMBEDDED is what makes CJK actually appear in + // the output PDF — without IDENTITY_H glyph indexing, Chinese characters + // render as blanks even when the font file is found. + // + // OpenPDF 2.0.5 has a known weakness with Apple-style .ttc font + // collections (PingFang.ttc, STHeiti.ttc, Songti.ttc on macOS): the + // load succeeds but the cmap is empty, charExists returns false even + // for ASCII, and the rendered PDF is a blank page. We probe the font + // with charExists below; if it cannot render the characters we need, + // we DO NOT register it and return null so the document keeps + // falling back to the next family in the CSS chain. + String fontKey = fontFileWithSubfontIndex(fontPath.get()); + BaseFont probe; + try { + probe = BaseFont.createFont(fontKey, BaseFont.IDENTITY_H, BaseFont.EMBEDDED); + } catch (Throwable t) { + log.error("[FlyingSaucerPdf] BaseFont.createFont failed for {} — Chinese " + + "will render as blank boxes. {}: {}", + fontKey, t.getClass().getSimpleName(), t.getMessage()); + return null; + } + if (!probe.charExists('你') || !probe.charExists('A')) { + log.error("[FlyingSaucerPdf] Font {} loaded but cmap is empty " + + "(charExists '你'={} 'A'={}). This is the known OpenPDF Apple-.ttc " + + "limitation — install a .ttf CJK font (e.g. HarmonyOS Sans SC, " + + "Noto Sans SC) and either drop it under ~/Library/Fonts/ or set " + + "mateclaw.pdf.font-path to its absolute path.", + fontKey, probe.charExists('你'), probe.charExists('A')); + return null; + } + String realFamily = readFamilyName(probe, fontKey); + try { + fonts.addFont(fontKey, BaseFont.IDENTITY_H, BaseFont.EMBEDDED); + log.info("[FlyingSaucerPdf] registered CJK font: {} (family=\"{}\", cmap OK)", + fontKey, realFamily); + return realFamily; + } catch (Exception e) { + log.error("[FlyingSaucerPdf] failed to register CJK font {} — Chinese " + + "characters in this PDF will render as blank boxes. {}: {}", + fontKey, e.getClass().getSimpleName(), e.getMessage()); + return null; + } + } + + /** + * Pull a usable family name out of the loaded font. Some fonts + * (HarmonyOS Sans SC) leave {@code getFamilyFontName} empty and + * carry the name only in {@code getPostscriptFontName}, so we fall + * back to that. + */ + private static String readFamilyName(BaseFont probe, String fontKey) { + try { + String[][] familyNames = probe.getFamilyFontName(); + if (familyNames != null && familyNames.length > 0) { + String fallback = null; + for (String[] row : familyNames) { + if (row == null || row.length < 4 || row[3] == null || row[3].isBlank()) continue; + if (fallback == null) fallback = row[3]; + if ("3".equals(row[0]) && "1033".equals(row[2])) { + return row[3]; + } + } + if (fallback != null) return fallback; + } + String psName = probe.getPostscriptFontName(); + if (psName != null && !psName.isBlank()) return psName; + } catch (Throwable t) { + log.warn("[FlyingSaucerPdf] could not read family name from {}: {}", + fontKey, t.getMessage()); + } + return "Helvetica"; // benign fallback + } + + private static String fontFileWithSubfontIndex(Path path) { + String name = path.getFileName().toString().toLowerCase(Locale.ROOT); + if (name.endsWith(".ttc") || name.endsWith(".otc")) { + return path.toString() + ",0"; + } + return path.toString(); + } + + /** + * Wrap the rendered markdown body in an XHTML envelope plus a CSS @page + * stylesheet that drives cover / header / footer / page numbers. + * + * @param cjkFamily the actual family name of the registered CJK font as + * reported by OpenPDF, or {@code null} if no font was + * registered. Injected verbatim into the body + * {@code font-family} declaration; when absent we fall + * through directly to Helvetica. + */ + private String wrapHtml(String bodyHtml, PdfRenderRequest request, String cjkFamily) { + PdfFrontmatter fm = request.frontmatter(); + String pageSize = request.pageSize(); + String cjkFamilyDecl = cjkFamily == null + ? "" + : "\"" + cssEscape(cjkFamily) + "\", "; + + // Only render a real cover page when the user explicitly asked for one + // via YAML frontmatter. A synthesised cover (H1 promoted into title) + // would otherwise duplicate the heading: once on the cover and again + // as the first body H1. + String coverHtml = fm.hasExplicitCover() + ? "

" + + "

" + escape(fm.title()) + "

" + + (fm.subtitleOpt().isPresent() + ? "

" + escape(fm.subtitle()) + "

" + : "") + + "
" + : ""; + + // Page margin boxes do NOT inherit `font-family` from body — Flying + // Saucer treats them as detached generated content boxes. If we don't + // give them a CJK-capable font here, header/footer Chinese characters + // silently drop ("Tech Daily · 每日科技精选" → "Tech Daily ·") because + // the default Helvetica has no CJK glyphs. We thread the same family + // we registered for body text through here so the rendering is + // consistent across the document. + String marginBoxFontDecl = "font-family: " + cjkFamilyDecl + + "\"Helvetica\", sans-serif; font-size: 9pt; color: #888;"; + String headerCss = fm.hasHeader() + ? "@top-center { content: \"" + cssEscape(fm.header()) + "\"; " + + marginBoxFontDecl + " }" + : ""; + String footerCss = "@bottom-center { content: " + footerContent(fm) + + "; " + marginBoxFontDecl + " }"; + + // No : Flying Saucer's default EntityResolver tries to fetch + // the W3C XHTML DTD over the network during setDocumentFromString(). + // On any host with no internet (or with W3C throttling) the document + // load silently fails and we emit a 1.3 KB blank PDF. Plain XHTML + // without a DOCTYPE renders just fine. + return """ + + + + document + + + + %s +
%s
+ + + """.formatted(pageSize, headerCss, footerCss, cjkFamilyDecl, coverHtml, bodyHtml); + } + + private String footerContent(PdfFrontmatter fm) { + // Always show page numbers; concatenate user footer ahead if provided. + String pageCounter = "\"" + cssEscape("第 ") + "\" counter(page) " + + "\" / \" counter(pages) \"" + cssEscape(" 页") + "\""; + if (fm.hasFooter()) { + return "\"" + cssEscape(fm.footer()) + " \" " + pageCounter; + } + return pageCounter; + } + + /** Escape user text for placement inside an HTML element. */ + private String escape(String s) { + if (s == null) return ""; + return s.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace("\"", """); + } + + /** Escape user text for placement inside a CSS string literal. */ + private String cssEscape(String s) { + if (s == null) return ""; + return s.replace("\\", "\\\\") + .replace("\"", "\\\"") + .replace("\n", " "); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/LibreOfficePdfBackend.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/LibreOfficePdfBackend.java new file mode 100644 index 00000000..58424340 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/LibreOfficePdfBackend.java @@ -0,0 +1,146 @@ +package vip.mate.tool.document.pdf; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.tool.document.MarkdownDocxRenderer; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * Render PDF by routing markdown through {@link MarkdownDocxRenderer} and + * then handing the docx to a {@code soffice --convert-to pdf} subprocess. + * LibreOffice's typesetter beats anything we can write in-process for plain + * narrative text, especially with mixed CJK + Latin scripts, so this is the + * preferred path when the local install has it. + * + *

Limitations the orchestrator must respect: + *

    + *
  • The intermediate docx has no first-class cover page, page header, + * or page footer the way {@link OpenHtmlToPdfBackend} does. Calls that + * want those features go to the HTML path instead — see + * {@link #supports(PdfRenderRequest)}.
  • + *
  • Page numbers themselves come for free: LibreOffice adds them by + * default during PDF export.
  • + *
+ */ +@Slf4j +@Component +@RequiredArgsConstructor +public class LibreOfficePdfBackend implements PdfBackend { + + private static final long CONVERT_TIMEOUT_SECONDS = 90; + + private final MarkdownDocxRenderer docxRenderer; + private final PdfProperties properties; + + @Override + public String name() { return "libreoffice"; } + + @Override + public boolean isAvailable() { + if (!properties.libreoffice().enabled()) return false; + try { + ProcessBuilder pb = new ProcessBuilder(properties.libreoffice().binary(), "--version"); + pb.redirectErrorStream(true); + Process p = pb.start(); + // Drain stdout so the child can exit even on systems whose pipe buffers + // are tiny; the version string is short, this won't block. + p.getInputStream().readAllBytes(); + boolean finished = p.waitFor(5, TimeUnit.SECONDS); + if (!finished) { + p.destroyForcibly(); + return false; + } + return p.exitValue() == 0; + } catch (Exception e) { + log.debug("[LibreOfficePdf] soffice probe failed: {}", e.getMessage()); + return false; + } + } + + /** + * The docx intermediate cannot carry page headers / footers / an explicit + * cover page, so we decline requests that need those. The orchestrator + * routes such requests to {@link FlyingSaucerPdfBackend} instead. + * + *

Synthetic covers (an H1 that {@code parseOrSynthesise} promoted into a + * cover title) are NOT rejected — those would otherwise force AUTO mode to + * pick the in-process backend for almost every markdown body, since LLM + * output overwhelmingly starts with a {@code # H1}. The H1 will simply + * render as the document's first heading, which is what users expect when + * they didn't ask for a cover explicitly. + */ + @Override + public boolean supports(PdfRenderRequest request) { + PdfFrontmatter fm = request.frontmatter(); + return !fm.hasExplicitCover() && !fm.hasHeader() && !fm.hasFooter(); + } + + @Override + public byte[] render(PdfRenderRequest request) throws Exception { + // Use the same A4/LETTER page-size argument shape MarkdownDocxRenderer expects. + byte[] docxBytes = docxRenderer.render(request.markdown(), request.pageSize()); + + Path tempDir = Files.createTempDirectory("mc_pdf_"); + try { + Path docxFile = tempDir.resolve("input.docx"); + Files.write(docxFile, docxBytes); + + ProcessBuilder pb = new ProcessBuilder( + properties.libreoffice().binary(), + "--headless", + "--convert-to", "pdf", + "--outdir", tempDir.toString(), + docxFile.toString()); + pb.redirectErrorStream(true); + Process p = pb.start(); + byte[] stderr = p.getInputStream().readAllBytes(); + boolean finished = p.waitFor(CONVERT_TIMEOUT_SECONDS, TimeUnit.SECONDS); + if (!finished) { + p.destroyForcibly(); + throw new IOException("soffice conversion timed out after " + CONVERT_TIMEOUT_SECONDS + "s"); + } + if (p.exitValue() != 0) { + throw new IOException("soffice exit " + p.exitValue() + ": " + + new String(stderr).strip()); + } + + Path pdfFile = tempDir.resolve("input.pdf"); + if (!Files.isRegularFile(pdfFile)) { + throw new IOException("soffice produced no PDF (stderr: " + + new String(stderr).strip() + ")"); + } + return Files.readAllBytes(pdfFile); + } finally { + cleanup(tempDir); + } + } + + private void cleanup(Path tempDir) { + try (var stream = Files.walk(tempDir)) { + List entries = stream.sorted(Comparator.reverseOrder()).toList(); + for (Path entry : entries) { + try { + Files.deleteIfExists(entry); + } catch (IOException ignored) { + // Best-effort cleanup; the temp dir lives inside java.io.tmpdir + // and will be reclaimed by the OS on next reboot if we lose the race. + } + } + } catch (IOException ignored) { + // ditto + } + // Suppress IDE warning about unused parameter when File.delete fails silently. + File f = tempDir.toFile(); + if (f.exists() && !f.delete()) { + log.debug("[LibreOfficePdf] could not delete temp dir {}", tempDir); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/MarkdownPdfRenderer.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/MarkdownPdfRenderer.java new file mode 100644 index 00000000..b169bb5b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/MarkdownPdfRenderer.java @@ -0,0 +1,74 @@ +package vip.mate.tool.document.pdf; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * Orchestrate PDF rendering. Picks a {@link PdfBackend} based on the caller's + * engine preference, the backend's {@link PdfBackend#isAvailable()} probe, and + * its {@link PdfBackend#supports(PdfRenderRequest)} declaration. Both backends + * receive a normalised {@link PdfRenderRequest} so they don't have to redo + * frontmatter parsing or page-size defaulting. + * + *

Dispatch table: + *

+ * engine=AUTO  + libreoffice ok + supports request → libreoffice
+ * engine=AUTO  + libreoffice missing OR can't do header/footer → openhtmltopdf
+ * engine=LIBREOFFICE  + supports → libreoffice (else throw)
+ * engine=HTML  → openhtmltopdf
+ * 
+ */ +@Slf4j +@Component +@RequiredArgsConstructor +@EnableConfigurationProperties(PdfProperties.class) +public class MarkdownPdfRenderer { + + private final LibreOfficePdfBackend libreOffice; + private final FlyingSaucerPdfBackend html; + private final PdfProperties properties; + + public record Result(byte[] bytes, String backend) {} + + public Result render(String markdown, String pageSize, PdfProperties.Engine engine) throws Exception { + if (engine == null) engine = properties.defaultEngine(); + + PdfFrontmatter fm = PdfFrontmatter.parseOrSynthesise(markdown); + String body = PdfFrontmatter.stripFrontmatter(markdown); + PdfRenderRequest request = new PdfRenderRequest(body, fm, pageSize, engine); + + PdfBackend chosen = pick(request); + long t0 = System.currentTimeMillis(); + byte[] bytes = chosen.render(request); + log.info("[Pdf] rendered via {} ({} bytes, {}ms, frontmatter cover={} header={} footer={})", + chosen.name(), bytes.length, System.currentTimeMillis() - t0, + fm.hasCover(), fm.hasHeader(), fm.hasFooter()); + return new Result(bytes, chosen.name()); + } + + private PdfBackend pick(PdfRenderRequest request) { + return switch (request.engine()) { + case LIBREOFFICE -> { + if (!libreOffice.isAvailable()) { + throw new IllegalStateException( + "engine=libreoffice but soffice is not available on PATH"); + } + if (!libreOffice.supports(request)) { + throw new IllegalStateException( + "engine=libreoffice but the request needs cover/header/footer; " + + "use engine=html or remove those frontmatter fields"); + } + yield libreOffice; + } + case HTML -> html; + case AUTO -> { + if (libreOffice.isAvailable() && libreOffice.supports(request)) { + yield libreOffice; + } + yield html; + } + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfBackend.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfBackend.java new file mode 100644 index 00000000..d5feb404 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfBackend.java @@ -0,0 +1,29 @@ +package vip.mate.tool.document.pdf; + +/** + * One way to turn markdown bytes into PDF bytes. {@link MarkdownPdfRenderer} + * picks an implementation at request time based on availability and the + * caller's {@link PdfRenderRequest#engine()} preference. + */ +public interface PdfBackend { + + /** Stable identifier surfaced in the tool result and in logs. */ + String name(); + + /** + * Whether this backend can run at all on the current host. The default + * implementation says yes; the LibreOffice backend overrides this to + * probe for {@code soffice}. + */ + default boolean isAvailable() { return true; } + + /** + * Whether this backend can faithfully render the request. The HTML + * backend always returns {@code true}; the LibreOffice backend declines + * requests that need cover / header / footer because those features + * cannot be expressed through the docx intermediate. + */ + default boolean supports(PdfRenderRequest request) { return true; } + + byte[] render(PdfRenderRequest request) throws Exception; +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfFrontmatter.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfFrontmatter.java new file mode 100644 index 00000000..fb752038 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfFrontmatter.java @@ -0,0 +1,165 @@ +package vip.mate.tool.document.pdf; + +import org.commonmark.ext.front.matter.YamlFrontMatterExtension; +import org.commonmark.ext.front.matter.YamlFrontMatterVisitor; +import org.commonmark.node.Node; +import org.commonmark.parser.Parser; + +import java.util.List; +import java.util.Map; +import java.util.Optional; + +/** + * Extract the YAML frontmatter block at the top of a markdown body so the PDF + * pipeline can drive cover / page header / page footer text from it. The + * frontmatter block — when present — has the form: + *
+ * ---
+ * title: 季度报告
+ * subtitle: Q1 2026
+ * header: 内部资料
+ * footer: Mate Inc. © 2026
+ * ---
+ * 
+ * + *

Markdown without frontmatter parses to {@link #empty()}; the renderer + * then synthesises a cover from the first {@code # H1} heading and uses + * default header / footer text. + */ +public record PdfFrontmatter( + String title, + String subtitle, + String header, + String footer, + boolean explicitCover) { + + /** + * Backwards-compat constructor for callers that only know about the four + * text slots; the cover-source flag defaults to {@code false} (synthetic). + */ + public PdfFrontmatter(String title, String subtitle, String header, String footer) { + this(title, subtitle, header, footer, false); + } + + public boolean hasCover() { + return notBlank(title) || notBlank(subtitle); + } + + /** + * Whether the cover came from a YAML frontmatter block (true) or was + * synthesised by promoting a leading {@code # H1} into a cover title (false). + * Synthetic covers are not real layout requirements — the LibreOffice + * backend can ignore them and render the H1 inline as part of the document. + */ + public boolean hasExplicitCover() { + return explicitCover && hasCover(); + } + + public boolean hasHeader() { + return notBlank(header); + } + + public boolean hasFooter() { + return notBlank(footer); + } + + /** Whether ANY of the frontmatter slots is populated. */ + public boolean isPresent() { + return hasCover() || hasHeader() || hasFooter(); + } + + public static PdfFrontmatter empty() { + return new PdfFrontmatter(null, null, null, null, false); + } + + public static PdfFrontmatter parse(String markdown) { + if (markdown == null || markdown.isBlank()) return empty(); + + Parser parser = Parser.builder() + .extensions(List.of(YamlFrontMatterExtension.create())) + .build(); + Node document = parser.parse(markdown); + + YamlFrontMatterVisitor visitor = new YamlFrontMatterVisitor(); + document.accept(visitor); + Map> data = visitor.getData(); + if (data == null || data.isEmpty()) return empty(); + + return new PdfFrontmatter( + first(data, "title"), + first(data, "subtitle"), + first(data, "header"), + first(data, "footer"), + /* explicitCover = */ true); + } + + private static String first(Map> data, String key) { + List values = data.get(key); + if (values == null || values.isEmpty()) return null; + String v = values.get(0); + if (v == null) return null; + // YAML scalar values come back with surrounding quotes preserved when the + // user wrote `title: "..."`. Strip a single matching pair so the rendered + // cover doesn't show literal quote characters. + v = v.trim(); + if ((v.startsWith("\"") && v.endsWith("\"") && v.length() >= 2) + || (v.startsWith("'") && v.endsWith("'") && v.length() >= 2)) { + v = v.substring(1, v.length() - 1); + } + return v; + } + + private static boolean notBlank(String s) { + return s != null && !s.isBlank(); + } + + /** + * Convenience: read frontmatter, if missing look for a leading {@code # H1} + * to use as the cover title. The synthesised result is flagged with + * {@code explicitCover=false} so backends that cannot render an actual + * cover page (LibreOffice via the docx intermediate) can safely ignore it + * — the H1 will still render as the first heading inline. + */ + public static PdfFrontmatter parseOrSynthesise(String markdown) { + PdfFrontmatter fm = parse(markdown); + if (fm.hasCover()) return fm; + + String firstHeading = firstHeading(markdown); + if (firstHeading != null) { + return new PdfFrontmatter(firstHeading, fm.subtitle(), fm.header(), fm.footer(), + /* explicitCover = */ false); + } + return fm; + } + + private static String firstHeading(String markdown) { + for (String rawLine : markdown.split("\\R", -1)) { + String line = rawLine.strip(); + if (line.startsWith("# ") && line.length() > 2) { + return line.substring(2).strip(); + } + } + return null; + } + + /** Strip a leading YAML frontmatter block from a markdown body. */ + public static String stripFrontmatter(String markdown) { + if (markdown == null) return ""; + String trimmed = markdown.stripLeading(); + if (!trimmed.startsWith("---")) return markdown; + int firstBreak = trimmed.indexOf('\n'); + if (firstBreak < 0) return markdown; + int closing = trimmed.indexOf("\n---", firstBreak); + if (closing < 0) return markdown; + int after = trimmed.indexOf('\n', closing + 4); + return after < 0 ? "" : trimmed.substring(after + 1); + } + + /** Try to find {@link Optional} variant for callers preferring null-safe accessors. */ + public Optional titleOpt() { return Optional.ofNullable(title).filter(PdfFrontmatter::nb); } + public Optional subtitleOpt() { return Optional.ofNullable(subtitle).filter(PdfFrontmatter::nb); } + public Optional headerOpt() { return Optional.ofNullable(header).filter(PdfFrontmatter::nb); } + public Optional footerOpt() { return Optional.ofNullable(footer).filter(PdfFrontmatter::nb); } + + private static boolean nb(String s) { return !s.isBlank(); } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfProperties.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfProperties.java new file mode 100644 index 00000000..0962c266 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfProperties.java @@ -0,0 +1,44 @@ +package vip.mate.tool.document.pdf; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Configuration for the markdown-to-PDF rendering pipeline. + * + *

Example {@code application.yml}: + *

+ * mateclaw:
+ *   pdf:
+ *     fontPath: /Library/Fonts/Songti.ttc
+ *     defaultEngine: AUTO
+ *     libreoffice:
+ *       enabled: true
+ *       binary: soffice
+ * 
+ */ +@ConfigurationProperties(prefix = "mateclaw.pdf") +public record PdfProperties( + String fontPath, + Engine defaultEngine, + Libreoffice libreoffice) { + + public PdfProperties { + if (defaultEngine == null) defaultEngine = Engine.AUTO; + if (libreoffice == null) libreoffice = new Libreoffice(true, "soffice"); + } + + public enum Engine { + /** Try LibreOffice first, fall back to OpenHTMLtoPDF. */ + AUTO, + /** Force the LibreOffice subprocess path. Fails if soffice is missing. */ + LIBREOFFICE, + /** Force the in-process OpenHTMLtoPDF path. */ + HTML + } + + public record Libreoffice(boolean enabled, String binary) { + public Libreoffice { + if (binary == null || binary.isBlank()) binary = "soffice"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfRenderRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfRenderRequest.java new file mode 100644 index 00000000..8f8920bf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/document/pdf/PdfRenderRequest.java @@ -0,0 +1,28 @@ +package vip.mate.tool.document.pdf; + +/** + * Request payload handed to a {@link PdfBackend}. The orchestrator builds + * this once per call after parsing frontmatter and resolving page size; the + * backends are read-only consumers. + * + * @param markdown markdown body with the YAML frontmatter block already stripped + * @param frontmatter parsed (or synthesised from a leading {@code # H1}) frontmatter + * @param pageSize "A4" or "LETTER" + * @param engine the engine preference the caller gave; the orchestrator + * uses this to decide which backend to ask, but each backend + * only sees the request after the choice has been made and + * may largely ignore the field + */ +public record PdfRenderRequest( + String markdown, + PdfFrontmatter frontmatter, + String pageSize, + PdfProperties.Engine engine) { + + public PdfRenderRequest { + if (markdown == null) markdown = ""; + if (frontmatter == null) frontmatter = PdfFrontmatter.empty(); + if (pageSize == null || pageSize.isBlank()) pageSize = "A4"; + if (engine == null) engine = PdfProperties.Engine.AUTO; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java index 77d18328..7e279fa9 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java @@ -5,6 +5,7 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.dao.DuplicateKeyException; import org.springframework.web.bind.annotation.*; import vip.mate.approval.ApprovalWorkflowService; import vip.mate.common.result.R; @@ -97,8 +98,12 @@ public class SecurityController { public R createRule(@RequestBody ToolGuardRuleEntity rule) { try { return R.ok(ruleService.createRule(rule)); - } catch (Exception e) { + } catch (IllegalArgumentException e) { return R.fail(e.getMessage()); + } catch (DuplicateKeyException e) { + // Race fallback: pre-check passed but a concurrent insert took the slot. + String ruleId = rule != null && rule.getRuleId() != null ? rule.getRuleId().trim() : ""; + return R.fail("Rule ID already exists: " + ruleId); } } @@ -138,6 +143,17 @@ public class SecurityController { } } + @Operation(summary = "按主键 ID 删除自定义规则(兜底,rule_id 异常时使用)") + @DeleteMapping("/guard/rules/by-id/{id}") + public R deleteRuleByPk(@PathVariable Long id) { + try { + ruleService.deleteRuleByPk(id); + return R.ok("删除成功"); + } catch (IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + // ==================== Audit ==================== @Operation(summary = "审计日志") diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java index 340124b4..66ab98b4 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java @@ -64,14 +64,30 @@ public class ToolGuardRuleService { * 新增自定义规则 */ public ToolGuardRuleEntity createRule(ToolGuardRuleEntity rule) { + if (rule == null) { + throw new IllegalArgumentException("Rule body is required"); + } + requireNonBlank(rule.getRuleId(), "Rule ID"); + requireNonBlank(rule.getName(), "Rule name"); + requireNonBlank(rule.getPattern(), "Rule pattern"); + rule.setRuleId(rule.getRuleId().trim()); + rule.setName(rule.getName().trim()); + rule.setPattern(rule.getPattern().trim()); rule.setBuiltin(false); + // Pre-check uniqueness so the API returns a friendly message instead of + // surfacing the raw JDBC UNIQUE-constraint violation through the global + // exception handler. The DB constraint still guards against races. + if (getByRuleId(rule.getRuleId()) != null) { + throw new IllegalArgumentException("Rule ID already exists: " + rule.getRuleId()); + } ruleMapper.insert(rule); ruleRegistry.reload(); return rule; } /** - * 更新规则 + * 更新规则。仅覆盖请求里显式提供的字段;显式传入的关键字段(name / pattern) + * 不允许置为空白,避免回写出无意义的"空名空模式"行。 */ public ToolGuardRuleEntity updateRule(String ruleId, ToolGuardRuleEntity update) { ToolGuardRuleEntity existing = getByRuleId(ruleId); @@ -79,14 +95,20 @@ public class ToolGuardRuleService { throw new IllegalArgumentException("Rule not found: " + ruleId); } - if (update.getName() != null) existing.setName(update.getName()); + if (update.getName() != null) { + requireNonBlank(update.getName(), "Rule name"); + existing.setName(update.getName().trim()); + } if (update.getDescription() != null) existing.setDescription(update.getDescription()); if (update.getToolName() != null) existing.setToolName(update.getToolName()); if (update.getParamName() != null) existing.setParamName(update.getParamName()); if (update.getCategory() != null) existing.setCategory(update.getCategory()); if (update.getSeverity() != null) existing.setSeverity(update.getSeverity()); if (update.getDecision() != null) existing.setDecision(update.getDecision()); - if (update.getPattern() != null) existing.setPattern(update.getPattern()); + if (update.getPattern() != null) { + requireNonBlank(update.getPattern(), "Rule pattern"); + existing.setPattern(update.getPattern().trim()); + } if (update.getExcludePattern() != null) existing.setExcludePattern(update.getExcludePattern()); if (update.getRemediation() != null) existing.setRemediation(update.getRemediation()); if (update.getEnabled() != null) existing.setEnabled(update.getEnabled()); @@ -97,6 +119,12 @@ public class ToolGuardRuleService { return existing; } + private static void requireNonBlank(String value, String fieldLabel) { + if (value == null || value.isBlank()) { + throw new IllegalArgumentException(fieldLabel + " is required"); + } + } + /** * 启用/禁用规则 */ @@ -124,4 +152,24 @@ public class ToolGuardRuleService { ruleMapper.deleteById(existing.getId()); ruleRegistry.reload(); } + + /** + * 按主键 ID 删除自定义规则。兜底通道:当 rule_id 因历史脏数据为空或无法走 + * /guard/rules/{ruleId} 路径变量时,UI 仍可通过主键删除。 + */ + public void deleteRuleByPk(Long id) { + if (id == null) { + throw new IllegalArgumentException("Rule primary key is required"); + } + ToolGuardRuleEntity existing = ruleMapper.selectById(id); + if (existing == null) { + throw new IllegalArgumentException("Rule not found: id=" + id); + } + if (Boolean.TRUE.equals(existing.getBuiltin())) { + throw new IllegalArgumentException( + "Cannot delete builtin rule: " + existing.getRuleId()); + } + ruleMapper.deleteById(id); + ruleRegistry.reload(); + } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java index 752ddb6a..07f982e3 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationRequest.java @@ -3,10 +3,11 @@ package vip.mate.tool.image; import lombok.Builder; import lombok.Data; +import java.util.List; import java.util.Map; /** - * 图片生成统一请求 + * Unified image-generation request. * * @author MateClaw Team */ @@ -14,30 +15,36 @@ import java.util.Map; @Builder public class ImageGenerationRequest { - /** 图片内容描述 */ + /** Prompt describing the desired image. */ private String prompt; - /** 生成模式(由 runtime 自动推断) */ + /** Generation mode (inferred by the runtime when null). */ private ImageCapability mode; - /** 指定模型名称(可选,provider 有默认值) */ + /** Model id; provider supplies a default when null/blank. */ private String model; - /** 图片尺寸:1024x1024 / 1024x1792 / 1792x1024 等 */ + /** Pixel size like {@code 1024x1024} / {@code 1024x1792}. */ @Builder.Default private String size = "1024x1024"; - /** 画面比例:1:1 / 16:9 / 9:16 */ + /** Aspect ratio: {@code 1:1} / {@code 16:9} / {@code 9:16}. */ @Builder.Default private String aspectRatio = "1:1"; - /** 生成数量 */ + /** Number of images to return. */ @Builder.Default private Integer count = 1; - /** 参考图片 URL(IMAGE_EDIT 模式) */ - private String referenceImageUrl; + /** + * Reference images for edit / image-to-image flows. Loaded as in-memory + * buffers so providers can either inline base64, upload via multipart, or + * forward as a URL — without each provider re-implementing path/URL/data + * resolution. + */ + @Builder.Default + private List inputImages = List.of(); - /** provider 特有的额外参数 */ + /** Provider-specific extras forwarded as-is. */ private Map extraParams; } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java index 44751781..65e483ef 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageGenerationService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import vip.mate.channel.AsyncTaskMediaDispatcher; import vip.mate.channel.web.ChatStreamTracker; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; @@ -39,6 +40,14 @@ public class ImageGenerationService { private final ImageFileDownloader fileDownloader; private final ObjectMapper objectMapper; private final ChatStreamTracker streamTracker; + /** + * Forward completion to the conversation's bound IM channel adapter so + * WeCom / DingTalk / Feishu / etc. users actually receive the generated + * image as a native attachment. Web SSE handling continues unchanged + * via {@link ChatStreamTracker} — the dispatcher is additive and skips + * Web channels to avoid double-rendering. + */ + private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher; private static final String TASK_TYPE = "image_generation"; @@ -180,7 +189,15 @@ public class ImageGenerationService { MessageContentPart imagePart = MessageContentPart.image(null, servingUrl); imagePart.setFileName(localPath.getFileName().toString()); + imagePart.setStoredName(localPath.getFileName().toString()); imagePart.setContentType("image/png"); + // Set absolute disk path so IM channel adapters can read the + // bytes locally instead of round-tripping through the + // /api/v1/chat/files endpoint (which would require auth). + imagePart.setPath(localPath.toAbsolutePath().toString()); + try { + imagePart.setFileSize(java.nio.file.Files.size(localPath)); + } catch (Exception ignored) { /* size is best-effort */ } contentParts.add(imagePart); } @@ -207,6 +224,11 @@ public class ImageGenerationService { streamTracker.broadcastObject(conversationId, "async_task_completed", data); } + // Forward to the conversation's bound IM channel adapter so + // WeCom / DingTalk / Feishu etc. users receive a native attachment + // (the SSE broadcast above only reaches Web subscribers). + asyncTaskMediaDispatcher.forwardToImIfBound(conversationId, contentParts); + log.info("[ImageGen] Sync generation completed, {} image(s) saved for conversation {}", servingUrls.size(), conversationId); @@ -221,6 +243,16 @@ public class ImageGenerationService { * 异步任务完成时的回写逻辑:下载图片 → 保存消息 → 广播 SSE */ private void handleAsyncCompletion(AsyncTaskEntity task, TaskPollResult result) { + // The conversation may have been deleted while the poller was running. + // Gate every post-completion side effect — file write, message save, + // success/failure broadcast — so we never write to a tombstoned + // conversation regardless of which sub-branch we'd take. + if (asyncTaskService.isConversationCanceled(task.getConversationId())) { + log.info("[ImageGen] Task {} (success={}) aborted: conversation {} was deleted", + task.getTaskId(), result.succeeded(), task.getConversationId()); + return; + } + if (result.succeeded()) { try { String imageUrl = result.imageUrl(); @@ -238,21 +270,40 @@ public class ImageGenerationService { // 保存 assistant 消息 MessageContentPart imagePart = MessageContentPart.image(null, servingUrl); imagePart.setFileName(localPath.getFileName().toString()); + imagePart.setStoredName(localPath.getFileName().toString()); imagePart.setContentType("image/png"); + // Set absolute disk path so IM channel adapters can read the + // bytes locally instead of round-tripping through the + // /api/v1/chat/files endpoint (which would require auth). + imagePart.setPath(localPath.toAbsolutePath().toString()); + try { + imagePart.setFileSize(java.nio.file.Files.size(localPath)); + } catch (Exception ignored) { /* size is best-effort */ } + List parts = List.of(imagePart); conversationService.saveMessage( task.getConversationId(), "assistant", "图片已生成完毕", - List.of(imagePart), "completed"); + parts, "completed"); // SSE 广播(使用 imageUrl 字段) asyncTaskService.broadcastTaskEvent(task, "async_task_completed", true, null, servingUrl, null); + // Forward to the conversation's bound IM channel adapter so + // WeCom / DingTalk / Feishu etc. users receive a native + // attachment (the SSE broadcast above only reaches Web). + asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts); + log.info("[ImageGen] Task {} completed, image saved: {}", task.getTaskId(), servingUrl); } catch (Exception e) { log.error("[ImageGen] Completion handling failed for task {}: {}", task.getTaskId(), e.getMessage(), e); + if (asyncTaskService.isConversationCanceled(task.getConversationId())) { + log.info("[ImageGen] Skipping failure broadcast for deleted conversation {}", + task.getConversationId()); + return; + } asyncTaskService.broadcastTaskEvent(task, "async_task_completed", false, null, null, "图片下载或保存失败: " + e.getMessage()); } @@ -264,7 +315,7 @@ public class ImageGenerationService { } private ImageCapability inferMode(ImageGenerationRequest request) { - if (request.getReferenceImageUrl() != null && !request.getReferenceImageUrl().isBlank()) { + if (request.getInputImages() != null && !request.getInputImages().isEmpty()) { return ImageCapability.IMAGE_EDIT; } return ImageCapability.TEXT_TO_IMAGE; diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java new file mode 100644 index 00000000..e4a92469 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageModelSpec.java @@ -0,0 +1,57 @@ +package vip.mate.tool.image; + +import lombok.Builder; +import lombok.Singular; + +import java.util.Map; +import java.util.Set; + +/** + * Per-model descriptor that drives payload construction without {@code if/else} + * chains inside provider classes. Adding a new model = adding a new spec entry. + * + *

Three things make this configuration-driven: + *

    + *
  • {@code endpoint} chooses which provider URL to hit. A single provider + * (e.g. DashScope) can host both an async legacy endpoint and a unified + * multimodal endpoint — the spec routes per model.
  • + *
  • {@code transport} ({@link Transport#SYNC} / {@link Transport#ASYNC}) + * lets the provider pick between immediate-return and submit+poll without + * hard-coding the choice.
  • + *
  • {@code supports} acts as a payload key whitelist. Build the full payload + * freely, then filter against {@code supports} so models never receive + * fields they reject.
  • + *
+ * + * @author MateClaw Team + */ +@Builder +public record ImageModelSpec( + String id, + String displayName, + String endpoint, + Transport transport, + SizeStyle sizeStyle, + @Singular("sizeMapping") Map sizeMap, + @Singular("defaultParam") Map defaults, + @Singular Set supports, + @Singular Set modes, + int maxInputImages, + int maxCount +) { + + public enum Transport { + /** Provider returns image bytes / URL in the same HTTP response. */ + SYNC, + /** Provider returns a task id; caller polls a status endpoint. */ + ASYNC + } + + public boolean supportsEdit() { + return modes != null && modes.contains(ImageCapability.IMAGE_EDIT); + } + + public boolean supportsGenerate() { + return modes != null && modes.contains(ImageCapability.TEXT_TO_IMAGE); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java index 5d79124b..cdec5780 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageProviderCapabilities.java @@ -7,7 +7,15 @@ import java.util.List; import java.util.Set; /** - * 图片生成 Provider 细粒度能力声明 + * Image generation provider capability declaration. + * + *

The flat top-level fields ({@code supportedSizes}, {@code aspectRatios}, + * {@code maxCount}, {@code modes}) describe the provider's combined surface + * area and remain in use by callers that don't need per-mode granularity. + * Newer code should consult the structured {@link Generate} / {@link Edit} / + * {@link Geometry} / {@link Output} fields, which let the picker show + * "edit supports up to N reference images" or "generate accepts these + * formats" without conflating the two modes. * * @author MateClaw Team */ @@ -15,29 +23,87 @@ import java.util.Set; @Builder public class ImageProviderCapabilities { - /** 支持的生成模式 */ + /** Combined modes the provider supports across all its models. */ @Builder.Default private Set modes = Set.of(ImageCapability.TEXT_TO_IMAGE); - /** 支持的图片尺寸,如 ["1024x1024", "1024x1792"] */ + /** Union of pixel sizes accepted by any model under this provider. */ @Builder.Default private List supportedSizes = List.of("1024x1024"); - /** 支持的画面比例 */ + /** Union of aspect ratio presets accepted by any model under this provider. */ @Builder.Default private List aspectRatios = List.of("1:1", "16:9", "9:16"); - /** 最大生成数量 */ + /** Largest {@code n} (image count) any model under this provider accepts. */ @Builder.Default private int maxCount = 1; - /** 默认模型 */ + /** Default model id. */ private String defaultModel; - /** 可用模型列表 */ + /** All callable model ids. */ @Builder.Default private List models = List.of(); + /** Per-mode generate capabilities. Optional — falls back to flat fields when absent. */ + private Generate generate; + + /** Per-mode edit capabilities. {@code null} or {@code enabled=false} means edits unsupported. */ + private Edit edit; + + /** Geometry surface (sizes / aspect ratios). Optional. */ + private Geometry geometry; + + /** Output knobs (formats, qualities, backgrounds). Optional. */ + private Output output; + + @Data + @Builder + public static class Generate { + @Builder.Default + private int maxCount = 1; + @Builder.Default + private boolean supportsSize = true; + @Builder.Default + private boolean supportsAspectRatio = true; + } + + @Data + @Builder + public static class Edit { + @Builder.Default + private boolean enabled = false; + @Builder.Default + private int maxCount = 1; + @Builder.Default + private int maxInputImages = 1; + @Builder.Default + private boolean supportsSize = true; + @Builder.Default + private boolean supportsAspectRatio = true; + } + + @Data + @Builder + public static class Geometry { + @Builder.Default + private List sizes = List.of(); + @Builder.Default + private List aspectRatios = List.of(); + } + + @Data + @Builder + public static class Output { + @Builder.Default + private List formats = List.of(); + @Builder.Default + private List qualities = List.of(); + @Builder.Default + private List backgrounds = List.of(); + } + /** * Match the requested size against supported sizes by area only. * Orientation-blind — prefer {@link #normalizeSize(String, String)} when an diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java new file mode 100644 index 00000000..87b5adec --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReference.java @@ -0,0 +1,21 @@ +package vip.mate.tool.image; + +/** + * In-memory image reference used for image-edit / image-to-image generation requests. + *

+ * The loader normalizes any of the agent-facing input forms (local paths, http(s) + * URLs, {@code data:} URLs, conversation message refs) into this single shape so + * providers receive bytes, mime type, and file name regardless of origin. + * + * @param data raw image bytes + * @param mimeType e.g. {@code image/png} + * @param fileName logical name (best-effort, may be synthesized) + * @param origin trace string identifying where the bytes came from + * ({@code path:/x.png}, {@code url:https://...}, {@code data-url}, + * {@code msg::}). Used for logging / audit, not + * forwarded to providers. + * + * @author MateClaw Team + */ +public record ImageReference(byte[] data, String mimeType, String fileName, String origin) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java new file mode 100644 index 00000000..613d23ab --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/ImageReferenceLoader.java @@ -0,0 +1,297 @@ +package vip.mate.tool.image; + +import cn.hutool.http.HttpUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.io.IOException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; + +/** + * Resolves agent-supplied image reference strings into in-memory + * {@link ImageReference} buffers. Five input forms are accepted: + * + *

    + *
  1. Local filesystem path: {@code /abs/path.png}, {@code ./rel.png}, + * {@code ~/x.png}, or {@code file://...}.
  2. + *
  3. Data URL: {@code data:image/png;base64,...} (base64 or URL-encoded body).
  4. + *
  5. HTTP(S) URL: downloaded with size + content-type guard.
  6. + *
  7. Conversation message reference: {@code msg::} — + * resolves to the local path stored on a {@link MessageContentPart} of + * type {@code image} on the named message. This is the channel an agent + * uses to forward a user-uploaded image into the image edit tool, so a + * non-vision model can still operate on attachments it cannot "see".
  8. + *
  9. Workspace-relative path: passed through as a regular path; the caller + * is expected to anchor it to the active workspace before invocation.
  10. + *
+ * + * @author MateClaw Team + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ImageReferenceLoader { + + private static final long MAX_REFERENCE_BYTES = 20L * 1024 * 1024; + private static final int HTTP_TIMEOUT_MS = 30_000; + + private final ConversationService conversationService; + + /** + * Resolve a list of input strings; null / blank entries are skipped. + * The caller is expected to enforce per-provider {@code maxInputImages} + * before calling. + */ + public List loadAll(List inputs, String conversationId) throws IOException { + if (inputs == null || inputs.isEmpty()) { + return List.of(); + } + List out = new ArrayList<>(inputs.size()); + for (String raw : inputs) { + if (raw == null || raw.isBlank()) { + continue; + } + out.add(load(raw.trim(), conversationId)); + } + return out; + } + + /** Resolve a single reference string. */ + public ImageReference load(String input, String conversationId) throws IOException { + if (input == null || input.isBlank()) { + throw new IOException("image reference is blank"); + } + String trimmed = input.trim(); + + if (trimmed.startsWith("data:")) { + return loadDataUrl(trimmed); + } + if (trimmed.startsWith("msg:")) { + return loadConversationMessageRef(trimmed, conversationId); + } + if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) { + return loadHttpUrl(trimmed); + } + return loadFilePath(trimmed); + } + + // ==================== form: local path / file:// ==================== + + private ImageReference loadFilePath(String input) throws IOException { + String pathStr = input.startsWith("file://") ? input.substring("file://".length()) : input; + if (pathStr.startsWith("~")) { + pathStr = System.getProperty("user.home") + pathStr.substring(1); + } + Path p = Paths.get(pathStr); + if (!Files.exists(p)) { + throw new IOException("Image file not found: " + pathStr); + } + if (Files.size(p) > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + pathStr); + } + byte[] data = Files.readAllBytes(p); + String mime = inferMimeFromName(p.getFileName().toString()); + return new ImageReference(data, mime, p.getFileName().toString(), "path:" + p); + } + + // ==================== form: data: URL ==================== + + private ImageReference loadDataUrl(String dataUrl) throws IOException { + int comma = dataUrl.indexOf(','); + if (comma < 0) { + throw new IOException("Malformed data URL: missing comma"); + } + String header = dataUrl.substring("data:".length(), comma); + String body = dataUrl.substring(comma + 1); + boolean isBase64 = header.toLowerCase().contains(";base64"); + String mime = isBase64 + ? header.substring(0, header.toLowerCase().indexOf(";base64")) + : (header.contains(";") ? header.substring(0, header.indexOf(';')) : header); + if (mime == null || mime.isBlank()) { + mime = "image/png"; + } + byte[] data; + try { + data = isBase64 + ? Base64.getDecoder().decode(body) + : URLDecoder.decode(body, StandardCharsets.UTF_8).getBytes(StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new IOException("Invalid base64 in data URL: " + e.getMessage(), e); + } + if (data.length > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit (data URL)"); + } + return new ImageReference(data, mime, "inline." + extensionFor(mime), "data-url"); + } + + // ==================== form: http(s) URL ==================== + + private ImageReference loadHttpUrl(String url) throws IOException { + URI uri = URI.create(url); + String host = uri.getHost(); + if (host == null) { + throw new IOException("URL has no host: " + url); + } + // Conservative SSRF guard: reject obvious internal targets. Refine later + // if the project gains a dedicated SsrFPolicy module. + String lowered = host.toLowerCase(); + if (lowered.equals("localhost") + || lowered.equals("127.0.0.1") + || lowered.startsWith("10.") + || lowered.startsWith("192.168.") + || lowered.startsWith("169.254.") + || lowered.startsWith("172.")) { + throw new IOException("Refusing to download image from internal host: " + host); + } + try { + byte[] data = HttpUtil.createGet(url).timeout(HTTP_TIMEOUT_MS).execute().bodyBytes(); + if (data == null || data.length == 0) { + throw new IOException("Empty response downloading image from " + url); + } + if (data.length > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + url); + } + String fileName = guessFileNameFromUrl(url); + String mime = inferMimeFromName(fileName); + return new ImageReference(data, mime, fileName, "url:" + url); + } catch (Exception e) { + throw new IOException("Failed to download image " + url + ": " + e.getMessage(), e); + } + } + + // ==================== form: msg:: ==================== + + private ImageReference loadConversationMessageRef(String ref, String conversationId) throws IOException { + // ref shape: "msg:" (first image part) or "msg::" + String body = ref.substring("msg:".length()); + String[] parts = body.split(":", 2); + long messageId; + try { + messageId = Long.parseLong(parts[0]); + } catch (NumberFormatException e) { + throw new IOException("Invalid msg: ref, expected msg:[:]: " + ref); + } + Integer wantedIdx = null; + if (parts.length == 2 && !parts[1].isBlank()) { + try { + wantedIdx = Integer.parseInt(parts[1]); + } catch (NumberFormatException e) { + throw new IOException("Invalid part index in: " + ref); + } + } + if (conversationId == null || conversationId.isBlank()) { + throw new IOException("Cannot resolve msg: reference without an active conversation"); + } + MessageEntity message = findMessageInConversation(conversationId, messageId); + if (message == null) { + throw new IOException("Message " + messageId + " not found in conversation " + conversationId); + } + List contentParts = conversationService.parseMessageParts(message); + MessageContentPart picked = pickImagePart(contentParts, wantedIdx); + if (picked == null) { + throw new IOException("No image part on message " + messageId + + (wantedIdx != null ? " at index " + wantedIdx : "")); + } + Path filePath = resolveLocalPath(picked); + if (filePath == null) { + throw new IOException("Message " + messageId + " image part has no local path: " + + picked.getFileName()); + } + if (Files.size(filePath) > MAX_REFERENCE_BYTES) { + throw new IOException("Image exceeds 20MB limit: " + filePath); + } + byte[] data = Files.readAllBytes(filePath); + String mime = picked.getContentType(); + if (mime == null || mime.isBlank() || "image/*".equals(mime)) { + mime = inferMimeFromName(picked.getFileName()); + } + String fileName = picked.getFileName() != null ? picked.getFileName() : filePath.getFileName().toString(); + return new ImageReference(data, mime, fileName, ref); + } + + private MessageEntity findMessageInConversation(String conversationId, long messageId) { + List all = conversationService.listMessages(conversationId); + for (MessageEntity m : all) { + if (m.getId() != null && m.getId() == messageId) { + return m; + } + } + return null; + } + + private MessageContentPart pickImagePart(List parts, Integer wantedIdx) { + if (parts == null || parts.isEmpty()) { + return null; + } + if (wantedIdx != null) { + int seen = 0; + for (MessageContentPart p : parts) { + if (p == null || !"image".equals(p.getType())) continue; + if (seen == wantedIdx) { + return p; + } + seen++; + } + return null; + } + for (MessageContentPart p : parts) { + if (p != null && "image".equals(p.getType())) { + return p; + } + } + return null; + } + + private Path resolveLocalPath(MessageContentPart part) { + if (part.getPath() != null && !part.getPath().isBlank()) { + Path p = Paths.get(part.getPath()); + if (Files.exists(p)) return p; + } + if (part.getStoredName() != null && !part.getStoredName().isBlank()) { + Path p = Paths.get(part.getStoredName()); + if (Files.exists(p)) return p; + } + return null; + } + + // ==================== shared helpers ==================== + + private static String inferMimeFromName(String name) { + if (name == null) return "image/png"; + String lower = name.toLowerCase(); + if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"; + if (lower.endsWith(".webp")) return "image/webp"; + if (lower.endsWith(".gif")) return "image/gif"; + if (lower.endsWith(".bmp")) return "image/bmp"; + return "image/png"; + } + + private static String extensionFor(String mime) { + return switch (mime.toLowerCase().trim()) { + case "image/jpeg", "image/jpg" -> "jpg"; + case "image/webp" -> "webp"; + case "image/gif" -> "gif"; + case "image/bmp" -> "bmp"; + default -> "png"; + }; + } + + private static String guessFileNameFromUrl(String url) { + String stripped = url.split("\\?", 2)[0]; + int slash = stripped.lastIndexOf('/'); + String tail = slash >= 0 ? stripped.substring(slash + 1) : stripped; + return tail.isBlank() ? "remote.png" : tail; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java b/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java new file mode 100644 index 00000000..a69ee163 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/PayloadBuilder.java @@ -0,0 +1,172 @@ +package vip.mate.tool.image; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Configuration-driven payload builder for image-generation providers. + * + *

Without this, every provider class collects an {@code if/else} chain + * mapping (model id) → (request shape, sizing dialect, available knobs). Each + * new model in a family forces another branch. With it, each provider holds a + * static {@code Map}; the builder consults that spec + * for sizing dialect, default parameters, and a {@code supports} whitelist of + * payload keys. Values not in the whitelist are dropped at the end so the API + * never sees keys it would reject. + * + *

Sizing dialect handling: + *

    + *
  • {@link SizeStyle#LITERAL_DIMENSION} — output {@code "1024x1024"} or + * a separator-replaced form (e.g. DashScope wants {@code "1024*1024"} — + * the spec's sizeMap can carry the alternative).
  • + *
  • {@link SizeStyle#ASPECT_RATIO} — output {@code "1:1"} / {@code "16:9"}.
  • + *
  • {@link SizeStyle#PRESET_NAME} — output the model-native preset + * (e.g. {@code square_hd}). The spec's sizeMap drives the lookup keyed + * by orientation token (landscape / square / portrait).
  • + *
+ * + * @author MateClaw Team + */ +public final class PayloadBuilder { + + private final ImageModelSpec spec; + private final Map entries = new LinkedHashMap<>(); + + private PayloadBuilder(ImageModelSpec spec) { + this.spec = spec; + if (spec.defaults() != null) { + entries.putAll(spec.defaults()); + } + } + + public static PayloadBuilder from(ImageModelSpec spec) { + return new PayloadBuilder(spec); + } + + public PayloadBuilder withPrompt(String prompt) { + if (prompt != null) { + entries.put("prompt", prompt); + } + return this; + } + + public PayloadBuilder withCount(Integer count) { + if (count != null && count > 0) { + entries.put("n", Math.min(count, Math.max(1, spec.maxCount() == 0 ? count : spec.maxCount()))); + } + return this; + } + + /** + * Translate the unified {@code size} / {@code aspectRatio} inputs to whichever + * key/value pair this model expects. The spec's {@link SizeStyle} drives + * which key is set; the spec's sizeMap (orientation → native value) drives + * the value when the caller did not pass an exact match. + */ + public PayloadBuilder withSize(String requestedSize, String requestedAspectRatio) { + SizeStyle style = spec.sizeStyle(); + if (style == null) { + return this; + } + Map sizeMap = spec.sizeMap(); + switch (style) { + case LITERAL_DIMENSION -> entries.put("size", + resolveLiteralDimension(requestedSize, requestedAspectRatio, sizeMap)); + case ASPECT_RATIO -> entries.put("aspect_ratio", + resolveAspectRatio(requestedAspectRatio, sizeMap)); + case PRESET_NAME -> entries.put("image_size", + resolvePreset(requestedAspectRatio, sizeMap)); + } + return this; + } + + public PayloadBuilder withSeed(Integer seed) { + if (seed != null) { + entries.put("seed", seed); + } + return this; + } + + public PayloadBuilder put(String key, Object value) { + if (value != null) { + entries.put(key, value); + } + return this; + } + + /** + * Produce a Jackson {@link ObjectNode} containing only the keys this model's + * {@code supports} whitelist allows. Empty whitelist means "passthrough". + */ + public ObjectNode toJsonNode(ObjectMapper mapper) { + ObjectNode out = mapper.createObjectNode(); + Set supports = spec.supports(); + boolean filter = supports != null && !supports.isEmpty(); + for (Map.Entry e : entries.entrySet()) { + if (filter && !supports.contains(e.getKey())) { + continue; + } + out.set(e.getKey(), mapper.valueToTree(e.getValue())); + } + return out; + } + + /** Read-only view of accumulated entries (post defaults / pre supports filter). */ + public Map entries() { + return Map.copyOf(entries); + } + + // ==================== size resolution ==================== + + private String resolveLiteralDimension(String requestedSize, String aspectRatio, + Map sizeMap) { + if (requestedSize != null && !requestedSize.isBlank()) { + // Allow the spec's sizeMap to translate (e.g. "1024x1024" -> "1024*1024"). + String mapped = sizeMap == null ? null : sizeMap.get(requestedSize); + return mapped != null ? mapped : requestedSize; + } + String orientation = orientationOf(aspectRatio); + if (sizeMap != null && sizeMap.containsKey(orientation)) { + return sizeMap.get(orientation); + } + return "1024x1024"; + } + + private String resolveAspectRatio(String requested, Map sizeMap) { + if (requested != null && !requested.isBlank()) { + String mapped = sizeMap == null ? null : sizeMap.get(requested); + return mapped != null ? mapped : requested; + } + return "1:1"; + } + + private String resolvePreset(String aspectRatio, Map sizeMap) { + String orientation = orientationOf(aspectRatio); + if (sizeMap != null && sizeMap.containsKey(orientation)) { + return sizeMap.get(orientation); + } + return "square_hd"; + } + + private static String orientationOf(String aspectRatio) { + if (aspectRatio == null || aspectRatio.isBlank()) { + return "square"; + } + String[] parts = aspectRatio.split(":"); + if (parts.length != 2) { + return "square"; + } + try { + double w = Double.parseDouble(parts[0].trim()); + double h = Double.parseDouble(parts[1].trim()); + if (w == h) return "square"; + return w > h ? "landscape" : "portrait"; + } catch (NumberFormatException e) { + return "square"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java b/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java new file mode 100644 index 00000000..3b2247c9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/SizeStyle.java @@ -0,0 +1,27 @@ +package vip.mate.tool.image; + +/** + * Describes how a particular image-generation model expects its size to be + * expressed. Three families cover all current providers: + * + *
    + *
  • {@link #LITERAL_DIMENSION} — explicit width/height string ({@code 1024x1024}, + * {@code 1536*1024}). Used by DashScope, OpenAI DALL-E, MiniMax.
  • + *
  • {@link #ASPECT_RATIO} — preset enum like {@code 16:9} or {@code 1:1}. + * Used by Gemini / nano-banana style APIs.
  • + *
  • {@link #PRESET_NAME} — provider-specific preset label + * ({@code square_hd}, {@code landscape_16_9}). Used by fal.ai's flux, + * z-image, qwen-image families.
  • + *
+ * + * Each {@link ImageModelSpec} declares one style and provides the sizeMap that + * translates the unified {@code aspectRatio} input ({@code landscape} / + * {@code square} / {@code portrait} or a literal ratio) to the model-native form. + * + * @author MateClaw Team + */ +public enum SizeStyle { + LITERAL_DIMENSION, + ASPECT_RATIO, + PRESET_NAME +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java new file mode 100644 index 00000000..d4815d3c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageModels.java @@ -0,0 +1,228 @@ +package vip.mate.tool.image.provider; + +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.ImageModelSpec; +import vip.mate.tool.image.SizeStyle; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; + +/** + * Catalog of DashScope-served image generation / editing models, organised so + * that adding a new model is a one-line spec entry. + * + *

Two transport families are present: + *

    + *
  • Async legacy ({@link #LEGACY_ASYNC_ENDPOINT} — + * {@code text2image/image-synthesis}) — wanx 2.0/2.1 and wan 2.2/2.5 + * turbo/plus models that exclusively do text-to-image. The caller + * submits and polls {@code /api/v1/tasks/{id}}.
  • + *
  • Sync multimodal ({@link #MULTIMODAL_ENDPOINT}) — wan 2.6/2.7, + * qwen-image, qwen-image-edit, z-image. Uses the OpenAI-style + * {@code messages.content[]} array and returns the generated image URL + * in the same response.
  • + *
+ * + * @author MateClaw Team + */ +final class DashScopeImageModels { + + /** + * Async text-to-image endpoint for the wanx 2.0/2.1 + wan 2.2/2.5 turbo/plus + * families. Despite Aliyun's docs occasionally describing a unified + * {@code image-generation/generation} path, the wanx-series turbo/plus + * models actually still go through {@code text2image/image-synthesis} and + * return {@code "url error, please check url"} on the other path. + */ + static final String LEGACY_ASYNC_ENDPOINT = + "https://dashscope.aliyuncs.com/api/v1/services/aigc/text2image/image-synthesis"; + static final String MULTIMODAL_ENDPOINT = + "https://dashscope.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation"; + static final String TASKS_ENDPOINT_PREFIX = + "https://dashscope.aliyuncs.com/api/v1/tasks/"; + + /** + * Default model when the request does not name one. + * + *

Kept on the legacy turbo so existing accounts that have not enrolled in + * the newer wan/qwen-image families do not see breakage. Callers that want + * edit support must name a model explicitly (e.g. {@code wan2.7-image} or + * {@code qwen-image-edit}) — the registry's edit-capability resolution then + * routes correctly. + */ + static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo"; + + /** + * Default model when an edit-capable spec is required but the request did + * not name one. Used by the provider when the request carries + * {@code inputImages} but the named model lacks {@link ImageCapability#IMAGE_EDIT}. + */ + static final String DEFAULT_EDIT_MODEL = "wan2.7-image"; + + private static final Map ASPECT_LITERAL_SIZES = Map.of( + "1:1", "1024x1024", + "16:9", "1280x720", + "9:16", "720x1280", + "landscape", "1280x720", + "square", "1024x1024", + "portrait", "720x1280" + ); + + private static final Map ASPECT_LITERAL_SIZES_2K = Map.of( + "1:1", "2048x2048", + "16:9", "2560x1440", + "9:16", "1440x2560", + "landscape", "2560x1440", + "square", "2048x2048", + "portrait", "1440x2560" + ); + + private DashScopeImageModels() {} + + private static final Map CATALOG = buildCatalog(); + + static Map all() { + return CATALOG; + } + + static ImageModelSpec get(String id) { + if (id == null || id.isBlank()) { + return CATALOG.get(DEFAULT_MODEL); + } + return CATALOG.getOrDefault(id, CATALOG.get(DEFAULT_MODEL)); + } + + private static Map buildCatalog() { + Map m = new LinkedHashMap<>(); + + // ========== Legacy async text-to-image (image-generation/generation) ========== + // No edit support; keeps backward compatibility for users on existing model ids. + addAsyncT2I(m, "wanx2.1-t2i-turbo"); + addAsyncT2I(m, "wanx2.1-t2i-plus"); + addAsyncT2I(m, "wanx2.0-t2i-turbo"); + addAsyncT2I(m, "wan2.2-t2i-flash"); + addAsyncT2I(m, "wan2.2-t2i-plus"); + addAsyncT2I(m, "wan2.5-t2i-preview"); + + // ========== Sync multimodal text-to-image only (multimodal-generation) ========== + m.put("z-image-turbo", ImageModelSpec.builder() + .id("z-image-turbo") + .displayName("Z-Image Turbo (fastest)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "seed", "prompt_extend")) + .maxCount(1) + .maxInputImages(0) + .build()); + + // ========== Sync multimodal text-to-image + edit (qwen-image series) ========== + addQwenImage(m, "qwen-image-2.0"); + addQwenImage(m, "qwen-image-2.0-pro"); + addQwenImageEdit(m, "qwen-image-edit"); + addQwenImageEdit(m, "qwen-image-edit-plus"); + addQwenImageEdit(m, "qwen-image-edit-max"); + + // ========== Sync multimodal text-to-image + edit (wan2.6 / 2.7 image) ========== + m.put("wan2.6-t2i", ImageModelSpec.builder() + .id("wan2.6-t2i") + .displayName("Wan 2.6 (sync T2I)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(0) + .build()); + + m.put("wan2.7-image", ImageModelSpec.builder() + .id("wan2.7-image") + .displayName("Wan 2.7 Image (T2I + edit, up to 2K)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + + m.put("wan2.7-image-pro", ImageModelSpec.builder() + .id("wan2.7-image-pro") + .displayName("Wan 2.7 Image Pro (T2I + edit, up to 4K)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + + return Map.copyOf(m); + } + + // ------------------- helper builders ------------------- + + private static void addAsyncT2I(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (async legacy T2I)") + .endpoint(LEGACY_ASYNC_ENDPOINT) + .transport(ImageModelSpec.Transport.ASYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + // Legacy endpoint uses '*' as the size separator; sizeMap stores + // the native form so PayloadBuilder can pass it through. + .sizeMapping("1:1", "1024*1024") + .sizeMapping("16:9", "1280*720") + .sizeMapping("9:16", "720*1280") + .sizeMapping("landscape", "1280*720") + .sizeMapping("square", "1024*1024") + .sizeMapping("portrait", "720*1280") + .sizeMapping("1024x1024", "1024*1024") + .sizeMapping("1280x720", "1280*720") + .sizeMapping("720x1280", "720*1280") + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n")) + .maxCount(4) + .maxInputImages(0) + .build()); + } + + private static void addQwenImage(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (T2I)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.TEXT_TO_IMAGE)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(0) + .build()); + } + + private static void addQwenImageEdit(Map m, String id) { + m.put(id, ImageModelSpec.builder() + .id(id) + .displayName(id + " (image edit)") + .endpoint(MULTIMODAL_ENDPOINT) + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMap(ASPECT_LITERAL_SIZES_2K) + .modes(Set.of(ImageCapability.IMAGE_EDIT)) + .supports(Set.of("size", "n", "seed", "negative_prompt", "prompt_extend", "watermark")) + .maxCount(4) + .maxInputImages(3) + .build()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java index d812db7c..cfe10c6a 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/provider/DashScopeImageProvider.java @@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,17 +12,38 @@ import org.springframework.stereotype.Component; import vip.mate.llm.service.ModelProviderService; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.task.AsyncTaskService.TaskPollResult; -import vip.mate.tool.image.*; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.ImageGenerationProvider; +import vip.mate.tool.image.ImageGenerationRequest; +import vip.mate.tool.image.ImageModelSpec; +import vip.mate.tool.image.ImageProviderCapabilities; +import vip.mate.tool.image.ImageReference; +import vip.mate.tool.image.ImageSubmitResult; +import vip.mate.tool.image.PayloadBuilder; +import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Set; /** - * DashScope 图片生成 Provider — 支持通义万相 Wanx 系列 - *

- * 异步模式:提交后返回 taskId,需轮询获取结果。 - * 复用已有的 DashScope LLM provider 的 API Key。 - * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/text-to-image + * DashScope image provider — routes per-model between two transports: + * + *

    + *
  • Async legacy ({@code services/aigc/text2image/image-synthesis}) + * for the wanx 2.0/2.1, wan 2.2/2.5 turbo/plus families. Submit returns a + * task id; the caller polls {@code /api/v1/tasks/{id}} until + * SUCCEEDED.
  • + *
  • Sync multimodal ({@code services/aigc/multimodal-generation/generation}) + * for wan 2.6/2.7 image, qwen-image, qwen-image-edit, z-image. The + * generated image URL is returned in the same response. This endpoint + * also accepts inline reference images, enabling the image edit / + * image-to-image flow.
  • + *
+ * + * The model catalog ({@link DashScopeImageModels}) drives endpoint selection, + * payload shape, and the {@code supports} whitelist — adding a new model is a + * one-line spec entry. * * @author MateClaw Team */ @@ -33,9 +55,6 @@ public class DashScopeImageProvider implements ImageGenerationProvider { private final ModelProviderService modelProviderService; private final ObjectMapper objectMapper; - private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; - private static final String DEFAULT_MODEL = "wanx2.1-t2i-turbo"; - @Override public String id() { return "dashscope"; @@ -43,7 +62,7 @@ public class DashScopeImageProvider implements ImageGenerationProvider { @Override public String label() { - return "DashScope (通义万相)"; + return "DashScope (Tongyi Wanxiang / Qwen-Image)"; } @Override @@ -58,18 +77,32 @@ public class DashScopeImageProvider implements ImageGenerationProvider { @Override public Set capabilities() { - return Set.of(ImageCapability.TEXT_TO_IMAGE); + return Set.of(ImageCapability.TEXT_TO_IMAGE, ImageCapability.IMAGE_EDIT); } @Override public ImageProviderCapabilities detailedCapabilities() { + List modelIds = new ArrayList<>(DashScopeImageModels.all().keySet()); return ImageProviderCapabilities.builder() .modes(capabilities()) - .supportedSizes(List.of("1024x1024", "720x1280", "1280x720")) + .supportedSizes(List.of( + "1024x1024", "1280x720", "720x1280", + "2048x2048", "2560x1440", "1440x2560")) .aspectRatios(List.of("1:1", "16:9", "9:16")) .maxCount(4) - .defaultModel(DEFAULT_MODEL) - .models(List.of("wanx2.1-t2i-turbo", "wanx-v1")) + .defaultModel(DashScopeImageModels.DEFAULT_MODEL) + .models(modelIds) + .generate(ImageProviderCapabilities.Generate.builder() + .maxCount(4).supportsSize(true).supportsAspectRatio(true).build()) + .edit(ImageProviderCapabilities.Edit.builder() + .enabled(true).maxCount(4).maxInputImages(3) + .supportsSize(true).supportsAspectRatio(true).build()) + .geometry(ImageProviderCapabilities.Geometry.builder() + .sizes(List.of( + "1024x1024", "1280x720", "720x1280", + "2048x2048", "2560x1440", "1440x2560")) + .aspectRatios(List.of("1:1", "16:9", "9:16")) + .build()) .build(); } @@ -86,51 +119,16 @@ public class DashScopeImageProvider implements ImageGenerationProvider { public ImageSubmitResult submit(ImageGenerationRequest request, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return ImageSubmitResult.failure(id(), "DashScope API Key 未配置"); + return ImageSubmitResult.failure(id(), "DashScope API Key not configured"); } + ImageModelSpec spec = resolveSpec(request); try { - String model = request.getModel() != null && !request.getModel().isBlank() - ? request.getModel() : DEFAULT_MODEL; - - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - - ObjectNode input = body.putObject("input"); - input.put("prompt", request.getPrompt()); - - ObjectNode parameters = body.putObject("parameters"); - // request.size already normalized by ImageGenerationService to one of supportedSizes. - // DashScope API uses '*' separator instead of 'x'. - String size = request.getSize(); - if (size != null && !size.isBlank()) { - parameters.put("size", size.replace("x", "*")); - } - int count = request.getCount() != null ? Math.min(request.getCount(), 4) : 1; - parameters.put("n", count); - - HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/text2image/image-synthesis") - .header("Authorization", "Bearer " + apiKey) - .header("Content-Type", "application/json") - .header("X-DashScope-Async", "enable") - .body(body.toString()) - .timeout(30_000) - .execute(); - - JsonNode result = objectMapper.readTree(response.body()); - - if (response.getStatus() == 200 && result.has("output")) { - String taskId = result.path("output").path("task_id").asText(); - log.info("[DashScope Image] Submitted task: {} (model={})", taskId, model); - return ImageSubmitResult.asyncSuccess(taskId, id()); - } else { - String errMsg = result.has("message") ? result.get("message").asText() - : "HTTP " + response.getStatus(); - log.warn("[DashScope Image] Submit failed: {}", errMsg); - return ImageSubmitResult.failure(id(), errMsg); - } + return spec.transport() == ImageModelSpec.Transport.SYNC + ? submitSyncMultimodal(request, spec, apiKey) + : submitAsyncLegacy(request, spec, apiKey); } catch (Exception e) { - log.error("[DashScope Image] Submit error: {}", e.getMessage(), e); + log.error("[DashScope Image] Submit error (model={}): {}", spec.id(), e.getMessage(), e); return ImageSubmitResult.failure(id(), e.getMessage()); } } @@ -139,11 +137,10 @@ public class DashScopeImageProvider implements ImageGenerationProvider { public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return TaskPollResult.failed("DashScope API Key 未配置"); + return TaskPollResult.failed("DashScope API Key not configured"); } - try { - HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + HttpResponse response = HttpRequest.get(DashScopeImageModels.TASKS_ENDPOINT_PREFIX + providerTaskId) .header("Authorization", "Bearer " + apiKey) .timeout(15_000) .execute(); @@ -154,11 +151,11 @@ public class DashScopeImageProvider implements ImageGenerationProvider { return switch (taskStatus) { case "SUCCEEDED" -> { - String imageUrl = extractImageUrl(output); + String imageUrl = extractLegacyImageUrl(output); yield TaskPollResult.imageSucceeded(imageUrl, output.toString()); } case "FAILED" -> { - String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + String errMsg = output.has("message") ? output.get("message").asText() : "task failed"; yield TaskPollResult.failed(errMsg); } case "RUNNING" -> TaskPollResult.running(null); @@ -170,6 +167,159 @@ public class DashScopeImageProvider implements ImageGenerationProvider { } } + // ==================== spec resolution ==================== + + /** + * Pick the model spec for this request. When the request asks for image + * editing but names a model that doesn't support edits (or names nothing), + * fall back to {@link DashScopeImageModels#DEFAULT_EDIT_MODEL} so the call + * doesn't silently degrade to a text-only generation. + * + *

Package-private for direct testing of the routing decision (the + * surrounding submit() goes over HTTP and is not a unit-test surface). + */ + ImageModelSpec resolveSpec(ImageGenerationRequest request) { + boolean wantsEdit = request.getInputImages() != null && !request.getInputImages().isEmpty(); + String requested = request.getModel(); + ImageModelSpec spec = DashScopeImageModels.get(requested); + if (wantsEdit && !spec.supportsEdit()) { + ImageModelSpec edit = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_EDIT_MODEL); + log.info("[DashScope Image] Model {} lacks edit support; routing to {}", spec.id(), edit.id()); + return edit; + } + return spec; + } + + // ==================== sync multimodal-generation ==================== + + private ImageSubmitResult submitSyncMultimodal(ImageGenerationRequest request, + ImageModelSpec spec, + String apiKey) throws Exception { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + // input.messages[].content[] — image blocks first when editing, + // followed by the text prompt block. + ObjectNode input = body.putObject("input"); + ArrayNode messages = input.putArray("messages"); + ObjectNode userMsg = messages.addObject(); + userMsg.put("role", "user"); + ArrayNode content = userMsg.putArray("content"); + + if (request.getInputImages() != null) { + for (ImageReference ref : request.getInputImages()) { + ObjectNode imgPart = content.addObject(); + imgPart.put("image", toDataUrl(ref)); + } + } + ObjectNode textPart = content.addObject(); + textPart.put("text", request.getPrompt() == null ? "" : request.getPrompt()); + + // parameters block — built and filtered against the model's supports set. + ObjectNode parameters = PayloadBuilder.from(spec) + .withSize(request.getSize(), request.getAspectRatio()) + .withCount(request.getCount()) + .toJsonNode(objectMapper); + body.set("parameters", parameters); + + HttpResponse response = HttpRequest.post(spec.endpoint()) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .body(body.toString()) + .timeout(180_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + if (response.getStatus() != 200) { + String errMsg = result.has("message") ? result.get("message").asText() : "HTTP " + response.getStatus(); + log.warn("[DashScope Image] Sync submit failed (model={}): {}", spec.id(), errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + + List imageUrls = extractMultimodalImageUrls(result); + if (imageUrls.isEmpty()) { + return ImageSubmitResult.failure(id(), "Multimodal response carried no image URL"); + } + log.info("[DashScope Image] Sync generated {} image(s) (model={})", imageUrls.size(), spec.id()); + return ImageSubmitResult.syncSuccess(id(), imageUrls); + } + + private List extractMultimodalImageUrls(JsonNode result) { + List urls = new ArrayList<>(); + JsonNode choices = result.path("output").path("choices"); + if (!choices.isArray()) { + return urls; + } + for (JsonNode choice : choices) { + JsonNode parts = choice.path("message").path("content"); + if (!parts.isArray()) continue; + for (JsonNode part : parts) { + String url = part.path("image").asText(null); + if (url != null && !url.isBlank()) { + urls.add(url); + } + } + } + return urls; + } + + // ==================== async legacy image-generation ==================== + + private ImageSubmitResult submitAsyncLegacy(ImageGenerationRequest request, + ImageModelSpec spec, + String apiKey) throws Exception { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + + ObjectNode parameters = PayloadBuilder.from(spec) + .withSize(request.getSize(), request.getAspectRatio()) + .withCount(request.getCount()) + .toJsonNode(objectMapper); + body.set("parameters", parameters); + + HttpResponse response = HttpRequest.post(spec.endpoint()) + .header("Authorization", "Bearer " + apiKey) + .header("Content-Type", "application/json") + .header("X-DashScope-Async", "enable") + .body(body.toString()) + .timeout(30_000) + .execute(); + + JsonNode result = objectMapper.readTree(response.body()); + if (response.getStatus() == 200 && result.has("output")) { + String taskId = result.path("output").path("task_id").asText(); + log.info("[DashScope Image] Async submitted task {} (model={})", taskId, spec.id()); + return ImageSubmitResult.asyncSuccess(taskId, id()); + } + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Image] Async submit failed (model={}): {}", spec.id(), errMsg); + return ImageSubmitResult.failure(id(), errMsg); + } + + private String extractLegacyImageUrl(JsonNode output) { + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + JsonNode first = results.get(0); + String url = first.path("url").asText(null); + if (url == null || url.isBlank()) { + url = first.path("image").asText(null); + } + return url; + } + return null; + } + + // ==================== shared helpers ==================== + + private String toDataUrl(ImageReference ref) { + String mime = ref.mimeType() == null || ref.mimeType().isBlank() ? "image/png" : ref.mimeType(); + return "data:" + mime + ";base64," + Base64.getEncoder().encodeToString(ref.data()); + } + private String getDashScopeApiKey() { try { var providerEntity = modelProviderService.getProviderConfig("dashscope"); @@ -178,12 +328,4 @@ public class DashScopeImageProvider implements ImageGenerationProvider { return null; } } - - private String extractImageUrl(JsonNode output) { - JsonNode results = output.path("results"); - if (results.isArray() && !results.isEmpty()) { - return results.get(0).path("url").asText(null); - } - return null; - } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java index 3650918d..fff04065 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/image/vision/provider/DashScopeVisionProvider.java @@ -8,10 +8,11 @@ import vip.mate.llm.service.ModelProviderService; * DashScope vision provider — uses {@code qwen-vl-max} via the * OpenAI-compatible endpoint at {@code /compatible-mode/v1/chat/completions}. * - *

Default for the Chinese cloud rollout: API keys are typically - * available (DASHSCOPE_API_KEY is mandatory for the rest of the - * platform) and per-image cost is the lowest of the supported vendors, - * so this provider sits at the front of the auto-detect chain. + *

Sits at the front of the auto-detect chain when a DashScope provider row + * is configured in the admin UI: per-image cost is the lowest of the supported + * vendors, and DashScope is the most common first provider added on the + * Chinese cloud rollout. Falls back to the next provider in the chain when no + * DashScope API key is available. */ @Component public class DashScopeVisionProvider extends OpenAiCompatibleVisionProvider { diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/model/McpServerEntity.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/model/McpServerEntity.java index 1e0bd228..2c5fbaf8 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/mcp/model/McpServerEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/model/McpServerEntity.java @@ -68,6 +68,20 @@ public class McpServerEntity { /** 远端暴露的工具数量 */ private Integer toolCount; + /** + * Last successful {@code listTools()} response, serialized as a JSON + * array of {@code {name, description, inputSchema}} entries. Refreshed + * by {@code McpServerService} after every successful (re)connect; never + * cleared on failure so the picker keeps working while the upstream + * server is briefly unavailable. Reverse-lookup of a prefixed callback + * name to its raw tool name reads from this column. + */ + @TableField(value = "tools_cache_json", updateStrategy = FieldStrategy.ALWAYS) + private String toolsCacheJson; + + /** Wall-clock timestamp of the last successful tools-cache write. */ + private LocalDateTime toolsCacheUpdatedAt; + /** 是否系统内置 */ private Boolean builtin; diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpClientManager.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpClientManager.java index 542b1cd5..ad835a4f 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpClientManager.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpClientManager.java @@ -17,6 +17,8 @@ import org.springframework.stereotype.Component; import vip.mate.tool.mcp.model.McpServerEntity; import jakarta.annotation.PreDestroy; +import java.net.URI; +import java.net.URISyntaxException; import java.time.Duration; import java.time.LocalDateTime; import java.util.*; @@ -149,24 +151,79 @@ public class McpClientManager { } /** - * 获取所有 active clients 的 ToolCallback 列表 + * Collect ToolCallbacks from every active MCP client, with each callback's + * name rewritten to a server-id-anchored prefix + * (see {@link McpToolNameResolver}). Two guarantees: + *

    + *
  • Two MCP servers can expose the same raw tool name without one + * silently overwriting the other in a name-keyed map downstream.
  • + *
  • If two raw names within the same server happen to hash to the + * same prefixed name, only the first survives — + * {@link McpHashCollisionDetector} flags the second so the picker + * can refuse to bind it.
  • + *
*/ public List getAllToolCallbacks() { List allCallbacks = new ArrayList<>(); for (Map.Entry entry : clients.entrySet()) { + long serverId = entry.getKey(); try { SyncMcpToolCallbackProvider provider = new SyncMcpToolCallbackProvider(entry.getValue()); ToolCallback[] cbs = provider.getToolCallbacks(); - if (cbs != null) { - Collections.addAll(allCallbacks, cbs); + if (cbs == null || cbs.length == 0) { + continue; } + allCallbacks.addAll(wrapServerCallbacks(serverId, cbs)); } catch (Exception e) { - log.warn("Failed to get tool callbacks from MCP server {}: {}", entry.getKey(), e.getMessage()); + log.warn("Failed to get tool callbacks from MCP server {}: {}", serverId, e.getMessage()); } } return allCallbacks; } + /** + * Apply per-server collision detection and wrap each surviving callback + * with its prefixed name. Walks {@code cbs} and the matching decision + * list in lockstep so that duplicate raw names are honored + * one-decision-per-callback — a {@code Map} would make + * every duplicate look up the first (bindable) decision and silently + * register two callbacks under the same prefixed name, breaking the + * "runtime and picker share one decision" contract. + * + *

Package-private so unit tests can drive it without standing up a + * real {@link McpSyncClient}. + */ + static List wrapServerCallbacks(long serverId, ToolCallback[] cbs) { + List rawNames = new ArrayList<>(cbs.length); + for (ToolCallback cb : cbs) { + rawNames.add(cb.getToolDefinition() != null ? cb.getToolDefinition().name() : null); + } + List decisions = + McpHashCollisionDetector.classify(serverId, rawNames); + + // classify() drops blank/null raws; advance the decision pointer + // only when the cb's raw is non-blank so the indices stay aligned. + List out = new ArrayList<>(cbs.length); + int dIdx = 0; + for (ToolCallback cb : cbs) { + String raw = cb.getToolDefinition() != null ? cb.getToolDefinition().name() : null; + if (raw == null || raw.isBlank()) { + continue; + } + if (dIdx >= decisions.size()) { + break; + } + McpHashCollisionDetector.Decision d = decisions.get(dIdx++); + if (!d.bindable()) { + log.error("Skipping MCP tool callback on server {} (raw='{}', prefixed='{}'): {}", + serverId, raw, d.prefixedName(), d.unavailableReason()); + continue; + } + out.add(new PrefixedNameToolCallback(d.prefixedName(), cb)); + } + return out; + } + /** * 获取连接结果 */ @@ -316,7 +373,9 @@ public class McpClientManager { Duration connectTimeout = Duration.ofSeconds( server.getConnectTimeoutSeconds() != null ? server.getConnectTimeoutSeconds() : 30); - var builder = HttpClientSseClientTransport.builder(server.getUrl()) + HttpEndpointConfig endpointConfig = splitHttpUrl(server.getUrl(), "/sse"); + var builder = HttpClientSseClientTransport.builder(endpointConfig.baseUrl()) + .sseEndpoint(endpointConfig.endpoint()) .connectTimeout(connectTimeout); // Add headers via request customizer @@ -336,7 +395,9 @@ public class McpClientManager { Duration connectTimeout = Duration.ofSeconds( server.getConnectTimeoutSeconds() != null ? server.getConnectTimeoutSeconds() : 30); - var builder = HttpClientStreamableHttpTransport.builder(server.getUrl()) + HttpEndpointConfig endpointConfig = splitHttpUrl(server.getUrl(), "/mcp"); + var builder = HttpClientStreamableHttpTransport.builder(endpointConfig.baseUrl()) + .endpoint(endpointConfig.endpoint()) .connectTimeout(connectTimeout); // Add headers via request customizer @@ -352,6 +413,45 @@ public class McpClientManager { return builder.build(); } + /** + * Splits a full HTTP MCP URL into a {@code scheme://authority} base and a + * {@code path[?query]} endpoint suffix. The underlying SDK builders take + * the two halves separately and resolve them via {@link URI#resolve(URI)}, + * which replaces the base URL's path with the endpoint when the endpoint + * starts with {@code /}. Passing a full URL as the base would therefore + * silently route every request to the SDK's default endpoint + * (e.g. {@code /mcp}) and drop any user-configured path or query string. + * + * @param url the user-configured full URL + * @param defaultEndpoint endpoint to use when the URL has no path + */ + static HttpEndpointConfig splitHttpUrl(String url, String defaultEndpoint) { + String trimmed = url != null ? url.trim() : ""; + if (trimmed.isEmpty()) { + throw new IllegalArgumentException("MCP server URL must not be empty"); + } + URI uri; + try { + uri = new URI(trimmed); + } catch (URISyntaxException e) { + throw new IllegalArgumentException("Invalid MCP server URL: " + url, e); + } + if (uri.getScheme() == null || uri.getRawAuthority() == null) { + throw new IllegalArgumentException("MCP server URL must include scheme and host: " + url); + } + String path = uri.getRawPath(); + String endpoint = (path == null || path.isEmpty() || "/".equals(path)) ? defaultEndpoint : path; + String query = uri.getRawQuery(); + if (query != null && !query.isEmpty()) { + endpoint += "?" + query; + } + String baseUrl = uri.getScheme() + "://" + uri.getRawAuthority(); + return new HttpEndpointConfig(baseUrl, endpoint); + } + + record HttpEndpointConfig(String baseUrl, String endpoint) { + } + private Map parseHeaders(McpServerEntity server) { if (server.getHeadersJson() != null && !server.getHeadersJson().isBlank()) { Map headers = JSONUtil.toBean(server.getHeadersJson(), diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetector.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetector.java new file mode 100644 index 00000000..00756ea9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetector.java @@ -0,0 +1,90 @@ +package vip.mate.tool.mcp.runtime; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Per-server hash collision detector for MCP tool names. + * + *

{@link McpToolNameResolver}'s 30-bit hash makes name collisions + * statistically rare but not impossible. The detector runs the same + * input set through the resolver and reports which raw names collide on + * the same prefixed name, so two callers can agree on which entries are + * "bindable" and which are not: + * + *

    + *
  • {@code McpClientManager} consults the detector before registering + * runtime callbacks, skipping the second of any colliding pair so + * {@link org.springframework.ai.tool.ToolCallback} names stay unique + * in the runtime tool set.
  • + *
  • {@code AvailableToolService} consults the detector when emitting + * picker DTOs, marking colliding entries {@code available=false} + * with reason {@code HASH_COLLISION} so the UI disables them.
  • + *
+ * + *

Sharing the detector keeps these two views in lockstep — without it, + * the picker could offer a tool whose runtime callback was silently + * skipped, letting the user save a binding that resolves to nothing at + * chat time. + * + *

Stateless and thread-safe. + */ +public final class McpHashCollisionDetector { + + private McpHashCollisionDetector() {} + + /** + * Decide which raw tool names are bindable for a given server. + * + *

The first occurrence of each prefixed name wins; later raw names + * that hash to the same prefix are recorded as collided. Iteration + * order of {@code rawToolNames} therefore determines which raw name + * is treated as canonical — callers should pass a stable order + * (typically the order returned by {@code listTools()}). + * + * @return one entry per non-blank input raw name, in input order + */ + public static List classify(long serverId, Collection rawToolNames) { + if (rawToolNames == null || rawToolNames.isEmpty()) { + return List.of(); + } + Map firstRawByPrefixed = new LinkedHashMap<>(); + List out = new ArrayList<>(rawToolNames.size()); + for (String raw : rawToolNames) { + if (raw == null || raw.isBlank()) { + // Defensive: an MCP server shouldn't surface a blank tool name, + // but if it does, drop it instead of letting resolver throw. + continue; + } + String prefixed = McpToolNameResolver.prefixedName(serverId, raw); + String prior = firstRawByPrefixed.putIfAbsent(prefixed, raw); + if (prior == null) { + out.add(new Decision(raw, prefixed, true, null)); + } else if (prior.equals(raw)) { + // Same raw name appearing twice in the input — duplicate + // declaration upstream, not a collision. Keep the first. + out.add(new Decision(raw, prefixed, false, "DUPLICATE_RAW_NAME")); + } else { + out.add(new Decision(raw, prefixed, false, "HASH_COLLISION:" + prior)); + } + } + return out; + } + + /** + * One decision per raw tool name. + * + * @param rawToolName name as discovered from the MCP server + * @param prefixedName resolved {@code mcp___} + * @param bindable {@code true} → runtime should register this + * callback and the picker should offer it as + * {@code available=true}; {@code false} → both + * must skip / disable it + * @param unavailableReason machine-readable cause when {@code !bindable} + */ + public record Decision(String rawToolName, String prefixedName, + boolean bindable, String unavailableReason) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpReturnDirectProperties.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpReturnDirectProperties.java index 6f83f4b9..ab456613 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpReturnDirectProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpReturnDirectProperties.java @@ -8,7 +8,7 @@ import java.util.LinkedHashSet; import java.util.Set; /** - * RFC-052 §3.4 / PR-4: MCP tool return-direct opt-in list. + * MCP tool return-direct opt-in list. * *

Tools listed here are wrapped in {@link ReturnDirectMcpToolCallback} so * their results bypass the LLM context (see {@code ToolExecutionExecutor} and @@ -20,14 +20,27 @@ import java.util.Set; * mcp: * return-direct: * tools: - * - query_employee_salary - * - read_medical_record + * - query_employee_salary # raw upstream name (legacy form, still supported) + * - mcp_42_query_employee_salary_aB3xYz # full prefixed callback name (server-scoped, precise) * * - *

Match is by tool name only (matching the upstream {@code ToolDefinition.name()}). - * Per-server scoping is intentionally out of scope for the first iteration; if - * the same tool name comes from two servers and only one should be direct, give - * one of them a name prefix at the MCP server config layer. + *

Two accepted name forms: + *

    + *
  • Raw upstream name ({@code query_employee_salary}) — + * matches the wrapped callback's underlying delegate name. This is + * the form that existed before the runtime started prefixing + * callback names; existing deployments keep working unchanged. + * A raw name matches every server that exposes that tool, so use + * this form when a sensitive name should be direct on every + * server it appears.
  • + *
  • Prefixed callback name + * ({@code mcp___}) — server-scoped, precise. + * Use this form when only one of several MCP servers exposing the + * same raw name should be treated as direct.
  • + *
+ * Matching happens via {@link #matches(String, String)} from the consumer + * side; see {@link McpToolCallbackProvider#getToolCallbacks} for the call + * site. * * @author MateClaw Team */ @@ -35,7 +48,7 @@ import java.util.Set; @ConfigurationProperties(prefix = "mateclaw.mcp.return-direct") public class McpReturnDirectProperties { - /** Tool names that should be treated as returnDirect. */ + /** Tool names (raw or prefixed) that should be treated as returnDirect. */ private Set tools = Collections.emptySet(); public Set getTools() { @@ -46,7 +59,27 @@ public class McpReturnDirectProperties { this.tools = tools != null ? new LinkedHashSet<>(tools) : Collections.emptySet(); } + /** + * Single-string check kept for back-compat with any caller that has + * only one form of the name. Prefer {@link #matches(String, String)} + * from the wrapping path so both prefixed and raw forms get a chance + * to match. + */ public boolean isReturnDirect(String toolName) { return toolName != null && tools.contains(toolName); } + + /** + * @return {@code true} iff the configured set contains either the + * prefixed callback name OR the raw upstream tool name. Either + * argument may be {@code null} (e.g. when a callback isn't a + * {@link PrefixedNameToolCallback} so no raw form is + * available); the other is checked on its own. + */ + public boolean matches(String prefixedName, String rawName) { + if (tools.isEmpty()) return false; + if (prefixedName != null && tools.contains(prefixedName)) return true; + if (rawName != null && tools.contains(rawName)) return true; + return false; + } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolCallbackProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolCallbackProvider.java index 4a8496ea..0710259a 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolCallbackProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolCallbackProvider.java @@ -40,14 +40,30 @@ public class McpToolCallbackProvider implements ToolCallbackProvider { callbacks.size(), mcpClientManager.getActiveCount()); } - // RFC-052: opt-in returnDirect wrapping. The decorator only changes + // Opt-in returnDirect wrapping. The decorator only changes // ToolMetadata.returnDirect(); guard/approval/observability still // see the original callback through the wrapper. + // + // Names registered by the manager are now prefixed + // (mcp___) — but operators have been + // configuring the return-direct list with raw upstream names + // (e.g. `query_employee_salary`) since long before the prefix + // existed. Match on EITHER form so an existing deployment's + // sensitive-tool isolation doesn't silently regress when this + // change rolls out: a tool counts as return-direct if its + // configured token equals (a) the prefixed callback name OR + // (b) the underlying raw tool name visible through the + // PrefixedNameToolCallback wrapper. List wrapped = new ArrayList<>(callbacks.size()); for (ToolCallback cb : callbacks) { - String name = cb.getToolDefinition() != null ? cb.getToolDefinition().name() : null; - if (returnDirectProperties.isReturnDirect(name)) { - log.info("[McpToolCallbackProvider] wrapping MCP tool '{}' as returnDirect (RFC-052)", name); + String prefixed = cb.getToolDefinition() != null ? cb.getToolDefinition().name() : null; + String raw = (cb instanceof PrefixedNameToolCallback w && w.getDelegate() != null + && w.getDelegate().getToolDefinition() != null) + ? w.getDelegate().getToolDefinition().name() + : null; + if (returnDirectProperties.matches(prefixed, raw)) { + log.info("[McpToolCallbackProvider] wrapping MCP tool as returnDirect (prefixed='{}', raw='{}')", + prefixed, raw); wrapped.add(new ReturnDirectMcpToolCallback(cb)); } else { wrapped.add(cb); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolNameResolver.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolNameResolver.java new file mode 100644 index 00000000..5464d1be --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/McpToolNameResolver.java @@ -0,0 +1,140 @@ +package vip.mate.tool.mcp.runtime; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Locale; +import java.util.regex.Pattern; + +/** + * Single source of truth for MCP tool callback names. + * + *

Format: {@code mcp___} where: + *

    + *
  • {@code } — immutable {@code mate_mcp_server.id} (numeric + * Snowflake). Anchoring to the DB primary key (not the user-visible + * display name) makes display-name renames transparent to bindings.
  • + *
  • {@code } — first 20 chars of {@code [^a-z0-9_-]→'_'} on the + * lowercased raw tool name. Kept for human readability in logs / SQL.
  • + *
  • {@code } — first 6 chars of base32-no-pad + * {@code SHA-256(raw_tool_name)}. Greatly reduces the chance of + * distinct raw names colliding under the same slug; residual collisions + * (probabilistic, not zero) are handled explicitly by + * {@link McpHashCollisionDetector} at registration and picker emission + * time — never relied on as a uniqueness guarantee.
  • + *
+ * + *

Length budget: {@code mcp_} (4) + serverId (≤19) + sep + slug (≤20) + + * sep + hash6 (6) = ≤51 chars, comfortably under any 64-char tool-name caps + * downstream tool engines may enforce. + * + *

The format is not a 1:1 string-only inverse of the raw name. + * The slug stage is lossy (multiple raw names can map to the same slug; + * non-ASCII names map to {@code "tool"}). The hash makes the full key + * statistically unique within {@code (serverId, raw_tool_name)} space, but + * recovering the raw name from the prefixed name alone is not possible. + * Reversal must go through the per-server cached tools list: given + * {@code (serverId, hash6)}, find the cached tool whose + * {@code SHA-256(raw)} hashes to the same prefix. + */ +public final class McpToolNameResolver { + + public static final String PREFIX = "mcp_"; + public static final int SLUG_MAX = 20; + public static final int HASH_LEN = 6; + + private static final Pattern UNSAFE = Pattern.compile("[^a-z0-9_-]"); + // RFC 4648 base32 lowercase, no padding. Lowercase keeps the prefixed + // name fully lowercase + digits + dashes — friendly to URL paths, + // filenames, log greps, and case-insensitive systems. + private static final char[] BASE32 = "abcdefghijklmnopqrstuvwxyz234567".toCharArray(); + + private McpToolNameResolver() {} + + /** Build the prefixed callback name for a given (serverId, raw tool name) pair. */ + public static String prefixedName(long serverId, String rawToolName) { + if (rawToolName == null || rawToolName.isBlank()) { + throw new IllegalArgumentException("rawToolName must not be blank"); + } + return PREFIX + serverId + "_" + slug(rawToolName) + "_" + hash6(rawToolName); + } + + /** + * Parse a prefixed name into its components. Returns {@code null} if the + * input does not match the MCP prefix shape — callers use this to route + * lookups between bridged MCP names and other namespaces. + * + *

Note that {@link ParsedRef} intentionally does not include the raw + * tool name: that requires a cache lookup (see class Javadoc). + */ + public static ParsedRef parse(String prefixedName) { + if (prefixedName == null || !prefixedName.startsWith(PREFIX)) { + return null; + } + int firstSep = prefixedName.indexOf('_', PREFIX.length()); + int lastSep = prefixedName.lastIndexOf('_'); + if (firstSep < 0 || lastSep <= firstSep) { + return null; + } + String serverIdStr = prefixedName.substring(PREFIX.length(), firstSep); + String slug = prefixedName.substring(firstSep + 1, lastSep); + String hash = prefixedName.substring(lastSep + 1); + if (hash.length() != HASH_LEN || slug.isEmpty()) { + return null; + } + long serverId; + try { + serverId = Long.parseLong(serverIdStr); + } catch (NumberFormatException e) { + return null; + } + return new ParsedRef(serverId, slug, hash); + } + + /** Cheap O(prefix length) check used by routing code paths. */ + public static boolean isMcpPrefixedName(String name) { + return name != null && name.startsWith(PREFIX); + } + + /** Reproduce the hash6 of a known raw name — used for cache reverse lookup. */ + public static String hash6(String rawToolName) { + if (rawToolName == null) { + throw new IllegalArgumentException("rawToolName must not be null"); + } + try { + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest(rawToolName.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(HASH_LEN); + for (int i = 0; sb.length() < HASH_LEN; i++) { + sb.append(BASE32[digest[i] & 0x1F]); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is mandated by every standard Java runtime — reaching + // this branch means the JVM is misconfigured and the application + // has bigger problems than tool naming. + throw new IllegalStateException("SHA-256 unavailable", e); + } + } + + private static String slug(String raw) { + String s = UNSAFE.matcher(raw.toLowerCase(Locale.ROOT)).replaceAll("_"); + if (s.length() > SLUG_MAX) { + s = s.substring(0, SLUG_MAX); + } + // A raw name composed entirely of non-ASCII chars (e.g. pure CJK) + // collapses to underscores and then to an empty slug after trimming; + // give it a stable placeholder so the prefixed name is still + // well-formed and the hash carries the actual identity. + if (s.replace("_", "").isEmpty()) { + return "tool"; + } + return s; + } + + /** + * Decoded prefix components. {@code rawToolName} is intentionally absent + * — recover it via the per-server tools cache when needed. + */ + public record ParsedRef(long serverId, String slug, String hash6) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallback.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallback.java new file mode 100644 index 00000000..f30e6cf5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallback.java @@ -0,0 +1,70 @@ +package vip.mate.tool.mcp.runtime; + +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; + +/** + * Wraps a {@link ToolCallback} from an MCP server and overrides + * {@link ToolDefinition#name()} with a stable + * {@code mcp___} key. + * + *

Why wrap rather than configure the upstream provider's prefix + * generator: the upstream extension point only sees protocol-level + * connection metadata, not the database server id we want to anchor + * to. Keeping the prefix logic inside this package binds the contract + * to one place and survives upstream API changes. + * + *

Description, input schema, metadata, and {@code call(...)} are + * forwarded verbatim — the wrapper changes only the name, so guard, + * approval, observability, and return-direct routing all see the same + * string they will write to bindings. + */ +public final class PrefixedNameToolCallback implements ToolCallback { + + private final ToolCallback delegate; + private final ToolDefinition prefixedDefinition; + + public PrefixedNameToolCallback(String prefixedName, ToolCallback delegate) { + if (prefixedName == null || prefixedName.isBlank()) { + throw new IllegalArgumentException("prefixedName must not be blank"); + } + if (delegate == null) { + throw new IllegalArgumentException("delegate must not be null"); + } + this.delegate = delegate; + ToolDefinition original = delegate.getToolDefinition(); + this.prefixedDefinition = DefaultToolDefinition.builder() + .name(prefixedName) + .description(original != null ? original.description() : "") + .inputSchema(original != null ? original.inputSchema() : "{}") + .build(); + } + + @Override + public ToolDefinition getToolDefinition() { + return prefixedDefinition; + } + + @Override + public ToolMetadata getToolMetadata() { + return delegate.getToolMetadata(); + } + + @Override + public String call(String toolInput) { + return delegate.call(toolInput); + } + + @Override + public String call(String toolInput, ToolContext toolContext) { + return delegate.call(toolInput, toolContext); + } + + /** Exposed for diagnostic / wrapping detection (e.g. by ReturnDirect logic). */ + public ToolCallback getDelegate() { + return delegate; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/mcp/service/McpServerService.java b/mateclaw-server/src/main/java/vip/mate/tool/mcp/service/McpServerService.java index 294ace0f..30b90d86 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/mcp/service/McpServerService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/mcp/service/McpServerService.java @@ -3,6 +3,7 @@ package vip.mate.tool.mcp.service; import cn.hutool.json.JSONUtil; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import io.modelcontextprotocol.spec.McpSchema; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; @@ -13,6 +14,7 @@ import vip.mate.tool.mcp.runtime.McpClientManager; import vip.mate.tool.mcp.runtime.McpClientManager.ConnectionResult; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -198,7 +200,7 @@ public class McpServerService { try { ConnectionResult result = mcpClientManager.connect(server); if (result.success()) { - updateStatus(server.getId(), "connected", null, result.toolCount()); + onConnectSuccess(server.getId()); } else { updateStatus(server.getId(), "error", result.message(), 0); } @@ -226,7 +228,7 @@ public class McpServerService { try { ConnectionResult result = mcpClientManager.connect(server); if (result.success()) { - updateStatus(server.getId(), "connected", null, result.toolCount()); + onConnectSuccess(server.getId()); } else { updateStatus(server.getId(), "error", result.message(), 0); } @@ -284,7 +286,7 @@ public class McpServerService { try { ConnectionResult result = mcpClientManager.connect(server); if (result.success()) { - updateStatus(server.getId(), "connected", null, result.toolCount()); + onConnectSuccess(server.getId()); } else { mcpClientManager.remove(server.getId()); updateStatus(server.getId(), "error", result.message(), 0); @@ -300,7 +302,7 @@ public class McpServerService { try { ConnectionResult result = mcpClientManager.replace(server); if (result.success()) { - updateStatus(server.getId(), "connected", null, result.toolCount()); + onConnectSuccess(server.getId()); } else { mcpClientManager.remove(server.getId()); updateStatus(server.getId(), "error", result.message(), 0); @@ -312,7 +314,29 @@ public class McpServerService { } } + /** + * Common success path for every connect entry point: snapshot the + * just-discovered tools into the {@code tools_cache_json} column in + * the same DB roundtrip as the status update, so downstream code that + * reads from the entity sees both pieces consistently. + * + *

Cache is only ever overwritten on success — failures preserve the + * last successful snapshot, keeping the agent picker rendering + * something useful while the upstream server is briefly down. + */ + private void onConnectSuccess(Long serverId) { + List tools = mcpClientManager.getServerTools(serverId); + String cacheJson = serializeToolsCache(tools); + updateStatusWithCache(serverId, "connected", null, tools.size(), cacheJson); + } + private void updateStatus(Long id, String status, String error, int toolCount) { + // Failure paths do NOT touch the tools cache — keep the last + // successful snapshot so the picker stays populated. + updateStatusWithCache(id, status, error, toolCount, null); + } + + private void updateStatusWithCache(Long id, String status, String error, int toolCount, String cacheJson) { try { LambdaUpdateWrapper wrapper = new LambdaUpdateWrapper<>(); wrapper.eq(McpServerEntity::getId, id); @@ -322,6 +346,10 @@ public class McpServerService { if ("connected".equals(status)) { wrapper.set(McpServerEntity::getLastConnectedTime, LocalDateTime.now()); } + if (cacheJson != null) { + wrapper.set(McpServerEntity::getToolsCacheJson, cacheJson); + wrapper.set(McpServerEntity::getToolsCacheUpdatedAt, LocalDateTime.now()); + } wrapper.set(McpServerEntity::getUpdateTime, LocalDateTime.now()); mcpServerMapper.update(null, wrapper); } catch (Exception e) { @@ -329,6 +357,37 @@ public class McpServerService { } } + /** + * Serialize the list returned by the upstream {@code listTools()} call + * into a stable JSON shape: an array of {@code {name, description, + * inputSchema}} entries. Schema is stored as the JSON text the upstream + * surfaces (already a JSON-Schema object) so the picker can show it + * verbatim without re-stringifying. + */ + private String serializeToolsCache(List tools) { + if (tools == null || tools.isEmpty()) { + return "[]"; + } + List> rows = new ArrayList<>(tools.size()); + for (McpSchema.Tool t : tools) { + if (t == null || t.name() == null || t.name().isBlank()) continue; + Map row = new java.util.LinkedHashMap<>(); + row.put("name", t.name()); + row.put("description", t.description() != null ? t.description() : ""); + // inputSchema in the MCP record is a JsonSchema record; let the + // JSON utility serialize it, falling back to "{}" if it can't. + try { + row.put("inputSchema", t.inputSchema() != null + ? JSONUtil.parse(JSONUtil.toJsonStr(t.inputSchema())) + : "{}"); + } catch (Exception e) { + row.put("inputSchema", "{}"); + } + rows.add(row); + } + return JSONUtil.toJsonStr(rows); + } + private void validateServer(McpServerEntity entity) { if (entity.getName() == null || entity.getName().isBlank()) { throw new MateClawException("err.mcp.name_required", "MCP server 名称不能为空"); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/model/AvailableToolDTO.java b/mateclaw-server/src/main/java/vip/mate/tool/model/AvailableToolDTO.java new file mode 100644 index 00000000..5eb1dc7a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/model/AvailableToolDTO.java @@ -0,0 +1,96 @@ +package vip.mate.tool.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Picker DTO for the unified agent tool selector. + * + *

One row per atomic tool the agent can be bound to — built-in tools + * appear under {@code source="builtin"}, MCP tools appear under + * {@code source="mcp"} and are grouped by their server. The {@link #name} + * field is the value the UI saves into {@code mate_agent_tool.tool_name}; + * for MCP tools it is the prefixed callback name returned by the resolver + * so picker and runtime use the same key. + */ +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class AvailableToolDTO { + + /** + * Stable per-row identifier for the picker. The frontend uses this as + * the {@code v-for :key} so two rows with the same prefixed + * {@link #name} (e.g. a hash-collision pair) don't reuse each other's + * DOM state. Server-assigned, opaque to the client. + */ + private String rowId; + + /** {@code "builtin"} or {@code "mcp"}. */ + private String source; + + /** MCP server id when {@code source == "mcp"}; null otherwise. */ + private Long providerId; + + /** Human-readable provider label — server display name for MCP, empty for builtin. */ + private String providerName; + + /** What the UI saves into {@code mate_agent_tool.tool_name}. */ + private String name; + + /** Original raw tool name as advertised upstream. UI shows this. */ + private String rawName; + + /** Tool description shown as the picker subtitle. */ + private String description; + + /** Group label for the picker UI section header (e.g. {@code "MCP · github"}). */ + private String group; + + /** Stable group key for collapse/expand state across renames. */ + private String groupId; + + /** + * {@code true} when the entry comes from the cache while the upstream + * MCP server is currently disconnected. The picker should grey it out; + * runtime callbacks for stale tools are absent so the LLM cannot call + * them either way. + */ + private boolean stale; + + /** + * {@code false} → the picker must disable selection. Currently set when + * a hash collision was detected for the same (serverId, prefixed-name) + * pair. {@code true} for everything that can be safely bound. + */ + private boolean available; + + /** + * Machine-readable cause when {@link #available} is {@code false}. + * Examples: {@code "HASH_COLLISION"} (with the conflicting raw name in + * a follow-up message), {@code "DUPLICATE_RAW_NAME"}. + */ + private String unavailableReason; + + public static AvailableToolDTO fromBuiltin(ToolEntity t) { + return AvailableToolDTO.builder() + // Built-in tool names are unique by ToolRegistry contract, + // so name suffices as a stable rowId. + .rowId("builtin#" + t.getName()) + .source("builtin") + .providerId(null) + .providerName(null) + .name(t.getName()) + .rawName(t.getName()) + .description(t.getDescription() != null ? t.getDescription() : "") + .group("builtin") + .groupId("builtin") + .stale(false) + .available(true) + .unavailableReason(null) + .build(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/model3d/Model3dGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/model3d/Model3dGenerationService.java index 9fed7eb9..66d29c20 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/model3d/Model3dGenerationService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/model3d/Model3dGenerationService.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import vip.mate.channel.AsyncTaskMediaDispatcher; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; import vip.mate.task.AsyncTaskService; @@ -44,6 +45,12 @@ public class Model3dGenerationService { private final ConversationService conversationService; private final Model3dFileDownloader fileDownloader; private final ObjectMapper objectMapper; + /** + * Forward async-task completion to the conversation's bound IM channel + * adapter so users on WeCom / DingTalk / Feishu / etc. receive the + * generated 3D model as a native attachment. SSE remains the Web path. + */ + private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher; private static final String TASK_TYPE = "model3d_generation"; @@ -173,6 +180,7 @@ public class Model3dGenerationService { String fileName = localPath.getFileName().toString(); MessageContentPart modelPart = MessageContentPart.model3d(null, fileName); modelPart.setFileUrl(servingUrl); + modelPart.setStoredName(fileName); // model/gltf-binary for .glb is the iana-registered MIME; downstream // only cares about the URL, not the MIME header. if (fileName.endsWith(".glb")) { @@ -184,11 +192,18 @@ public class Model3dGenerationService { } else if (fileName.endsWith(".usdz")) { modelPart.setContentType("model/vnd.usdz+zip"); } + // Set absolute disk path so IM adapters can read bytes locally + // instead of round-tripping through /api/v1/chat/files (auth). + modelPart.setPath(localPath.toAbsolutePath().toString()); + try { + modelPart.setFileSize(java.nio.file.Files.size(localPath)); + } catch (Exception ignored) { /* best-effort */ } + List parts = List.of(modelPart); conversationService.saveMessage( task.getConversationId(), "assistant", "3D 模型已生成完毕", - List.of(modelPart), "completed"); + parts, "completed"); Map extra = new LinkedHashMap<>(); extra.put("modelUrl", servingUrl); @@ -196,6 +211,14 @@ public class Model3dGenerationService { asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed", true, extra, null); + // Forward to the conversation's bound IM channel adapter so + // WeCom / DingTalk / Feishu etc. users receive the model as a + // native attachment (the SSE broadcast above only reaches Web). + // Most IM channels will fall back to a markdown link via + // sendFallbackText if their adapter doesn't natively support + // model/* media — that's fine, the dispatcher logs and continues. + asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts); + log.info("[Model3dGen] Task {} completed, model saved: {}", task.getTaskId(), servingUrl); } catch (Exception e) { log.error("[Model3dGen] Completion handling failed for task {}: {}", diff --git a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java index 98c581f0..ad72b750 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/music/MusicGenerationService.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import vip.mate.channel.AsyncTaskMediaDispatcher; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; import vip.mate.task.AsyncTaskService; @@ -48,6 +49,12 @@ public class MusicGenerationService { private final AsyncTaskService asyncTaskService; private final ConversationService conversationService; private final ObjectMapper objectMapper; + /** + * Forward async-task completion to the conversation's bound IM channel + * adapter so WeCom / DingTalk / Feishu / etc. users receive the generated + * audio as a native attachment. Web-class channels keep using SSE only. + */ + private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher; private static final Path UPLOAD_ROOT = Paths.get("data", "chat-uploads"); private static final String TASK_TYPE = "music_generation"; @@ -110,6 +117,18 @@ public class MusicGenerationService { asyncTaskService.updateStatus(task.getTaskId(), "running", null, null, null); MusicGenerationResult result = generateWithFallback(request, config); + + // The conversation may have been deleted while the provider was + // blocking (~120s). Gate the entire post-provider tail — status + // update, broadcast, persistence — so we never write to a + // tombstoned conversation regardless of whether the provider + // succeeded or failed. + if (asyncTaskService.isConversationCanceled(conversationId)) { + log.info("[Music] Task {} (success={}) aborted: conversation {} was deleted", + task.getTaskId(), result.isSuccess(), conversationId); + return; + } + if (!result.isSuccess()) { asyncTaskService.updateStatus(task.getTaskId(), "failed", null, null, result.getErrorMessage()); @@ -118,9 +137,10 @@ public class MusicGenerationService { return; } - String audioUrl = persistAudio(conversationId, task.getTaskId(), result); + PersistedAudio persisted = persistAudio(conversationId, task.getTaskId(), result); + String audioUrl = persisted.servingUrl(); - saveAssistantMessage(conversationId, audioUrl, result); + List parts = saveAssistantMessage(conversationId, persisted, result); ObjectNode resultJson = objectMapper.createObjectNode(); resultJson.put("audioUrl", audioUrl); @@ -140,9 +160,19 @@ public class MusicGenerationService { asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed", true, extra, null); + // Forward to the conversation's bound IM channel adapter so + // WeCom / DingTalk / Feishu etc. users receive the audio as a + // native attachment (the SSE broadcast above only reaches Web). + asyncTaskMediaDispatcher.forwardToImIfBound(conversationId, parts); + log.info("[Music] Task {} succeeded, audio at {}", task.getTaskId(), audioUrl); } catch (Exception e) { log.error("[Music] Task {} worker failed: {}", task.getTaskId(), e.getMessage(), e); + if (asyncTaskService.isConversationCanceled(conversationId)) { + log.info("[Music] Skipping failure status/broadcast for deleted conversation {}", + conversationId); + return; + } asyncTaskService.updateStatus(task.getTaskId(), "failed", null, null, "音乐生成异常: " + e.getMessage()); asyncTaskService.broadcastTaskEventWithData(task, "async_task_completed", @@ -150,29 +180,48 @@ public class MusicGenerationService { } } - private String persistAudio(String conversationId, String taskId, - MusicGenerationResult result) throws IOException { + /** + * Stash audio bytes on disk and surface both the absolute local path and + * the browser-servable URL so callers can hand both to the + * {@link MessageContentPart}. The local path is what IM channel adapters + * read directly (faster, no auth round-trip); the serving URL is what + * the Web bubble renders. + */ + private record PersistedAudio(Path localPath, String servingUrl, String fileName) {} + + private PersistedAudio persistAudio(String conversationId, String taskId, + MusicGenerationResult result) throws IOException { Path dir = UPLOAD_ROOT.resolve(conversationId); Files.createDirectories(dir); String fileName = "music_" + taskId + "." + result.getFormat(); Path filePath = dir.resolve(fileName); Files.write(filePath, result.getAudioData()); - return "/api/v1/chat/files/" + conversationId + "/" + fileName; + String servingUrl = "/api/v1/chat/files/" + conversationId + "/" + fileName; + return new PersistedAudio(filePath, servingUrl, fileName); } - private void saveAssistantMessage(String conversationId, String audioUrl, - MusicGenerationResult result) { - MessageContentPart audioPart = MessageContentPart.audio(null, - audioUrl.substring(audioUrl.lastIndexOf('/') + 1)); - audioPart.setFileUrl(audioUrl); + private List saveAssistantMessage(String conversationId, + PersistedAudio persisted, + MusicGenerationResult result) { + MessageContentPart audioPart = MessageContentPart.audio(null, persisted.fileName()); + audioPart.setFileUrl(persisted.servingUrl()); + audioPart.setStoredName(persisted.fileName()); audioPart.setContentType(result.getContentType()); + // Set absolute disk path so IM adapters can read bytes locally + // instead of round-tripping through /api/v1/chat/files (auth). + audioPart.setPath(persisted.localPath().toAbsolutePath().toString()); + try { + audioPart.setFileSize(Files.size(persisted.localPath())); + } catch (Exception ignored) { /* best-effort */ } StringBuilder content = new StringBuilder("音乐生成完成"); if (result.getLyrics() != null && !result.getLyrics().isBlank()) { content.append("\n\n歌词:\n").append(result.getLyrics()); } + List parts = List.of(audioPart); conversationService.saveMessage(conversationId, "assistant", - content.toString(), List.of(audioPart), "completed"); + content.toString(), parts, "completed"); + return parts; } private MusicGenerationResult generateWithFallback(MusicGenerationRequest request, diff --git a/mateclaw-server/src/main/java/vip/mate/tool/service/AvailableToolService.java b/mateclaw-server/src/main/java/vip/mate/tool/service/AvailableToolService.java new file mode 100644 index 00000000..a39a68cf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/service/AvailableToolService.java @@ -0,0 +1,184 @@ +package vip.mate.tool.service; + +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.tool.mcp.model.McpServerEntity; +import vip.mate.tool.mcp.runtime.McpHashCollisionDetector; +import vip.mate.tool.mcp.service.McpServerService; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.model.ToolEntity; + +import java.util.ArrayList; +import java.util.List; + +/** + * Aggregator behind {@code GET /api/v1/tools/available}. + * + *

Returns one DTO per atomic tool the agent edit picker can offer: + * built-in tools (from {@link ToolService#listEnabledTools()}) plus every + * MCP tool persisted in {@link McpServerEntity#getToolsCacheJson()}. + * + *

Reads the cache rather than making a live MCP {@code listTools()} + * roundtrip so the picker stays fast and stable through brief upstream + * disconnects. The {@code stale} flag tells the UI when the entry came + * from a server that isn't currently connected. + * + *

Hash collisions are handled by reusing the same + * {@link McpHashCollisionDetector} the runtime uses, so an entry the + * runtime would skip never appears in the picker as bindable. Without + * this, the user could save a {@code mate_agent_tool.tool_name} that + * resolves to nothing at chat time. + * + *

Scope: this aggregator covers the two tool sources users can + * bind from the agent edit screen — built-in {@code @Tool} beans + * (persisted in {@code mate_tool}) and MCP-discovered tools (cached on + * the server row). Plugin-registered {@code ToolCallback} beans surfaced + * by other parts of the runtime are intentionally NOT listed here: those + * are not user-bindable from the agent picker today, and the picker's + * "saved name == runtime callback key" contract only needs to hold for + * the rows the picker actually emits. If plugin tools later become + * user-bindable, extend this aggregator (or accept that they go through + * a separate config path) — see {@code AgentBindingService}'s + * {@code SYSTEM_LEVEL_TOOLS} carve-out for the same reasoning. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class AvailableToolService { + + private final ToolService toolService; + private final McpServerService mcpServerService; + + public List listAvailable() { + List out = new ArrayList<>(); + appendBuiltinTools(out); + appendMcpTools(out); + return out; + } + + private void appendBuiltinTools(List out) { + for (ToolEntity t : toolService.listEnabledTools()) { + if (t == null || t.getName() == null || t.getName().isBlank()) continue; + out.add(AvailableToolDTO.fromBuiltin(t)); + } + } + + private void appendMcpTools(List out) { + List servers; + try { + servers = mcpServerService.listEnabled(); + } catch (Exception e) { + log.warn("AvailableToolService: listEnabled MCP servers failed: {}", e.getMessage()); + return; + } + + for (McpServerEntity s : servers) { + try { + appendOneMcpServer(out, s); + } catch (Exception e) { + log.warn("AvailableToolService: skipping MCP server {} due to: {}", + s.getId(), e.getMessage()); + } + } + } + + private void appendOneMcpServer(List out, McpServerEntity server) { + List cached = parseCache(server.getToolsCacheJson()); + if (cached.isEmpty()) { + return; + } + boolean stale = !"connected".equalsIgnoreCase(nullSafe(server.getLastStatus())); + String groupLabel = "MCP · " + nullSafe(server.getName()); + String groupKey = "mcp:" + server.getId(); + + // Run the collision check on the same raw-name list the runtime uses + // when it registers callbacks. Sharing this exact decision shape is + // what guarantees picker rows and AgentToolSet entries stay in sync. + List rawNames = new ArrayList<>(cached.size()); + for (CachedTool c : cached) rawNames.add(c.name); + List decisions = + McpHashCollisionDetector.classify(server.getId(), rawNames); + + // Walk cache and decisions in lockstep — classify() drops blank + // raws, so advance the decision pointer only when the cache row's + // name is non-blank. This is the same alignment McpClientManager's + // wrapServerCallbacks uses; both must agree on which entry got + // which decision when the same raw appears more than once. + int dIdx = 0; + int rowIdx = 0; + for (CachedTool c : cached) { + if (c.name == null || c.name.isBlank()) { + continue; + } + if (dIdx >= decisions.size()) { + break; + } + McpHashCollisionDetector.Decision d = decisions.get(dIdx++); + out.add(buildMcpDto(server, groupLabel, groupKey, stale, c, d, rowIdx++)); + } + } + + private AvailableToolDTO buildMcpDto(McpServerEntity server, String groupLabel, String groupKey, + boolean stale, CachedTool cached, + McpHashCollisionDetector.Decision decision, int rowIdx) { + // rowId distinguishes rows that share the same prefixed `name` but + // arose from distinct raw entries (e.g. duplicate-raw, hash + // collision). Without it, a Vue v-for keyed on `name` reuses DOM + // for the unavailable twin and selection/disabled state goes + // stale. Including the raw and a per-server index makes the key + // stable across re-renders without depending on array order. + String rowId = groupKey + "#" + rowIdx + "#" + cached.name; + return AvailableToolDTO.builder() + .rowId(rowId) + .source("mcp") + .providerId(server.getId()) + .providerName(server.getName()) + .name(decision.prefixedName()) + .rawName(cached.name) + .description(cached.description) + .group(groupLabel) + .groupId(groupKey) + .stale(stale) + .available(decision.bindable()) + .unavailableReason(decision.unavailableReason()) + .build(); + } + + /** + * Parse the {@code tools_cache_json} column written by + * {@link vip.mate.tool.mcp.service.McpServerService}. Returns an empty + * list when the column is null/blank/malformed — the picker can render + * a server with no tools just as well as one with tools. + */ + private static List parseCache(String json) { + if (json == null || json.isBlank()) { + return List.of(); + } + try { + JSONArray arr = JSONUtil.parseArray(json); + List out = new ArrayList<>(arr.size()); + for (Object o : arr) { + if (!(o instanceof JSONObject jo)) continue; + String name = jo.getStr("name"); + if (name == null || name.isBlank()) continue; + String desc = jo.getStr("description", ""); + out.add(new CachedTool(name, desc != null ? desc : "")); + } + return out; + } catch (Exception e) { + log.debug("AvailableToolService: failed to parse tools_cache_json: {}", e.getMessage()); + return List.of(); + } + } + + private static String nullSafe(String s) { + return s == null ? "" : s; + } + + /** Trivial bag struct for the cached tool fields the picker needs. */ + private record CachedTool(String name, String description) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java index 4a282ee2..d9ce7753 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/VideoGenerationService.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; +import vip.mate.channel.AsyncTaskMediaDispatcher; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; import vip.mate.task.AsyncTaskService; @@ -34,6 +35,12 @@ public class VideoGenerationService { private final ConversationService conversationService; private final VideoFileDownloader fileDownloader; private final ObjectMapper objectMapper; + /** + * Forward async-task completion to the conversation's bound IM channel + * adapter so WeCom / DingTalk / Feishu / etc. users actually receive the + * generated video as a native attachment. SSE remains the Web path. + */ + private final AsyncTaskMediaDispatcher asyncTaskMediaDispatcher; private static final String TASK_TYPE = "video_generation"; @@ -151,6 +158,16 @@ public class VideoGenerationService { * 任务完成时的回写逻辑:下载视频 → 保存消息 → 广播 SSE */ private void handleCompletion(AsyncTaskEntity task, TaskPollResult result) { + // The conversation may have been deleted while the poller was running. + // Gate every post-completion side effect — file write, message save, + // success/failure broadcast — so we never write to a tombstoned + // conversation regardless of which sub-branch we'd take. + if (asyncTaskService.isConversationCanceled(task.getConversationId())) { + log.info("[VideoGen] Task {} (success={}) aborted: conversation {} was deleted", + task.getTaskId(), result.succeeded(), task.getConversationId()); + return; + } + if (result.succeeded()) { try { String videoUrl = result.videoUrl(); @@ -166,23 +183,42 @@ public class VideoGenerationService { String servingUrl = fileDownloader.toServingUrl(task.getConversationId(), localPath); // 保存 assistant 消息(含 video content part) - MessageContentPart videoPart = MessageContentPart.video(null, localPath.getFileName().toString()); + String videoFileName = localPath.getFileName().toString(); + MessageContentPart videoPart = MessageContentPart.video(null, videoFileName); videoPart.setFileUrl(servingUrl); + videoPart.setStoredName(videoFileName); videoPart.setContentType("video/mp4"); + // Set absolute disk path so IM adapters can read bytes locally + // instead of round-tripping through /api/v1/chat/files (auth). + videoPart.setPath(localPath.toAbsolutePath().toString()); + try { + videoPart.setFileSize(java.nio.file.Files.size(localPath)); + } catch (Exception ignored) { /* best-effort */ } + List parts = List.of(videoPart); conversationService.saveMessage( task.getConversationId(), "assistant", "视频已生成完毕", - List.of(videoPart), "completed"); + parts, "completed"); // SSE 广播 asyncTaskService.broadcastTaskEvent(task, "async_task_completed", true, servingUrl, null); + // Forward to the conversation's bound IM channel adapter so + // WeCom / DingTalk / Feishu etc. users receive the video as a + // native attachment (the SSE broadcast above only reaches Web). + asyncTaskMediaDispatcher.forwardToImIfBound(task.getConversationId(), parts); + log.info("[VideoGen] Task {} completed, video saved: {}", task.getTaskId(), servingUrl); } catch (Exception e) { log.error("[VideoGen] Completion handling failed for task {}: {}", task.getTaskId(), e.getMessage(), e); + if (asyncTaskService.isConversationCanceled(task.getConversationId())) { + log.info("[VideoGen] Skipping failure broadcast for deleted conversation {}", + task.getConversationId()); + return; + } asyncTaskService.broadcastTaskEvent(task, "async_task_completed", false, null, "视频下载或保存失败: " + e.getMessage()); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java index e71bb868..e86f40f8 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/video/provider/DashScopeVideoProvider.java @@ -4,6 +4,7 @@ import cn.hutool.http.HttpRequest; import cn.hutool.http.HttpResponse; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,16 +12,35 @@ import org.springframework.stereotype.Component; import vip.mate.llm.service.ModelProviderService; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.task.AsyncTaskService.TaskPollResult; -import vip.mate.tool.video.*; +import vip.mate.tool.video.VideoCapability; +import vip.mate.tool.video.VideoGenerationProvider; +import vip.mate.tool.video.VideoGenerationRequest; +import vip.mate.tool.video.VideoProviderCapabilities; +import vip.mate.tool.video.VideoSubmitResult; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; /** - * DashScope 视频生成 Provider — 支持通义万相 Wan 2.5 / Wanx 2.1 - *

- * 复用已有的 DashScope LLM provider 的 API Key。 - * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/video-generation + * DashScope video provider — supports two payload families on the same async + * task model, selected per model id: + * + *

    + *
  • Legacy ({@code services/aigc/video-generation/generation}) for + * wanx 2.1 and wan 2.5 turbo lines. Body uses {@code input.img_url} for + * image-to-video and {@code parameters.size} for sizing.
  • + *
  • Unified video-synthesis + * ({@code services/aigc/video-generation/video-synthesis}) for wan 2.7 + * and the happyhorse t2v line. Body uses {@code input.media[]} for the + * first frame plus {@code parameters.resolution} + {@code parameters.ratio} + * for sizing.
  • + *
+ * + * Routing is data-driven: each model is registered with its endpoint, body + * shape, and capability set; submit/build code consults the spec rather than + * branching on model id strings. * * @author MateClaw Team */ @@ -33,9 +53,54 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { private final ObjectMapper objectMapper; private static final String BASE_URL = "https://dashscope.aliyuncs.com/api/v1"; + private static final String LEGACY_ENDPOINT = BASE_URL + "/services/aigc/video-generation/generation"; + private static final String UNIFIED_ENDPOINT = BASE_URL + "/services/aigc/video-generation/video-synthesis"; + private static final String TASKS_ENDPOINT_PREFIX = BASE_URL + "/tasks/"; + private static final String DEFAULT_T2V_MODEL = "wan2.5-t2v-turbo"; private static final String DEFAULT_I2V_MODEL = "wan2.5-i2v-turbo"; + /** Package-private so per-routing tests can switch on it without reflection. */ + enum BodyShape { + /** input.img_url + parameters.size("1280*720") + parameters.duration. */ + LEGACY, + /** input.media[].first_frame + parameters.resolution + parameters.ratio + parameters.duration. */ + UNIFIED + } + + /** Package-private for unit tests; the MODELS map is the routing source of truth. */ + record ModelSpec( + String id, + String endpoint, + BodyShape bodyShape, + Set modes + ) {} + + private static final Map MODELS = buildCatalog(); + + private static Map buildCatalog() { + Map m = new LinkedHashMap<>(); + // Legacy line — text-to-video + m.put("wan2.5-t2v-turbo", new ModelSpec("wan2.5-t2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE))); + m.put("wanx2.1-t2v-turbo", new ModelSpec("wanx2.1-t2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.GENERATE))); + // Legacy line — image-to-video + m.put("wan2.5-i2v-turbo", new ModelSpec("wan2.5-i2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + m.put("wanx2.1-i2v-turbo", new ModelSpec("wanx2.1-i2v-turbo", + LEGACY_ENDPOINT, BodyShape.LEGACY, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + // Unified video-synthesis line — wan 2.7 + m.put("wan2.7-t2v-2026-04-25", new ModelSpec("wan2.7-t2v-2026-04-25", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE))); + m.put("wan2.7-i2v-2026-04-25", new ModelSpec("wan2.7-i2v-2026-04-25", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.IMAGE_TO_VIDEO))); + // Unified video-synthesis line — happyhorse text-to-video + m.put("happyhorse-1.0-t2v", new ModelSpec("happyhorse-1.0-t2v", + UNIFIED_ENDPOINT, BodyShape.UNIFIED, Set.of(VideoCapability.GENERATE))); + return Map.copyOf(m); + } + @Override public String id() { return "dashscope"; @@ -43,7 +108,7 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { @Override public String label() { - return "DashScope (通义万相)"; + return "DashScope (Tongyi Wanxiang / HappyHorse)"; } @Override @@ -66,10 +131,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { return VideoProviderCapabilities.builder() .modes(capabilities()) .aspectRatios(List.of("16:9", "9:16", "1:1")) - .supportedDurations(List.of(5, 10)) - .maxDurationSeconds(10) + .supportedDurations(List.of(2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)) + .maxDurationSeconds(15) .defaultModel(DEFAULT_T2V_MODEL) - .models(List.of("wan2.5-t2v-turbo", "wan2.5-i2v-turbo", "wanx2.1-t2v-turbo", "wanx2.1-i2v-turbo")) + .models(List.copyOf(MODELS.keySet())) .build(); } @@ -86,14 +151,12 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { public VideoSubmitResult submit(VideoGenerationRequest request, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return VideoSubmitResult.failure(id(), "DashScope API Key 未配置"); + return VideoSubmitResult.failure(id(), "DashScope API Key not configured"); } - + ModelSpec spec = resolveSpec(request); try { - String model = resolveModel(request); - ObjectNode body = buildRequestBody(request, model); - - HttpResponse response = HttpRequest.post(BASE_URL + "/services/aigc/video-generation/generation") + ObjectNode body = buildRequestBody(request, spec); + HttpResponse response = HttpRequest.post(spec.endpoint()) .header("Authorization", "Bearer " + apiKey) .header("Content-Type", "application/json") .header("X-DashScope-Async", "enable") @@ -102,19 +165,17 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { .execute(); JsonNode result = objectMapper.readTree(response.body()); - if (response.getStatus() == 200 && result.has("output")) { String taskId = result.path("output").path("task_id").asText(); - log.info("[DashScope Video] Submitted task: {} (model={})", taskId, model); + log.info("[DashScope Video] Submitted task {} (model={})", taskId, spec.id()); return VideoSubmitResult.success(taskId, id()); - } else { - String errMsg = result.has("message") ? result.get("message").asText() - : "HTTP " + response.getStatus(); - log.warn("[DashScope Video] Submit failed: {}", errMsg); - return VideoSubmitResult.failure(id(), errMsg); } + String errMsg = result.has("message") ? result.get("message").asText() + : "HTTP " + response.getStatus(); + log.warn("[DashScope Video] Submit failed (model={}): {}", spec.id(), errMsg); + return VideoSubmitResult.failure(id(), errMsg); } catch (Exception e) { - log.error("[DashScope Video] Submit error: {}", e.getMessage(), e); + log.error("[DashScope Video] Submit error (model={}): {}", spec.id(), e.getMessage(), e); return VideoSubmitResult.failure(id(), e.getMessage()); } } @@ -123,11 +184,10 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { public TaskPollResult checkStatus(String providerTaskId, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); if (apiKey == null) { - return TaskPollResult.failed("DashScope API Key 未配置"); + return TaskPollResult.failed("DashScope API Key not configured"); } - try { - HttpResponse response = HttpRequest.get(BASE_URL + "/tasks/" + providerTaskId) + HttpResponse response = HttpRequest.get(TASKS_ENDPOINT_PREFIX + providerTaskId) .header("Authorization", "Bearer " + apiKey) .timeout(15_000) .execute(); @@ -135,14 +195,13 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { JsonNode result = objectMapper.readTree(response.body()); JsonNode output = result.path("output"); String taskStatus = output.path("task_status").asText(); - return switch (taskStatus) { case "SUCCEEDED" -> { String videoUrl = extractVideoUrl(output); yield TaskPollResult.succeeded(videoUrl, null, output.toString()); } case "FAILED" -> { - String errMsg = output.has("message") ? output.get("message").asText() : "任务失败"; + String errMsg = output.has("message") ? output.get("message").asText() : "task failed"; yield TaskPollResult.failed(errMsg); } case "RUNNING" -> TaskPollResult.running(null); @@ -150,11 +209,109 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { }; } catch (Exception e) { log.error("[DashScope Video] Poll error for task {}: {}", providerTaskId, e.getMessage()); - return null; // 轮询异常不终止,等下次重试 + return null; } } - // ==================== 内部方法 ==================== + // ==================== spec resolution ==================== + + /** Package-private for direct unit tests — submit() goes over HTTP and is not a unit-test surface. */ + ModelSpec resolveSpec(VideoGenerationRequest request) { + String requested = request.getModel(); + if (requested != null && !requested.isBlank() && MODELS.containsKey(requested)) { + return MODELS.get(requested); + } + // Fall back to a default by mode. + String defaultId = request.getMode() == VideoCapability.IMAGE_TO_VIDEO + ? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL; + return MODELS.get(defaultId); + } + + // ==================== body building ==================== + + /** Package-private for unit tests; verify the JSON shape per body family without HTTP. */ + ObjectNode buildRequestBody(VideoGenerationRequest request, ModelSpec spec) { + return switch (spec.bodyShape()) { + case LEGACY -> buildLegacyBody(request, spec); + case UNIFIED -> buildUnifiedBody(request, spec); + }; + } + + private ObjectNode buildLegacyBody(VideoGenerationRequest request, ModelSpec spec) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO) + && request.getImageUrl() != null && !request.getImageUrl().isBlank()) { + input.put("img_url", request.getImageUrl()); + } + + ObjectNode parameters = body.putObject("parameters"); + String size = aspectRatioToLegacySize(request.getAspectRatio()); + if (size != null) { + parameters.put("size", size); + } + if (request.getDurationSeconds() != null) { + parameters.put("duration", String.valueOf(request.getDurationSeconds())); + } + return body; + } + + private ObjectNode buildUnifiedBody(VideoGenerationRequest request, ModelSpec spec) { + ObjectNode body = objectMapper.createObjectNode(); + body.put("model", spec.id()); + + ObjectNode input = body.putObject("input"); + input.put("prompt", request.getPrompt() == null ? "" : request.getPrompt()); + if (spec.modes().contains(VideoCapability.IMAGE_TO_VIDEO) + && request.getImageUrl() != null && !request.getImageUrl().isBlank()) { + ArrayNode media = input.putArray("media"); + ObjectNode firstFrame = media.addObject(); + firstFrame.put("type", "first_frame"); + firstFrame.put("url", request.getImageUrl()); + } + + ObjectNode parameters = body.putObject("parameters"); + String resolution = aspectRatioToUnifiedResolution(request.getAspectRatio()); + parameters.put("resolution", resolution); + if (request.getAspectRatio() != null && !request.getAspectRatio().isBlank()) { + parameters.put("ratio", request.getAspectRatio()); + } + if (request.getDurationSeconds() != null) { + // Unified endpoint expects the duration as an integer. + parameters.put("duration", request.getDurationSeconds()); + } + return body; + } + + private String aspectRatioToLegacySize(String aspectRatio) { + if (aspectRatio == null) return null; + return switch (aspectRatio) { + case "16:9" -> "1280*720"; + case "9:16" -> "720*1280"; + case "1:1" -> "720*720"; + default -> null; + }; + } + + private String aspectRatioToUnifiedResolution(String aspectRatio) { + // Default to 720P; the unified endpoint also accepts 1080P. Callers that + // want to override should pass it via extraParams in a future iteration. + return "720P"; + } + + private String extractVideoUrl(JsonNode output) { + if (output.has("video_url")) { + return output.get("video_url").asText(); + } + JsonNode results = output.path("results"); + if (results.isArray() && !results.isEmpty()) { + return results.get(0).path("url").asText(null); + } + return null; + } private String getDashScopeApiKey() { try { @@ -164,59 +321,4 @@ public class DashScopeVideoProvider implements VideoGenerationProvider { return null; } } - - private String resolveModel(VideoGenerationRequest request) { - if (request.getModel() != null && !request.getModel().isBlank()) { - return request.getModel(); - } - return request.getMode() == VideoCapability.IMAGE_TO_VIDEO - ? DEFAULT_I2V_MODEL : DEFAULT_T2V_MODEL; - } - - private ObjectNode buildRequestBody(VideoGenerationRequest request, String model) { - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - - ObjectNode input = body.putObject("input"); - input.put("prompt", request.getPrompt()); - - if (request.getMode() == VideoCapability.IMAGE_TO_VIDEO && request.getImageUrl() != null) { - input.put("img_url", request.getImageUrl()); - } - - ObjectNode parameters = body.putObject("parameters"); - if (request.getAspectRatio() != null) { - // DashScope 使用 size 参数,如 "1280*720" - String size = aspectRatioToSize(request.getAspectRatio()); - if (size != null) { - parameters.put("size", size); - } - } - if (request.getDurationSeconds() != null) { - parameters.put("duration", String.valueOf(request.getDurationSeconds())); - } - - return body; - } - - private String aspectRatioToSize(String aspectRatio) { - return switch (aspectRatio) { - case "16:9" -> "1280*720"; - case "9:16" -> "720*1280"; - case "1:1" -> "720*720"; - default -> null; - }; - } - - private String extractVideoUrl(JsonNode output) { - JsonNode results = output.path("results"); - if (results.isArray() && !results.isEmpty()) { - return results.get(0).path("url").asText(null); - } - // 有些模型返回 video_url - if (output.has("video_url")) { - return output.get("video_url").asText(); - } - return null; - } } diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/api/TriggerController.java b/mateclaw-server/src/main/java/vip/mate/trigger/api/TriggerController.java new file mode 100644 index 00000000..89f78112 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/api/TriggerController.java @@ -0,0 +1,119 @@ +package vip.mate.trigger.api; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.trigger.ingest.TriggerEventEnvelope; +import vip.mate.trigger.ingest.TriggerEventIngestService; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.service.TriggerService; + +import java.util.List; +import java.util.Map; + +/** + * REST surface for cron / event triggers. The generic event ingest endpoint + * exists so external systems (n8n, GitHub webhooks, ad-hoc curl) can post + * events without going through a dedicated channel adapter — useful for + * smoke-testing a trigger before the channel integration lands. + */ +@Tag(name = "触发器管理") +@RestController +@RequestMapping("/api/v1/triggers") +@RequiredArgsConstructor +public class TriggerController { + + private final TriggerService triggerService; + private final TriggerEventIngestService ingestService; + + @Operation(summary = "List triggers in the caller's workspace.") + @GetMapping + public R> list(@RequestHeader("X-Workspace-Id") long workspaceId) { + return R.ok(triggerService.listByWorkspace(workspaceId)); + } + + @Operation(summary = "Get a trigger by id, scoped to the caller's workspace.") + @GetMapping("/{id}") + public R get(@PathVariable long id, + @RequestHeader("X-Workspace-Id") long workspaceId) { + TriggerEntity row = triggerService.get(id, workspaceId); + if (row == null) return R.fail("trigger not found: " + id); + return R.ok(row); + } + + @Operation(summary = "Create a trigger; if enabled, registers it with the scheduler.") + @PostMapping + public R create(@RequestBody TriggerEntity trigger, + @RequestHeader("X-Workspace-Id") long workspaceId) { + // The controller forces workspace from the trusted header — the + // body's workspaceId is ignored so a caller can't plant a trigger + // into another workspace by tweaking the JSON. + try { + return R.ok(triggerService.create(trigger, workspaceId)); + } catch (IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + + @Operation(summary = "Update a trigger; pattern_version bumps when the cron expression changes.") + @PutMapping("/{id}") + public R update(@PathVariable long id, + @RequestBody TriggerEntity trigger, + @RequestHeader("X-Workspace-Id") long workspaceId) { + try { + return R.ok(triggerService.update(id, workspaceId, trigger)); + } catch (IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + + @Operation(summary = "Delete a trigger and unregister its schedule.") + @DeleteMapping("/{id}") + public R delete(@PathVariable long id, + @RequestHeader("X-Workspace-Id") long workspaceId) { + triggerService.delete(id, workspaceId); + return R.ok(); + } + + /** + * Ingest one event envelope through the dedup / rate-limit / bot-self + * pipeline. The endpoint is the operator-facing surface — workspace + * is taken from the trusted {@code X-Workspace-Id} header. Body + * {@code workspaceId} is intentionally ignored so a caller in + * workspace A can't fan-fire triggers in workspace B by hand-rolling + * a JSON body. + * + *

External webhooks should NOT use this endpoint directly — + * production deployments wire their own signed-token webhook + * (e.g. Feishu / DingTalk adapters) which authenticates first and + * publishes a {@link vip.mate.channel.event.ChannelMessageReceivedEvent} + * with a workspace fixed by the channel-token mapping. The + * {@code ChannelMessageEventBridge} then forwards into ingest. + */ + @Operation(summary = "Ingest one event envelope; returns per-trigger fire / drop summary.") + @PostMapping("/events") + public R> ingestEvent( + @RequestBody EventIngestRequest body, + @RequestHeader("X-Workspace-Id") long workspaceId) { + TriggerEventEnvelope env = new TriggerEventEnvelope( + // Header wins — body.workspaceId is dropped on purpose. + workspaceId, + body.patternType(), + body.eventId(), + body.senderId(), + body.data() == null ? Map.of() : body.data()); + return R.ok(ingestService.ingest(env)); + } + + /** {@code workspaceId} is retained on the request shape for backwards + * compatibility but ignored at the controller — the trusted header + * is the source of truth. */ + public record EventIngestRequest( + long workspaceId, + String patternType, + String eventId, + String senderId, + Map data) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/AgentLifecycleEventBridge.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/AgentLifecycleEventBridge.java new file mode 100644 index 00000000..becbad48 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/AgentLifecycleEventBridge.java @@ -0,0 +1,55 @@ +package vip.mate.trigger.dispatch; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import vip.mate.agent.event.AgentLifecycleEvent; +import vip.mate.trigger.ingest.TriggerEventEnvelope; +import vip.mate.trigger.ingest.TriggerEventIngestService; + +import java.util.HashMap; +import java.util.Map; + +/** + * Forwards {@link AgentLifecycleEvent} into the trigger pipeline as + * {@code agent_lifecycle} envelopes. Lives in the trigger module so the + * agent runtime stays free of trigger / ingest dependencies, matching + * the workflow_completion + channel_message bridge pattern. + * + *

The dedup key composes phase + agentId + timestamp so the same + * agent flipping enabled / disabled repeatedly stays observable, but + * an at-least-once retry of the same exact lifecycle event collapses. + * Failures inside ingest are logged and swallowed. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class AgentLifecycleEventBridge { + + private final TriggerEventIngestService ingestService; + + @EventListener + public void onLifecycle(AgentLifecycleEvent event) { + if (event == null) return; + try { + Map data = new HashMap<>(); + // The matcher reads `agentId` and `phase` out of the envelope + // data; the field names mirror the matcher's vocabulary so + // pattern_json can narrow precisely. + data.put("agentId", event.agentId()); + if (event.agentName() != null) data.put("agentName", event.agentName()); + data.put("phase", event.phase()); + data.put("timestamp", event.timestamp()); + ingestService.ingest(new TriggerEventEnvelope( + event.workspaceId(), + "agent_lifecycle", + event.phase() + ":" + event.agentId() + ":" + event.timestamp(), + "system", + data)); + } catch (Exception e) { + log.warn("[AgentLifecycleBridge] forwarding agent {} phase={} failed: {}", + event.agentId(), event.phase(), e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/ChannelMessageEventBridge.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/ChannelMessageEventBridge.java new file mode 100644 index 00000000..bd868192 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/ChannelMessageEventBridge.java @@ -0,0 +1,70 @@ +package vip.mate.trigger.dispatch; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import vip.mate.channel.event.ChannelMessageReceivedEvent; +import vip.mate.trigger.ingest.TriggerEventEnvelope; +import vip.mate.trigger.ingest.TriggerEventIngestService; + +import java.util.HashMap; +import java.util.Map; + +/** + * Bridges {@link ChannelMessageReceivedEvent} from the channel module + * into two trigger pattern types — {@code channel_message} (matches by + * {@code channelType} / {@code senderEquals}) and {@code content_match} + * (matches by substring inside the message content). The same envelope + * fans out to both since the matcher's per-pattern key on the SQL + * candidate query selects which triggers actually run. + * + *

Lives in the trigger module so the channel runtime stays free of + * trigger / ingest dependencies. Failures inside ingest are logged and + * swallowed — a bad downstream trigger MUST NOT corrupt the primary + * chat-routing path that just published the event. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ChannelMessageEventBridge { + + private final TriggerEventIngestService ingestService; + + @EventListener + public void onChannelMessage(ChannelMessageReceivedEvent event) { + if (event == null) return; + try { + Map data = new HashMap<>(); + data.put("channelType", event.channelType()); + data.put("senderId", event.senderId()); + if (event.senderName() != null) data.put("senderName", event.senderName()); + if (event.chatId() != null) data.put("chatId", event.chatId()); + // The matcher's content_match pattern reads `data.content`, + // so we put the message body there even when it's blank. + data.put("content", event.content() == null ? "" : event.content()); + + // Fan to channel_message pattern triggers. + ingestService.ingest(new TriggerEventEnvelope( + event.workspaceId(), + "channel_message", + event.messageId(), + event.senderId(), + data)); + // And to content_match triggers, which live under a different + // patternType but read the same envelope shape. Two separate + // ingests instead of one because the SQL candidate query + // filters on patternType — a single dispatch with one + // patternType cannot reach the other set. + ingestService.ingest(new TriggerEventEnvelope( + event.workspaceId(), + "content_match", + event.messageId(), + event.senderId(), + data)); + } catch (Exception e) { + log.warn("[ChannelMessageBridge] forwarding message {} from {} failed: {}", + event.messageId(), event.senderId(), e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DefaultWorkflowGraphLoader.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DefaultWorkflowGraphLoader.java new file mode 100644 index 00000000..bbb18411 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DefaultWorkflowGraphLoader.java @@ -0,0 +1,77 @@ +package vip.mate.trigger.dispatch; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.WorkflowParser; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.model.WorkflowRevisionEntity; +import vip.mate.workflow.repository.WorkflowMapper; +import vip.mate.workflow.repository.WorkflowRevisionMapper; + +/** + * Production binding for {@link WorkflowGraphLoader}. Looks up + * {@code mate_workflow.latest_revision_id} and parses the corresponding + * {@code mate_workflow_revision.graph_json}. Returns + * {@link Loaded#missing()} when either lookup fails or the workflow is + * disabled — triggers should not fire workflows that the user already + * paused or removed. + */ +@Slf4j +@Component +public class DefaultWorkflowGraphLoader implements WorkflowGraphLoader { + + private final WorkflowMapper workflowMapper; + private final WorkflowRevisionMapper revisionMapper; + private final WorkflowParser parser; + + public DefaultWorkflowGraphLoader(WorkflowMapper workflowMapper, + WorkflowRevisionMapper revisionMapper, + WorkflowParser parser) { + this.workflowMapper = workflowMapper; + this.revisionMapper = revisionMapper; + this.parser = parser; + } + + @Override + public Loaded load(long workflowId, long workspaceId) { + WorkflowEntity workflow = workflowMapper.selectById(workflowId); + if (workflow == null || Boolean.FALSE.equals(workflow.getEnabled()) + || workflow.getLatestRevisionId() == null) { + return Loaded.missing(); + } + // Workspace ownership check — the trigger must live in the same + // workspace as the workflow. Without this gate, fixture data / + // manual imports / a service-bypass code path could let a + // workspace A trigger fire a workspace B workflow. + if (workflow.getWorkspaceId() == null || workflow.getWorkspaceId() != workspaceId) { + log.warn("Trigger graph load: workflow {} is in workspace {}, caller asked for {}", + workflowId, workflow.getWorkspaceId(), workspaceId); + return Loaded.missing(); + } + WorkflowRevisionEntity revision = revisionMapper.selectById(workflow.getLatestRevisionId()); + if (revision == null) return Loaded.missing(); + try { + return new Loaded(parser.parse(revision.getGraphJson()), revision.getId()); + } catch (Exception e) { + log.warn("Trigger graph load: revision {} failed to parse: {}", + revision.getId(), e.getMessage()); + return Loaded.missing(); + } + } + + /** + * @deprecated production callers MUST use the workspace-scoped overload + * {@link #load(long, long)}. Kept available for legacy test + * stubs that bind a fake workspace context. Returns + * {@code missing()} unconditionally so a production code + * path that accidentally hits this overload doesn't silently + * cross workspaces. + */ + @Override + @Deprecated + public Loaded load(long workflowId) { + log.warn("Workspace-blind WorkflowGraphLoader.load({}) called — refusing. " + + "Use load(workflowId, workspaceId) instead.", workflowId); + return Loaded.missing(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DispatchResult.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DispatchResult.java new file mode 100644 index 00000000..8367ac0a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/DispatchResult.java @@ -0,0 +1,44 @@ +package vip.mate.trigger.dispatch; + +/** + * Outcome of a trigger fire. The dispatcher used to return either a + * {@code WorkflowRunResult} or {@code null}, which led the ingest and + * scheduler paths to treat null as "fired" — incrementing + * {@code fireCount} / {@code lastFiredAt} even when the dispatch was + * a no-op or an error. This record makes the outcome explicit so each + * caller can update bookkeeping honestly. + * + *

    + *
  • {@link Kind#FIRED} — a workflow run row was actually created. + * {@link #runId()} carries its id; {@link #reason()} is null.
  • + *
  • {@link Kind#SKIPPED} — pre-flight rejected the dispatch + * (no published revision, unsupported target type, payload render + * failed). {@link #reason()} carries the human-readable cause; + * {@link #runId()} is null.
  • + *
  • {@link Kind#FAILED} — runner threw / persisted with an error + * state. {@link #reason()} is the failure message; {@link #runId()} + * may be set if a row was created before the failure.
  • + *
+ */ +public record DispatchResult(Kind kind, Long runId, String reason) { + + public enum Kind { FIRED, SKIPPED, FAILED } + + public boolean fired() { return kind == Kind.FIRED; } + + public static DispatchResult fired(Long runId) { + return new DispatchResult(Kind.FIRED, runId, null); + } + + public static DispatchResult skipped(String reason) { + return new DispatchResult(Kind.SKIPPED, null, reason); + } + + public static DispatchResult failed(String message) { + return new DispatchResult(Kind.FAILED, null, message); + } + + public static DispatchResult failed(Long runId, String message) { + return new DispatchResult(Kind.FAILED, runId, message); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/TriggerDispatcher.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/TriggerDispatcher.java new file mode 100644 index 00000000..473a2450 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/TriggerDispatcher.java @@ -0,0 +1,141 @@ +package vip.mate.trigger.dispatch; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.workflow.compiler.PebbleSubsetEvaluator; +import vip.mate.workflow.runtime.WorkflowRunRequest; +import vip.mate.workflow.runtime.WorkflowRunResult; +import vip.mate.workflow.runtime.WorkflowRunner; + +import java.util.Map; + +/** + * Translates a fired trigger into a workflow run. Renders the trigger's + * {@code payloadTemplate} as JSON via Pebble, parses the result into the + * input map, and asks the runner to execute the latest revision of the + * target workflow. Logs and swallows failures so a bad trigger never takes + * the scheduler thread down. + */ +@Slf4j +@Component +public class TriggerDispatcher { + + private static final TypeReference> MAP_REF = new TypeReference<>() {}; + + private final WorkflowGraphLoader graphLoader; + private final WorkflowRunner runner; + private final PebbleSubsetEvaluator pebble; + private final ObjectMapper objectMapper; + + public TriggerDispatcher(WorkflowGraphLoader graphLoader, + WorkflowRunner runner, + PebbleSubsetEvaluator pebble, + ObjectMapper objectMapper) { + this.graphLoader = graphLoader; + this.runner = runner; + this.pebble = pebble; + this.objectMapper = objectMapper; + } + + /** + * Dispatch a single fire of {@code trigger}. {@code event} is the + * source-event context (cron tick metadata, channel message, etc.) — + * its top-level fields are exposed to the payload template under + * {@code event.*}. Returns a {@link DispatchResult} so the caller + * can distinguish a real fire from a pre-flight skip or a runner + * failure and update {@code fireCount} / {@code lastFiredAt} / + * {@code lastError} accordingly. + */ + public DispatchResult dispatch(TriggerEntity trigger, Map event) { + if (!"workflow".equalsIgnoreCase(trigger.getTargetType())) { + log.warn("Trigger {} target_type {} not supported in v0; skipping fire", + trigger.getId(), trigger.getTargetType()); + return DispatchResult.skipped( + "unsupported target_type: " + trigger.getTargetType()); + } + // Workspace-scoped lookup so a workspace A trigger can never fire + // a workspace B workflow even if fixture data / manual imports / + // a service-bypass path somehow planted a cross-workspace + // targetId. The loader returns missing() on mismatch. + long workspaceId = trigger.getWorkspaceId() == null ? 0L : trigger.getWorkspaceId(); + WorkflowGraphLoader.Loaded loaded = graphLoader.load(trigger.getTargetId(), workspaceId); + if (loaded.graph() == null) { + log.info("Trigger {} dispatch skipped: no published revision for workflow {} in workspace {}", + trigger.getId(), trigger.getTargetId(), workspaceId); + return DispatchResult.skipped( + "no published revision for workflow " + trigger.getTargetId()); + } + + Map inputs; + try { + inputs = renderInputs(trigger, event); + } catch (Exception e) { + return DispatchResult.failed("payload render failed: " + e.getMessage()); + } + WorkflowRunRequest req = new WorkflowRunRequest( + trigger.getTargetId(), + loaded.revisionId(), + trigger.getWorkspaceId(), + "trigger:" + trigger.getId(), + inputs); + try { + WorkflowRunResult result = runner.run(loaded.graph(), req); + if (result == null) { + return DispatchResult.failed("runner returned null result"); + } + // The runner's state taxonomy: succeeded / paused / running / + // failed. Anything other than failed counts as a real fire — a + // paused run still consumed the trigger and produced a + // workflow_run row that the operator can resume. + if ("failed".equalsIgnoreCase(result.state())) { + return DispatchResult.failed(result.runId(), + "workflow run failed: " + + (result.errorMessage() == null ? "(no message)" : result.errorMessage())); + } + return DispatchResult.fired(result.runId()); + } catch (Exception e) { + log.error("Trigger {} dispatch failed for workflow {}: {}", + trigger.getId(), trigger.getTargetId(), e.getMessage(), e); + return DispatchResult.failed("runner threw: " + e.getMessage()); + } + } + + /** + * Render the trigger's payload template into the workflow's input map. + * + *

Failure mode is strict. If the template fails to parse, + * fails to render, or produces output that isn't a JSON object, this + * method throws and {@link #dispatch} returns + * {@link DispatchResult#failed(String)} so the trigger row records a + * non-null {@code last_error} and the operator can see why this fire + * didn't run. The previous "fall back to raw event" behaviour is the + * exact silent-failure trap the design forbade — a typo'd template + * would keep firing the workflow with the wrong inputs and lastError + * would stay clean. + * + *

An empty / null {@code payloadTemplate} is the explicit + * opt-in to "use the raw event as inputs" — that path stays + * supported because it's intentional, not accidental. + */ + private Map renderInputs(TriggerEntity trigger, Map event) { + if (trigger.getPayloadTemplate() == null || trigger.getPayloadTemplate().isBlank()) { + return event == null ? Map.of() : event; + } + var compiled = pebble.parseTemplate(trigger.getPayloadTemplate()); + String rendered = pebble.evaluateAsString(compiled, + Map.of("event", event == null ? Map.of() : event, + "trigger", Map.of( + "id", trigger.getId(), + "name", trigger.getName() == null ? "" : trigger.getName()))); + try { + return objectMapper.readValue(rendered, MAP_REF); + } catch (Exception e) { + // Wrap so the dispatcher's catch surfaces the JSON parse failure + // distinctly from a Pebble parse / evaluate failure. + throw new RuntimeException("payloadTemplate produced non-JSON output: " + e.getMessage(), e); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowCompletionEventBridge.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowCompletionEventBridge.java new file mode 100644 index 00000000..7aa0e69d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowCompletionEventBridge.java @@ -0,0 +1,63 @@ +package vip.mate.trigger.dispatch; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import vip.mate.trigger.ingest.TriggerEventEnvelope; +import vip.mate.trigger.ingest.TriggerEventIngestService; +import vip.mate.workflow.runtime.WorkflowCompletionEvent; + +import java.util.HashMap; +import java.util.Map; + +/** + * Bridges {@link WorkflowCompletionEvent} from the workflow module into the + * trigger ingest pipeline. Lives in the trigger module so the workflow + * runtime stays free of trigger / ingest dependencies — that's how we + * dodge the Runner ↔ Dispatcher ↔ Ingest ↔ Runner cycle Spring would + * otherwise refuse to construct. + * + *

Each terminal-state run is translated into a {@code workflow_completion} + * envelope with a deterministic {@code wf-run-{runId}} eventId, so the + * {@code mate_trigger_event} unique constraint dedups any re-publish + * (e.g. a runner crash + retry). Failures inside the ingest pipeline are + * logged and swallowed — a bad downstream trigger MUST NOT corrupt the + * just-completed run. + * + *

The listener fires in the runner thread by default; if a downstream + * ingest does heavy work, switch to {@code @Async} once a dedicated + * executor is wired. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class WorkflowCompletionEventBridge { + + private final TriggerEventIngestService ingestService; + + @EventListener + public void onCompletion(WorkflowCompletionEvent event) { + if (event == null) return; + try { + Map data = new HashMap<>(); + data.put("sourceWorkflowId", event.workflowId()); + data.put("revisionId", event.revisionId()); + data.put("runId", event.runId()); + data.put("state", event.state()); + if (event.finalOutputRef() != null) data.put("finalOutputRef", event.finalOutputRef()); + if (event.errorMessage() != null) data.put("errorMessage", event.errorMessage()); + TriggerEventEnvelope envelope = new TriggerEventEnvelope( + event.workspaceId(), + "workflow_completion", + "wf-run-" + event.runId(), + "system", + data); + ingestService.ingest(envelope); + } catch (Exception e) { + log.warn("[WorkflowCompletionBridge] forwarding run {} completion failed: {}", + event.runId(), e.getMessage()); + } + } + +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowGraphLoader.java b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowGraphLoader.java new file mode 100644 index 00000000..7de8e9bb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/dispatch/WorkflowGraphLoader.java @@ -0,0 +1,45 @@ +package vip.mate.trigger.dispatch; + +import vip.mate.workflow.compiler.ir.WorkflowGraph; + +/** + * SPI for "given a workflow id, load the published WorkflowGraph the trigger + * should fire". Production binding reads {@code mate_workflow.latest_revision_id} + * and parses {@code mate_workflow_revision.graph_json}; tests stub this so a + * fire path can be exercised without standing up the publish pipeline. + */ +public interface WorkflowGraphLoader { + + /** + * Result of a graph load. {@code graph == null} indicates the workflow + * has no published revision yet (or was deleted) and the fire should + * be skipped instead of erroring. + */ + record Loaded(WorkflowGraph graph, Long revisionId) { + public static Loaded missing() { return new Loaded(null, null); } + } + + /** + * Workspace-scoped lookup. Production callers MUST use this overload + * so a trigger in workspace A can never resolve to a workflow in + * workspace B (e.g. via fixture data, manual DB import, or a + * service-bypass code path). The default binding validates that + * {@code mate_workflow.workspace_id == workspaceId}; tests that don't + * care override this to delegate to the workspace-blind overload. + */ + default Loaded load(long workflowId, long workspaceId) { + // Default: fall through to the single-arg lookup. The production + // {@link DefaultWorkflowGraphLoader} overrides this to enforce + // ownership; test stubs that don't care inherit the lenient + // default. + return load(workflowId); + } + + /** + * @deprecated workspace-blind lookup; only kept for legacy test stubs + * and the deprecated path inside the production binding. + * New callers must use {@link #load(long, long)}. + */ + @Deprecated + Loaded load(long workflowId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/BotSelfFilter.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/BotSelfFilter.java new file mode 100644 index 00000000..c7a254fc --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/BotSelfFilter.java @@ -0,0 +1,22 @@ +package vip.mate.trigger.ingest; + +/** + * Drops events whose sender id matches a registered bot identity for the + * workspace. The intent: MateClaw's own outbound channel messages would + * otherwise loop back through the channel webhook, fire a trigger, and + * dispatch a fresh workflow run — a recipe for a runaway echo loop on any + * channel where the bot account can read its own posts. + * + *

v0 keeps the bot identity registry in-memory; production will likely + * wire this to {@code mate_channel.bot_identity} once that schema lands. + * The interface lets tests inject a deterministic resolver. + */ +public interface BotSelfFilter { + + /** + * Whether {@code senderId} matches a known bot identity in + * {@code workspaceId}. Returning {@code true} causes the ingest pipeline + * to drop the event silently. + */ + boolean isBotSelf(long workspaceId, String senderId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/NoopBotSelfFilter.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/NoopBotSelfFilter.java new file mode 100644 index 00000000..a73cb0eb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/NoopBotSelfFilter.java @@ -0,0 +1,18 @@ +package vip.mate.trigger.ingest; + +import org.springframework.stereotype.Component; + +/** + * Default {@link BotSelfFilter} binding — never identifies a sender as a + * bot. Acts as the v0 placeholder until the channel-side bot identity + * registry is wired through; channels that already know their own bot id + * may also call the filter directly to skip ingest before it begins. + */ +@Component +public class NoopBotSelfFilter implements BotSelfFilter { + + @Override + public boolean isBotSelf(long workspaceId, String senderId) { + return false; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventEnvelope.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventEnvelope.java new file mode 100644 index 00000000..d5ed478a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventEnvelope.java @@ -0,0 +1,33 @@ +package vip.mate.trigger.ingest; + +import java.util.Map; + +/** + * Generic event envelope used by upstream sources (channel webhooks, + * agent-lifecycle hooks, workflow-completion hooks, ad-hoc REST callers) + * to feed the trigger pipeline. The pipeline owns dedup / rate-limit / + * bot-self filtering; sources only need to fill this record: + * + *

    + *
  • {@code workspaceId} — scopes which triggers can fire on this event.
  • + *
  • {@code patternType} — matched against {@code mate_trigger.pattern_type}; + * the ingest looks up only triggers whose pattern type equals this.
  • + *
  • {@code eventId} — stable upstream identifier used as the dedup key + * when present; the ingest falls back to a content hash when blank.
  • + *
  • {@code senderId} — the upstream actor; used by the bot-self filter + * to drop events that originate from MateClaw's own outbound traffic.
  • + *
  • {@code data} — free-form payload exposed to the trigger's payload + * template under {@code event.*}.
  • + *
+ */ +public record TriggerEventEnvelope( + long workspaceId, + String patternType, + String eventId, + String senderId, + Map data +) { + public TriggerEventEnvelope { + data = data == null ? Map.of() : Map.copyOf(data); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventIngestService.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventIngestService.java new file mode 100644 index 00000000..e5ab9bf9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerEventIngestService.java @@ -0,0 +1,338 @@ +package vip.mate.trigger.ingest; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import vip.mate.trigger.dispatch.DispatchResult; +import vip.mate.trigger.dispatch.TriggerDispatcher; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.model.TriggerEventEntity; +import vip.mate.trigger.repository.TriggerEventMapper; +import vip.mate.trigger.repository.TriggerMapper; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; + +/** + * Single ingress for every event-driven trigger. Runs the four-stage filter + * the design committee picked for v0: + * + *
    + *
  1. Look up enabled triggers in the workspace whose {@code patternType} + * matches the envelope. Triggers in disabled workspaces, soft-deleted + * triggers, and triggers exhausted on {@code max_fires} are skipped.
  2. + *
  3. Bot-self filter — drop events whose sender matches a registered + * bot identity, even if the trigger config has it disabled, because + * a runaway echo from our own outbound traffic is the worst-case + * failure and not worth a per-trigger opt-out.
  4. + *
  5. Dedup window — insert a {@code mate_trigger_event} row keyed on + * {@code (trigger_id, dedup_key)} where the dedup key is the envelope + * eventId or a SHA-256 of the payload data when the upstream channel + * did not provide a stable id. A duplicate-key error short-circuits + * the dispatch silently.
  6. + *
  7. Sliding-window rate limit — per-trigger 60s cap; an over-cap event + * is logged and dropped without dispatching.
  8. + *
+ * + *

Each accepted event is then handed to {@link TriggerDispatcher} which + * runs the workflow synchronously. v0 does not queue dispatches; if the + * sender's webhook holds the connection open, the trigger runs in the + * caller's thread. + */ +@Slf4j +@Service +public class TriggerEventIngestService { + + private final TriggerMapper triggerMapper; + private final TriggerEventMapper eventMapper; + private final TriggerDispatcher dispatcher; + private final BotSelfFilter botSelfFilter; + private final ObjectMapper objectMapper; + private final TriggerPatternMatcher patternMatcher; + private final TriggerRateLimiter rateLimiter = new TriggerRateLimiter(); + + /** When true (production default), {@code dispatcher.dispatch} runs on + * a worker thread so the caller (webhook / scheduler / runner) returns + * quickly. When false, ingest runs the workflow inline on the caller + * thread; tests pin to false so they can assert against downstream + * workflow state immediately after {@code ingest()} returns. */ + @Value("${mateclaw.workflow.trigger.async-dispatch:true}") + private boolean asyncDispatch; + + @Value("${mateclaw.workflow.trigger.dispatch-pool-size:8}") + private int dispatchPoolSize; + + @Value("${mateclaw.workflow.trigger.dispatch-queue-capacity:256}") + private int dispatchQueueCapacity; + + /** Lazy-built bounded thread pool used when {@link #asyncDispatch} is + * true. CallerRunsPolicy is the back-pressure: when the queue is full + * the calling thread runs the dispatch itself, which guarantees no + * silent drop while still capping in-flight work. */ + private volatile java.util.concurrent.ThreadPoolExecutor dispatchExecutor; + + public TriggerEventIngestService(TriggerMapper triggerMapper, + TriggerEventMapper eventMapper, + TriggerDispatcher dispatcher, + BotSelfFilter botSelfFilter, + ObjectMapper objectMapper, + TriggerPatternMatcher patternMatcher) { + this.triggerMapper = triggerMapper; + this.eventMapper = eventMapper; + this.dispatcher = dispatcher; + this.botSelfFilter = botSelfFilter; + this.objectMapper = objectMapper; + this.patternMatcher = patternMatcher; + } + + private java.util.concurrent.ThreadPoolExecutor dispatchExecutor() { + java.util.concurrent.ThreadPoolExecutor local = dispatchExecutor; + if (local != null) return local; + synchronized (this) { + if (dispatchExecutor == null) { + int size = Math.max(1, dispatchPoolSize); + int cap = Math.max(1, dispatchQueueCapacity); + dispatchExecutor = new java.util.concurrent.ThreadPoolExecutor( + size, size, + 60L, java.util.concurrent.TimeUnit.SECONDS, + new java.util.concurrent.LinkedBlockingQueue<>(cap), + r -> { + Thread t = new Thread(r, "trigger-dispatch-" + System.currentTimeMillis()); + t.setDaemon(true); + return t; + }, + new java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy()); + } + return dispatchExecutor; + } + } + + @PreDestroy + void shutdownDispatchExecutor() { + java.util.concurrent.ThreadPoolExecutor local = dispatchExecutor; + if (local != null) { + local.shutdown(); + try { + if (!local.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)) { + local.shutdownNow(); + } + } catch (InterruptedException e) { + local.shutdownNow(); + Thread.currentThread().interrupt(); + } + } + } + + /** + * Process one envelope through the pipeline. Returns a result per + * candidate trigger so callers can surface a partial-accept summary. + */ + public List ingest(TriggerEventEnvelope envelope) { + if (envelope.patternType() == null || envelope.patternType().isBlank()) { + return List.of(); + } + List candidates = triggerMapper.selectList(new LambdaQueryWrapper() + .eq(TriggerEntity::getWorkspaceId, envelope.workspaceId()) + .eq(TriggerEntity::getPatternType, envelope.patternType()) + .eq(TriggerEntity::getEnabled, true) + .eq(TriggerEntity::getDeleted, 0)); + if (candidates.isEmpty()) return List.of(); + + List results = new ArrayList<>(candidates.size()); + for (TriggerEntity trigger : candidates) { + results.add(processSingle(trigger, envelope)); + } + return results; + } + + private IngestResult processSingle(TriggerEntity trigger, TriggerEventEnvelope envelope) { + // Pattern matching is the first gate — without it, every channel + // event would broadcast to every channel-message trigger in the + // workspace, which is exactly the storm hazard the design forbade. + // Run it before all the other filters so a non-matching trigger + // doesn't even allocate a dedup row. + if (!patternMatcher.matches(trigger, envelope)) { + return IngestResult.dropped(trigger.getId(), Reason.PATTERN_MISMATCH); + } + if (Boolean.TRUE.equals(trigger.getBotSelfFilter()) + && botSelfFilter.isBotSelf(envelope.workspaceId(), envelope.senderId())) { + return IngestResult.dropped(trigger.getId(), Reason.BOT_SELF); + } + if (trigger.getMaxFires() != null && trigger.getMaxFires() > 0 + && trigger.getFireCount() != null && trigger.getFireCount() >= trigger.getMaxFires()) { + return IngestResult.dropped(trigger.getId(), Reason.EXHAUSTED); + } + if (!recordDedupRow(trigger, envelope)) { + return IngestResult.dropped(trigger.getId(), Reason.DUPLICATE); + } + int limit = trigger.getRateLimitPerMin() == null ? 0 : trigger.getRateLimitPerMin(); + if (!rateLimiter.tryAcquire(trigger.getId(), limit, Instant.now())) { + return IngestResult.dropped(trigger.getId(), Reason.RATE_LIMITED); + } + if (asyncDispatch) { + // Async path — submit dispatch to the bounded pool so the + // caller (webhook / scheduler / runner thread) returns + // quickly. Bookkeeping happens inside the worker, so + // last_error / fireCount stay accurate. The IngestResult + // signals "accepted, fanning out" rather than "ran to + // completion"; that's the honest contract for an async + // pipeline. CallerRunsPolicy on the executor means we + // self-throttle instead of dropping under back-pressure. + try { + dispatchExecutor().execute(() -> runDispatchAndPersist(trigger, envelope)); + } catch (Exception e) { + log.error("Trigger {} dispatch submit failed: {}", + trigger.getId(), e.getMessage(), e); + persistDispatchOutcome(trigger, + DispatchResult.failed("dispatch submit failed: " + e.getMessage())); + return IngestResult.dropped(trigger.getId(), Reason.DISPATCH_ERROR); + } + return IngestResult.fired(trigger.getId()); + } + // Synchronous path — used by tests and any deployment that + // explicitly opts out via mateclaw.workflow.trigger.async-dispatch=false. + DispatchResult outcome = runDispatchAndPersist(trigger, envelope); + return switch (outcome.kind()) { + case FIRED -> IngestResult.fired(trigger.getId()); + case SKIPPED -> IngestResult.dropped(trigger.getId(), Reason.DISPATCH_SKIPPED); + case FAILED -> IngestResult.dropped(trigger.getId(), Reason.DISPATCH_ERROR); + }; + } + + /** Runs dispatch + bookkeeping on whatever thread invokes it (the + * caller in sync mode, a worker in async mode). Returns the + * outcome so sync callers can map it back to an IngestResult. */ + private DispatchResult runDispatchAndPersist(TriggerEntity trigger, TriggerEventEnvelope envelope) { + DispatchResult outcome; + try { + outcome = dispatcher.dispatch(trigger, envelope.data()); + } catch (Exception e) { + log.error("Trigger {} dispatch threw on event ingest: {}", + trigger.getId(), e.getMessage(), e); + outcome = DispatchResult.failed("dispatch threw: " + e.getMessage()); + } + persistDispatchOutcome(trigger, outcome); + return outcome; + } + + /** + * Update the trigger row's bookkeeping based on the dispatch outcome. + * Only FIRED bumps {@code fireCount} and {@code lastFiredAt} — SKIPPED + * and FAILED outcomes were treated as fires before, which made the + * stats lie. {@code lastDispatchedAt} stamps every attempt so the UI + * can distinguish "never attempted" from "attempted but skipped". + */ + private void persistDispatchOutcome(TriggerEntity trigger, DispatchResult outcome) { + try { + LocalDateTime now = LocalDateTime.now(); + trigger.setLastDispatchedAt(now); + if (outcome.fired()) { + trigger.setFireCount( + (trigger.getFireCount() == null ? 0L : trigger.getFireCount()) + 1); + trigger.setLastFiredAt(now); + trigger.setLastError(null); + } else { + trigger.setLastError(outcome.reason()); + } + triggerMapper.updateById(trigger); + } catch (Exception e) { + // Best-effort bookkeeping — never let a stats write fail ingest. + log.warn("Trigger {} bookkeeping update failed: {}", trigger.getId(), e.getMessage()); + } + } + + private boolean recordDedupRow(TriggerEntity trigger, TriggerEventEnvelope envelope) { + TriggerEventEntity row = new TriggerEventEntity(); + row.setTriggerId(trigger.getId()); + row.setDedupKey(resolveDedupKey(envelope)); + int windowSecs = trigger.getDedupWindowSecs() == null ? 60 : trigger.getDedupWindowSecs(); + Instant now = Instant.now(); + row.setReceivedAt(LocalDateTime.ofInstant(now, ZoneOffset.systemDefault())); + row.setExpiresAt(LocalDateTime.ofInstant(now.plusSeconds(windowSecs), + ZoneOffset.systemDefault())); + try { + eventMapper.insert(row); + return true; + } catch (DuplicateKeyException e) { + // Within the dedup window — silently drop. + return false; + } + } + + private String resolveDedupKey(TriggerEventEnvelope envelope) { + if (envelope.eventId() != null && !envelope.eventId().isBlank()) { + return truncate(envelope.eventId()); + } + try { + byte[] body = objectMapper.writeValueAsBytes(envelope.data()); + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return "sha256:" + HexFormat.of().formatHex(digest.digest(body)); + } catch (Exception e) { + // Fall back to a per-call random so we never hard-fail ingest. + return "rand:" + java.util.UUID.randomUUID(); + } + } + + private static String truncate(String s) { + if (s == null) return null; + // Column is VARCHAR(128) — keep some headroom for trigger-prefixed keys. + return s.length() <= 120 ? s : s.substring(0, 120); + } + + /** Cleanup tick for expired dedup rows. Run from a scheduler in production. */ + public int sweepExpired() { + return eventMapper.delete(new LambdaQueryWrapper() + .lt(TriggerEventEntity::getExpiresAt, + LocalDateTime.ofInstant(Instant.now(), ZoneOffset.systemDefault()))); + } + + /** + * Periodic sweep of expired {@code mate_trigger_event} dedup rows. + * Default cadence is every 5 minutes, tunable via + * {@code mateclaw.workflow.trigger.dedup-sweep-interval-ms}. The + * initial delay matches the cadence so a JVM that just started doesn't + * race {@code recordDedupRow} for the same window. + */ + @Scheduled( + fixedDelayString = "${mateclaw.workflow.trigger.dedup-sweep-interval-ms:300000}", + initialDelayString = "${mateclaw.workflow.trigger.dedup-sweep-initial-delay-ms:300000}") + public void scheduledSweepExpired() { + try { + int dropped = sweepExpired(); + if (dropped > 0) { + log.info("[TriggerIngest] swept {} expired dedup rows", dropped); + } + } catch (Exception e) { + // Best-effort — never let the sweep crash the scheduler thread. + log.warn("[TriggerIngest] dedup sweep failed: {}", e.getMessage()); + } + } + + public enum Reason { + PATTERN_MISMATCH, BOT_SELF, DUPLICATE, RATE_LIMITED, EXHAUSTED, + /** Dispatcher returned SKIPPED — pre-flight rejected (no published revision, etc.). */ + DISPATCH_SKIPPED, + /** Dispatcher returned FAILED — runner threw or workflow run ended in failed state. */ + DISPATCH_ERROR + } + + public record IngestResult(long triggerId, boolean fired, Reason droppedReason) { + public static IngestResult fired(long triggerId) { + return new IngestResult(triggerId, true, null); + } + public static IngestResult dropped(long triggerId, Reason r) { + return new IngestResult(triggerId, false, r); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerPatternMatcher.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerPatternMatcher.java new file mode 100644 index 00000000..59fc102a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerPatternMatcher.java @@ -0,0 +1,188 @@ +package vip.mate.trigger.ingest; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.trigger.model.TriggerEntity; + +import java.util.Map; + +/** + * Decides whether a trigger's stored {@code pattern_json} actually matches + * an inbound envelope, beyond the coarse {@code (workspaceId, patternType)} + * filter the SQL query already does. + * + *

Without this layer the ingest service broadcasts every event to every + * trigger in the same workspace that happens to share a {@code patternType}, + * which is the event-storm hazard the design has warned about — one channel + * message would fire every channel-message trigger regardless of intent. + * + *

v0 supports four pattern shapes: + *

    + *
  • cron — never matches an inbound envelope. Cron triggers run + * through the scheduler, not the ingest pipeline.
  • + *
  • channel_message — optional {@code channelType} narrows by + * which adapter the envelope came from; optional {@code senderEquals} + * narrows to a specific sender id.
  • + *
  • agent_lifecycle — optional {@code agentId} narrows to a + * specific agent's lifecycle events; optional {@code phase} narrows + * to {@code spawned} / {@code terminated} / {@code crashed}.
  • + *
  • content_match — required {@code substring} must appear in + * the envelope's {@code data.content} field (case-insensitive); this + * is the explicit pattern that the design always intended to require + * payload-level evaluation.
  • + *
  • workflow_completion — optional {@code sourceWorkflowId} + * narrows to a specific upstream workflow; optional {@code stateFilter} + * narrows to {@code completed} / {@code failed} / {@code any}.
  • + *
  • webhook — opaque pass-through. v0 doesn't filter further.
  • + *
+ * + *

Unknown pattern types fail closed (no match) so a typo'd or future + * pattern type can't silently fire every workspace trigger. + */ +@Slf4j +@Component +public class TriggerPatternMatcher { + + private final ObjectMapper objectMapper; + + public TriggerPatternMatcher(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public boolean matches(TriggerEntity trigger, TriggerEventEnvelope envelope) { + String type = trigger.getPatternType(); + if (type == null) return false; + JsonNode pattern = parsePattern(trigger); + return switch (type) { + case "cron" -> false; // scheduler-driven, not ingested + case "channel_message" -> matchesChannelMessage(pattern, envelope); + case "agent_lifecycle" -> matchesAgentLifecycle(pattern, envelope); + case "content_match" -> matchesContent(pattern, envelope); + case "workflow_completion" -> matchesWorkflowCompletion(pattern, envelope); + case "webhook" -> true; // pass-through; secret check happens at the HTTP boundary + default -> { + log.warn("Trigger {} uses unknown patternType '{}' — failing closed", + trigger.getId(), type); + yield false; + } + }; + } + + private JsonNode parsePattern(TriggerEntity trigger) { + String json = trigger.getPatternJson(); + if (json == null || json.isBlank()) return objectMapper.nullNode(); + try { + return objectMapper.readTree(json); + } catch (Exception e) { + // A trigger with malformed pattern_json should never have been + // accepted at create / update time; fail closed at fire time. + log.warn("Trigger {} pattern_json parse failed: {}", trigger.getId(), e.getMessage()); + return objectMapper.nullNode(); + } + } + + private boolean matchesChannelMessage(JsonNode pattern, TriggerEventEnvelope envelope) { + if (envelope == null) return false; + // channelType lives in envelope.data ("channelType" key) — the upstream + // ChannelWebhookController stuffs it there. envelope itself is generic + // and doesn't have a typed channel field. + String wantChannel = textOrNull(pattern, "channelType"); + if (wantChannel != null) { + Object actual = envelope.data() == null ? null : envelope.data().get("channelType"); + if (!(actual instanceof String s) || !wantChannel.equalsIgnoreCase(s)) return false; + } + String wantSender = textOrNull(pattern, "senderEquals"); + if (wantSender != null && !wantSender.equals(envelope.senderId())) { + return false; + } + // contentContains is the keyword filter the templates relied on + // — without it a "feishu + 发票" trigger would fire on every + // feishu message. Matches case-insensitively against + // envelope.data.content, the same field content_match uses. + // Keeping the field on channel_message also folds the redundant + // content_match pattern type into the more general channel one; + // content_match remains supported for backwards compatibility + // via {@link #matchesContent}. + String wantContains = textOrNull(pattern, "contentContains"); + if (wantContains != null) { + Object content = envelope.data() == null ? null : envelope.data().get("content"); + if (!(content instanceof String body)) return false; + if (!body.toLowerCase().contains(wantContains.toLowerCase())) return false; + } + return true; + } + + private boolean matchesAgentLifecycle(JsonNode pattern, TriggerEventEnvelope envelope) { + Map data = envelope.data(); + if (data == null) return false; + Long wantAgent = longOrNull(pattern, "agentId"); + if (wantAgent != null) { + Object actual = data.get("agentId"); + if (!(actual instanceof Number n) || n.longValue() != wantAgent) return false; + } + String wantPhase = textOrNull(pattern, "phase"); + if (wantPhase != null) { + Object phase = data.get("phase"); + if (!(phase instanceof String s) || !wantPhase.equalsIgnoreCase(s)) return false; + } + return true; + } + + private boolean matchesContent(JsonNode pattern, TriggerEventEnvelope envelope) { + String needle = textOrNull(pattern, "substring"); + if (needle == null || needle.isBlank()) { + // content_match without a substring is a misconfiguration — refuse + // to fire blanket-on-every-event rather than acting as a wildcard. + return false; + } + Map data = envelope.data(); + if (data == null) return false; + Object content = data.get("content"); + if (!(content instanceof String s)) return false; + return s.toLowerCase().contains(needle.toLowerCase()); + } + + private boolean matchesWorkflowCompletion(JsonNode pattern, TriggerEventEnvelope envelope) { + Map data = envelope.data(); + if (data == null) return false; + Long wantSource = longOrNull(pattern, "sourceWorkflowId"); + if (wantSource != null) { + Object actual = data.get("sourceWorkflowId"); + if (!(actual instanceof Number n) || n.longValue() != wantSource) return false; + } + String wantState = textOrNull(pattern, "stateFilter"); + if (wantState != null && !"any".equalsIgnoreCase(wantState)) { + Object stateObj = data.get("state"); + if (!(stateObj instanceof String actualState)) return false; + // The runtime emits "succeeded" / "failed"; pattern authors + // commonly type "completed" to mean "non-failed terminal". + // Treat the two as equivalent so authors don't have to care + // which vocabulary the runner happens to use today. + if ("completed".equalsIgnoreCase(wantState)) { + if (!"succeeded".equalsIgnoreCase(actualState)) return false; + } else if (!wantState.equalsIgnoreCase(actualState)) { + return false; + } + } + return true; + } + + private static String textOrNull(JsonNode node, String key) { + if (node == null || !node.hasNonNull(key)) return null; + String s = node.get(key).asText(null); + return (s == null || s.isBlank()) ? null : s; + } + + private static Long longOrNull(JsonNode node, String key) { + if (node == null || !node.hasNonNull(key)) return null; + JsonNode v = node.get(key); + if (v.isNumber()) return v.asLong(); + try { + return Long.parseLong(v.asText()); + } catch (NumberFormatException e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerRateLimiter.java b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerRateLimiter.java new file mode 100644 index 00000000..e76212aa --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/ingest/TriggerRateLimiter.java @@ -0,0 +1,53 @@ +package vip.mate.trigger.ingest; + +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Per-trigger sliding-window rate limiter. Each {@code triggerId} keeps a + * 60-second window of fire timestamps; an event is allowed iff fewer than + * the trigger's {@code rate_limit_per_min} entries already live in the + * window. The window is local to this node — for a multi-node deployment + * the cap is a per-node bound, not a global one. v0 accepts that trade + * because the alternative (DB-backed counters) costs a round-trip on every + * event and event volumes are well below the cap in practice. + */ +public class TriggerRateLimiter { + + private final Map> windows = new ConcurrentHashMap<>(); + private final Duration windowSize; + + public TriggerRateLimiter() { + this(Duration.ofMinutes(1)); + } + + TriggerRateLimiter(Duration windowSize) { + this.windowSize = windowSize; + } + + /** + * Try to admit an event for {@code triggerId} at {@code now}. Returns + * {@code true} when the event fits under {@code limitPerMin}; {@code false} + * when the window is full. The window is purged of expired entries first + * so a long-idle trigger reverts to full capacity. + * + *

{@code limitPerMin <= 0} disables the limiter for that trigger. + */ + public boolean tryAcquire(long triggerId, int limitPerMin, Instant now) { + if (limitPerMin <= 0) return true; + Deque window = windows.computeIfAbsent(triggerId, k -> new ArrayDeque<>()); + Instant cutoff = now.minus(windowSize); + synchronized (window) { + while (!window.isEmpty() && !window.peekFirst().isAfter(cutoff)) { + window.pollFirst(); + } + if (window.size() >= limitPerMin) return false; + window.addLast(now); + return true; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEntity.java b/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEntity.java new file mode 100644 index 00000000..543624a1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEntity.java @@ -0,0 +1,80 @@ +package vip.mate.trigger.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Trigger row. {@code patternVersion} is a lamport counter that fire callbacks + * compare against the row on every fire; mismatches mean another instance has + * updated the cron expression and the local schedule must self-cancel. + */ +@Data +@TableName("mate_trigger") +public class TriggerEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long workspaceId; + + @TableField(value = "name", updateStrategy = FieldStrategy.ALWAYS) + private String name; + + /** Pattern flavour: cron / webhook / channel_message / agent_lifecycle / content_match / workflow_completion. */ + private String patternType; + + @TableField(value = "pattern_json", updateStrategy = FieldStrategy.ALWAYS) + private String patternJson; + + /** Routing target type: agent or workflow. */ + private String targetType; + + private Long targetId; + + @TableField(value = "payload_template", updateStrategy = FieldStrategy.ALWAYS) + private String payloadTemplate; + + private Integer rateLimitPerMin; + + private Integer dedupWindowSecs; + + private Boolean botSelfFilter; + + private Boolean enabled; + + private Long fireCount; + + private Long maxFires; + + @TableField(value = "last_fired_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime lastFiredAt; + + /** Most recent dispatch outcome message; null on success, populated on + * SKIPPED / FAILED so the UI can show why a trigger has stopped firing. */ + @TableField(value = "last_error", updateStrategy = FieldStrategy.ALWAYS) + private String lastError; + + /** Stamp of the last dispatch attempt regardless of outcome — used to + * distinguish "never attempted" from "attempted but skipped". */ + @TableField(value = "last_dispatched_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime lastDispatchedAt; + + /** Lamport counter — bump on every cron expression / payload template change. */ + private Long patternVersion; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + // Hard-delete only (project convention); column kept for schema compat. + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEventEntity.java b/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEventEntity.java new file mode 100644 index 00000000..d76907a1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/model/TriggerEventEntity.java @@ -0,0 +1,31 @@ +package vip.mate.trigger.model; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Trigger dedup-window row. {@code dedupKey} carries envelope.eventId, falling + * back to a content sha256 when the upstream channel did not provide a stable + * id. {@code expiresAt} is set on insert to {@code receivedAt + dedupWindowSecs} + * so the cleanup task can sweep expired rows. + */ +@Data +@TableName("mate_trigger_event") +public class TriggerEventEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long triggerId; + + private String dedupKey; + + /** Filled by DB DEFAULT CURRENT_TIMESTAMP when left null on insert. */ + private LocalDateTime receivedAt; + + private LocalDateTime expiresAt; +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerEventMapper.java b/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerEventMapper.java new file mode 100644 index 00000000..2fd612d6 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerEventMapper.java @@ -0,0 +1,9 @@ +package vip.mate.trigger.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.trigger.model.TriggerEventEntity; + +@Mapper +public interface TriggerEventMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerMapper.java b/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerMapper.java new file mode 100644 index 00000000..54f49629 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/repository/TriggerMapper.java @@ -0,0 +1,9 @@ +package vip.mate.trigger.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.trigger.model.TriggerEntity; + +@Mapper +public interface TriggerMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/scheduler/TriggerScheduler.java b/mateclaw-server/src/main/java/vip/mate/trigger/scheduler/TriggerScheduler.java new file mode 100644 index 00000000..03b1fe62 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/scheduler/TriggerScheduler.java @@ -0,0 +1,285 @@ +package vip.mate.trigger.scheduler; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import jakarta.annotation.PostConstruct; +import jakarta.annotation.PreDestroy; +import lombok.extern.slf4j.Slf4j; +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockProvider; +import net.javacrumbs.shedlock.core.SimpleLock; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.scheduling.support.CronTrigger; +import org.springframework.stereotype.Component; +import vip.mate.trigger.dispatch.TriggerDispatcher; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.repository.TriggerMapper; + +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Map; +import java.util.Optional; +import java.util.TimeZone; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; + +/** + * Maintains the in-memory map of cron-pattern triggers active on this node + * and fires them through {@link TriggerDispatcher}. Coordination across + * nodes uses ShedLock (per-trigger lock keyed by id) so simultaneous fires + * collapse into one. Each scheduled task captures the trigger's + * {@code patternVersion} at register time; on fire the live row's version + * is re-read and the local task self-cancels when it has fallen behind a + * newer cron expression — no need to chase a stale {@link ScheduledFuture}. + * + *

Only the {@code cron} pattern type registers here. Other pattern + * flavours (channel_message, workflow_completion, ...) drive triggers + * through their own ingestion pipeline and do not occupy a scheduler tick. + */ +@Slf4j +@Component +public class TriggerScheduler { + + private static final String PATTERN_CRON = "cron"; + + private final TriggerMapper triggerMapper; + private final TriggerDispatcher dispatcher; + private final LockProvider lockProvider; + private final ObjectMapper objectMapper; + + private final ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + private final Map registrations = new ConcurrentHashMap<>(); + + public TriggerScheduler(TriggerMapper triggerMapper, + TriggerDispatcher dispatcher, + LockProvider lockProvider, + ObjectMapper objectMapper) { + this.triggerMapper = triggerMapper; + this.dispatcher = dispatcher; + this.lockProvider = lockProvider; + this.objectMapper = objectMapper; + } + + @PostConstruct + void initScheduler() { + scheduler.setPoolSize(4); + scheduler.setThreadNamePrefix("trigger-tick-"); + scheduler.setDaemon(true); + scheduler.initialize(); + } + + @PreDestroy + void shutdownScheduler() { + scheduler.shutdown(); + registrations.clear(); + } + + /** Boot-time registration sweep; runs after Flyway and bean wiring complete. */ + @EventListener(ApplicationReadyEvent.class) + void registerEnabledTriggersOnStartup() { + syncFromDatabase(); + } + + /** + * Periodic sweep that converges this node's local registrations with + * the canonical state in {@code mate_trigger}. + * + *

Reasons this exists: + *

    + *
  • Multi-instance: when node A creates / updates / disables a + * cron trigger, node B never gets the local-only register call. + * The fire-time {@code patternVersion} guard self-cancels stale + * schedules but does NOT register newly-created or newly-enabled + * triggers — only this sweep does.
  • + *
  • Recovery from missed events: if a register / unregister call + * races with a node restart, the in-memory map can drift from + * the row state. Refreshing every minute caps the divergence.
  • + *
+ * + *

Convergence rules: + *

    + *
  • Row enabled + cron type + not registered locally → register.
  • + *
  • Row enabled but local {@code capturedVersion} differs from + * row's {@code pattern_version} → re-register (the schedule + * carries the new expression).
  • + *
  • Local registration exists for a row that's now disabled, + * deleted, or no longer cron-typed → unregister.
  • + *
+ */ + @Scheduled(fixedDelayString = "${mateclaw.workflow.trigger.sync-interval-ms:60000}", + initialDelayString = "${mateclaw.workflow.trigger.sync-initial-delay-ms:60000}") + public void syncFromDatabase() { + var enabled = triggerMapper.selectList(new LambdaQueryWrapper() + .eq(TriggerEntity::getEnabled, true) + .eq(TriggerEntity::getDeleted, 0)); + java.util.Set seenIds = new java.util.HashSet<>(); + int registered = 0, refreshed = 0, removed = 0; + for (TriggerEntity t : enabled) { + if (!PATTERN_CRON.equalsIgnoreCase(t.getPatternType())) continue; + seenIds.add(t.getId()); + Registration current = registrations.get(t.getId()); + long liveVersion = t.getPatternVersion() == null ? 1L : t.getPatternVersion(); + if (current == null) { + if (registerInternal(t)) registered++; + } else if (current.capturedVersion != liveVersion) { + if (registerInternal(t)) refreshed++; + } + } + // Drop registrations whose row was disabled / deleted / changed type + // since the last sweep. Snapshot the keys first to avoid concurrent + // modification on the underlying map. + for (Long localId : new java.util.ArrayList<>(registrations.keySet())) { + if (!seenIds.contains(localId)) { + unregister(localId); + removed++; + } + } + if (registered + refreshed + removed > 0) { + log.info("[TriggerScheduler] sync: registered={} refreshed={} removed={} active={}", + registered, refreshed, removed, registrations.size()); + } + } + + /** Register or replace a single trigger (called from {@code TriggerService} on save). */ + public boolean register(TriggerEntity trigger) { + if (trigger == null || !PATTERN_CRON.equalsIgnoreCase(trigger.getPatternType())) { + return false; + } + return registerInternal(trigger); + } + + /** Cancel any active schedule for {@code triggerId}. Idempotent. */ + public void unregister(long triggerId) { + Registration r = registrations.remove(triggerId); + if (r != null) { + r.future.cancel(false); + } + } + + /** + * Whether {@code triggerId} currently occupies an active scheduled task on + * this node. Visible because monitoring / health endpoints surface the + * same fact, and the alternative would be exposing the raw registration + * map. + */ + public boolean isRegistered(long triggerId) { + return registrations.containsKey(triggerId); + } + + /** + * Manually drive the lamport + dispatch path the cron tick would otherwise + * call. Used by integration tests; production code should never call this + * directly — the scheduler owns its own tick. + */ + public void fireForTest(long triggerId, long capturedVersion) { + fireWithCoordination(triggerId, capturedVersion); + } + + private boolean registerInternal(TriggerEntity trigger) { + unregister(trigger.getId()); + ParsedCron parsed = parseCron(trigger); + if (parsed == null) return false; + + long capturedVersion = trigger.getPatternVersion() == null ? 1L : trigger.getPatternVersion(); + Runnable task = () -> fireWithCoordination(trigger.getId(), capturedVersion); + ScheduledFuture future = scheduler.schedule(task, + new CronTrigger(parsed.expression, parsed.timeZone)); + registrations.put(trigger.getId(), new Registration(future, capturedVersion)); + log.info("[TriggerScheduler] Registered trigger {} cron='{}' tz={} version={}", + trigger.getId(), parsed.expression, parsed.timeZone.getID(), capturedVersion); + return true; + } + + private void fireWithCoordination(long triggerId, long capturedVersion) { + // Per-fire lamport check: a newer expression in the DB invalidates + // this scheduled task. Drop the fire and unregister so the next + // registration cycle picks up the new schedule. + TriggerEntity live = triggerMapper.selectById(triggerId); + if (live == null || Boolean.FALSE.equals(live.getEnabled())) { + unregister(triggerId); + return; + } + long liveVersion = live.getPatternVersion() == null ? 1L : live.getPatternVersion(); + if (liveVersion != capturedVersion) { + log.info("[TriggerScheduler] trigger {} self-cancelling (version changed {} -> {})", + triggerId, capturedVersion, liveVersion); + unregister(triggerId); + return; + } + if (live.getMaxFires() != null && live.getMaxFires() > 0 + && live.getFireCount() != null && live.getFireCount() >= live.getMaxFires()) { + log.info("[TriggerScheduler] trigger {} reached max_fires={}, unregistering", + triggerId, live.getMaxFires()); + unregister(triggerId); + return; + } + + // Cross-node coordination: at-most-one node fires per tick. + Optional lock = lockProvider.lock(new LockConfiguration( + Instant.now(), + "trigger-fire-" + triggerId, + Duration.ofSeconds(60), + Duration.ofSeconds(5))); + if (lock.isEmpty()) { + return; // peer is firing + } + try { + vip.mate.trigger.dispatch.DispatchResult outcome = + dispatcher.dispatch(live, Map.of("firedAt", Instant.now().toString())); + // Bookkeeping is honest: only a real fire bumps fireCount / + // lastFiredAt. Skipped (no published revision, etc.) and failed + // outcomes still record lastDispatchedAt + lastError so the UI + // can show why a cron stopped firing. + LocalDateTime now = LocalDateTime.now(); + live.setLastDispatchedAt(now); + if (outcome != null && outcome.fired()) { + live.setFireCount((live.getFireCount() == null ? 0L : live.getFireCount()) + 1); + live.setLastFiredAt(now); + live.setLastError(null); + } else { + live.setLastError(outcome == null ? "dispatcher returned null" : outcome.reason()); + } + triggerMapper.updateById(live); + } catch (Exception e) { + log.error("[TriggerScheduler] trigger {} fire failed: {}", triggerId, e.getMessage(), e); + try { + live.setLastDispatchedAt(LocalDateTime.now()); + live.setLastError("scheduler threw: " + e.getMessage()); + triggerMapper.updateById(live); + } catch (Exception ignored) { + // Best-effort — don't let a bookkeeping failure mask the dispatch failure. + } + } finally { + lock.get().unlock(); + } + } + + private record ParsedCron(String expression, TimeZone timeZone) {} + + private ParsedCron parseCron(TriggerEntity trigger) { + try { + JsonNode node = objectMapper.readTree( + trigger.getPatternJson() == null ? "{}" : trigger.getPatternJson()); + String expr = node.path("cron").asText(""); + if (expr.isBlank()) { + log.warn("[TriggerScheduler] trigger {} missing 'cron' in pattern_json; skipping", + trigger.getId()); + return null; + } + String tz = node.path("timezone").asText("UTC"); + return new ParsedCron(expr, TimeZone.getTimeZone(ZoneId.of(tz))); + } catch (Exception e) { + log.warn("[TriggerScheduler] trigger {} pattern_json parse failed: {}", + trigger.getId(), e.getMessage()); + return null; + } + } + + private record Registration(ScheduledFuture future, long capturedVersion) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/trigger/service/TriggerService.java b/mateclaw-server/src/main/java/vip/mate/trigger/service/TriggerService.java new file mode 100644 index 00000000..dcef2f47 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/trigger/service/TriggerService.java @@ -0,0 +1,245 @@ +package vip.mate.trigger.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.repository.TriggerMapper; +import vip.mate.trigger.scheduler.TriggerScheduler; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.repository.WorkflowMapper; + +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * CRUD facade for {@code mate_trigger} that keeps the in-memory cron + * registration in sync with the persisted row. Pattern_version is the + * lamport counter the scheduler uses to invalidate stale schedules across + * a multi-node deployment — every change to {@code patternJson}, + * {@code patternType}, or the disabled→enabled transition bumps it. + * + *

The service intentionally does not wrap reads in transactions; only + * mutating paths are {@code @Transactional} so the scheduler hand-off + * (which reads the row again under its own connection) sees committed + * data. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class TriggerService { + + /** Pattern types accepted by the v0 matcher; anything else fails closed at ingest. */ + private static final Set SUPPORTED_PATTERNS = Set.of( + "cron", "channel_message", "webhook", "agent_lifecycle", + "content_match", "workflow_completion"); + + /** v0 only dispatches workflow targets; agent target requires a v1 dispatcher. */ + private static final Set SUPPORTED_TARGETS = Set.of("workflow"); + + private final TriggerMapper triggerMapper; + private final TriggerScheduler scheduler; + /** Optional — only present in production. Tests can null it out via constructor. */ + @Autowired(required = false) + private WorkflowMapper workflowMapper; + + public List listByWorkspace(long workspaceId) { + return triggerMapper.selectList(new LambdaQueryWrapper() + .eq(TriggerEntity::getWorkspaceId, workspaceId) + .orderByDesc(TriggerEntity::getCreateTime)); + } + + /** + * Lookup that scopes to a single workspace. Returns {@code null} when the + * trigger doesn't exist OR when it belongs to another workspace, so the + * caller can surface the same "not found" status either way and avoid + * leaking foreign trigger ids. + */ + public TriggerEntity get(long id, long workspaceId) { + TriggerEntity row = triggerMapper.selectById(id); + if (row == null || row.getWorkspaceId() == null || row.getWorkspaceId() != workspaceId) { + return null; + } + return row; + } + + /** Backwards-compatible single-arg get; only used by internal pipelines that + * already know they hold a trusted id (scheduler, ingest). New callers must + * use {@link #get(long, long)}. */ + public TriggerEntity get(long id) { + return triggerMapper.selectById(id); + } + + @Transactional + public TriggerEntity create(TriggerEntity trigger, long workspaceId) { + // Ignore whatever workspace / id the caller put on the body — we + // trust the workspace from the request header alone. + trigger.setId(null); + trigger.setWorkspaceId(workspaceId); + validatePatternAndTargetShape(trigger); + validateTargetOwnership(trigger, workspaceId); + ensureDefaults(trigger); + trigger.setPatternVersion(1L); + trigger.setFireCount(0L); + triggerMapper.insert(trigger); + if (Boolean.TRUE.equals(trigger.getEnabled())) { + scheduler.register(trigger); + } + return trigger; + } + + /** @deprecated use {@link #create(TriggerEntity, long)} so the workspace + * isn't trusted from the body. Kept for tests that already supply a + * workspace id on the entity and reference fixture workflow ids that + * may not have a real row in mate_workflow. */ + @Deprecated + @Transactional + public TriggerEntity create(TriggerEntity trigger) { + Long ws = trigger.getWorkspaceId(); + if (ws == null) { + throw new IllegalArgumentException("workspaceId required"); + } + validatePatternAndTargetShape(trigger); + ensureDefaults(trigger); + trigger.setPatternVersion(1L); + trigger.setFireCount(0L); + triggerMapper.insert(trigger); + if (Boolean.TRUE.equals(trigger.getEnabled())) { + scheduler.register(trigger); + } + return trigger; + } + + @Transactional + public TriggerEntity update(long id, long workspaceId, TriggerEntity updated) { + TriggerEntity existing = get(id, workspaceId); + if (existing == null) { + throw new IllegalArgumentException("trigger not found: " + id); + } + // Force the canonical id + workspace; reject any body-side override. + updated.setId(id); + updated.setWorkspaceId(workspaceId); + validatePatternAndTargetShape(updated); + validateTargetOwnership(updated, workspaceId); + return updateInternal(existing, updated); + } + + /** @deprecated use the workspace-scoped overload. */ + @Deprecated + @Transactional + public TriggerEntity update(TriggerEntity updated) { + TriggerEntity existing = triggerMapper.selectById(updated.getId()); + if (existing == null) { + throw new IllegalArgumentException("trigger not found: " + updated.getId()); + } + return updateInternal(existing, updated); + } + + private TriggerEntity updateInternal(TriggerEntity existing, TriggerEntity updated) { + // Bump pattern_version whenever ANY field that changes the + // schedule's behavior, payload rendering, or rate decisions + // changes. This is the lamport other instances rely on at fire + // time to decide whether their captured registration is stale — + // missing a field here means a peer fires the new payload with + // the old throttling settings (or vice versa) until it next + // self-cancels for some other reason. + boolean patternChanged = !Objects.equals(existing.getPatternJson(), updated.getPatternJson()) + || !Objects.equals(existing.getPatternType(), updated.getPatternType()); + boolean payloadChanged = !Objects.equals(existing.getPayloadTemplate(), updated.getPayloadTemplate()); + boolean targetChanged = !Objects.equals(existing.getTargetType(), updated.getTargetType()) + || !Objects.equals(existing.getTargetId(), updated.getTargetId()); + boolean fireConfigChanged = !Objects.equals(existing.getRateLimitPerMin(), updated.getRateLimitPerMin()) + || !Objects.equals(existing.getDedupWindowSecs(), updated.getDedupWindowSecs()) + || !Objects.equals(existing.getMaxFires(), updated.getMaxFires()) + || !Objects.equals(existing.getBotSelfFilter(), updated.getBotSelfFilter()); + boolean enableTransition = !Objects.equals(existing.getEnabled(), updated.getEnabled()); + + if (patternChanged || payloadChanged || targetChanged || fireConfigChanged || enableTransition) { + long bumped = (existing.getPatternVersion() == null ? 1L : existing.getPatternVersion()) + 1L; + updated.setPatternVersion(bumped); + } else { + updated.setPatternVersion(existing.getPatternVersion()); + } + // Preserve fireCount / lastFiredAt / lastError — those are scheduler / ingest owned. + updated.setFireCount(existing.getFireCount()); + updated.setLastFiredAt(existing.getLastFiredAt()); + + triggerMapper.updateById(updated); + + if (Boolean.TRUE.equals(updated.getEnabled())) { + scheduler.register(updated); + } else { + scheduler.unregister(updated.getId()); + } + return updated; + } + + @Transactional + public void delete(long id, long workspaceId) { + TriggerEntity row = get(id, workspaceId); + if (row == null) return; // 404-equivalent: idempotent for missing rows + scheduler.unregister(id); + triggerMapper.deleteById(id); + } + + /** @deprecated workspace-blind delete; only retained for tests. */ + @Deprecated + @Transactional + public void delete(long id) { + scheduler.unregister(id); + triggerMapper.deleteById(id); + } + + /** + * Pattern + target shape validation — runs on every entry path so a + * trigger can never silently land in a "looks enabled, never fires" + * state. The acceptance set deliberately mirrors what + * {@code TriggerPatternMatcher} understands AND what + * {@code TriggerDispatcher} can actually route — extending one + * without the other would re-introduce the silent-skip bug. + */ + private static void validatePatternAndTargetShape(TriggerEntity t) { + String pt = t.getPatternType(); + if (pt == null || !SUPPORTED_PATTERNS.contains(pt)) { + throw new IllegalArgumentException("unsupported patternType: " + pt + + " (expected one of " + SUPPORTED_PATTERNS + ")"); + } + String tt = t.getTargetType(); + if (tt == null || !SUPPORTED_TARGETS.contains(tt)) { + throw new IllegalArgumentException("unsupported targetType: " + tt + + " (v0 only supports 'workflow')"); + } + } + + /** + * Cross-workspace ownership check — a trigger in workspace A must + * not be able to point at a workflow in workspace B. Only runs on + * the workspace-aware entry points (create / update with explicit + * workspaceId). The deprecated overloads skip this so legacy tests + * that reference fixture workflow ids without inserting them keep + * working. + */ + private void validateTargetOwnership(TriggerEntity t, long workspaceId) { + if ("workflow".equals(t.getTargetType()) && t.getTargetId() != null + && workflowMapper != null) { + WorkflowEntity wf = workflowMapper.selectById(t.getTargetId()); + if (wf == null || wf.getWorkspaceId() == null + || wf.getWorkspaceId() != workspaceId) { + throw new IllegalArgumentException( + "target workflow not found in workspace: " + t.getTargetId()); + } + } + } + + private static void ensureDefaults(TriggerEntity t) { + if (t.getRateLimitPerMin() == null) t.setRateLimitPerMin(60); + if (t.getDedupWindowSecs() == null) t.setDedupWindowSecs(60); + if (t.getBotSelfFilter() == null) t.setBotSelfFilter(true); + if (t.getEnabled() == null) t.setEnabled(true); + if (t.getMaxFires() == null) t.setMaxFires(0L); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java index cf3b4c00..4df212e6 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/WikiProperties.java @@ -130,6 +130,37 @@ public class WikiProperties { */ private int embeddingMaxChars = 6000; + /** + * Expected embedding-input format version. The authoritative source is + * the builder's {@code CURRENT_INPUT_VERSION} constant; this property + * exists for staged rollouts and ops overrides. + *

+ * Behavior on startup: + *

    + *
  • Blank: use the builder version.
  • + *
  • Less than builder version: WARN and continue, so a KB can be + * embedded against an older format during a gradual rollback.
  • + *
  • Greater than builder version: fail fast — this usually means the + * config was deployed ahead of the code that implements that format.
  • + *
+ */ + private String embeddingTextVersionCurrent = ""; + + /** + * Circuit-breaker threshold: abort an embedding pass after this many + * consecutive batch failures (auth / rate-limit / network errors that + * cause an entire batch to embed zero chunks). Without it, a broken + * provider would silently iterate through every pending chunk in the + * KB, producing only log noise and wasted wall-clock time before the + * user can intervene. + *

+ * Set too low and a transient blip aborts a healthy pass; set too + * high and the user waits forever on a clearly-broken provider. + * Default 5 covers most real outages while tolerating a couple of + * isolated 5xx hiccups. + */ + private int embeddingConsecutiveFailureThreshold = 5; + /** 混合搜索默认模式:keyword / semantic / hybrid */ private String searchDefaultMode = "hybrid"; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java index b765f46d..0de20d85 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiController.java @@ -300,6 +300,23 @@ public class WikiController { return R.ok(); } + @RequireWorkspaceRole("member") + @Operation(summary = "请求取消正在进行的处理(仅在 processing 状态有效)") + @PostMapping("/knowledge-bases/{kbId}/raw/{rawId}/cancel") + public R cancelRaw(@PathVariable Long kbId, @PathVariable Long rawId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + verifyKBWorkspace(kbId, workspaceId); + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw == null || !kbId.equals(raw.getKbId())) { + return R.fail("Raw material not found in this knowledge base"); + } + // requestCancel is idempotent: a no-op when the row is not processing, + // so repeated clicks (or a click after the run already finished) are + // safe and do not surface an error to the user. + rawService.requestCancel(rawId); + return R.ok(); + } + @RequireWorkspaceRole("viewer") @Operation(summary = "下载原始材料") @GetMapping("/knowledge-bases/{kbId}/raw/{rawId}/download") diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java index 854d9d3a..484be855 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiRelationController.java @@ -35,6 +35,7 @@ public class WikiRelationController { private final HybridRetriever hybridRetriever; private final ApplicationEventPublisher eventPublisher; private final ObjectMapper objectMapper; + private final WikiEmbeddingService embeddingService; // ==================== RFC-029: Relations ==================== @@ -104,11 +105,14 @@ public class WikiRelationController { .filter(j -> "running".equals(j.getStatus())) .count(); + WikiEmbeddingService.EmbeddingDrift drift = embeddingService.describeDrift(kbId); + return Map.of( "pageCount", pageCount, "enrichedPageCount", enrichedCount, "failedJobCount", failedJobCount, - "runningJobCount", runningJobCount + "runningJobCount", runningJobCount, + "embeddingDrift", drift ); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java new file mode 100644 index 00000000..be3cb422 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/controller/WikiTransformationController.java @@ -0,0 +1,276 @@ +package vip.mate.wiki.controller; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.exception.MateClawException; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; +import vip.mate.wiki.service.WikiKnowledgeBaseService; +import vip.mate.wiki.service.WikiTransformationAggregator; +import vip.mate.wiki.service.WikiTransformationExecutor; +import vip.mate.wiki.service.WikiTransformationService; +import vip.mate.workspace.core.annotation.RequireWorkspaceRole; + +import java.util.List; +import java.util.Map; + +/** + * Management surface for user-defined wiki transformation templates and + * their execution history. Templates live under the workspace; a template + * with non-null {@code kbId} is pinned to a single KB, otherwise it is + * available to every KB in the workspace. + */ +@Slf4j +@Tag(name = "Wiki Transformations", + description = "User-defined prompt templates run over wiki raw materials") +@RestController +@RequestMapping("/api/v1/wiki/transformations") +@RequiredArgsConstructor +public class WikiTransformationController { + + private final WikiTransformationService transformationService; + private final WikiTransformationExecutor executor; + private final WikiTransformationAggregator aggregator; + private final WikiKnowledgeBaseService kbService; + + // ==================== Templates ==================== + + @RequireWorkspaceRole("viewer") + @Operation(summary = "List transformations available to a KB", + description = "Returns templates pinned to the KB plus workspace-wide templates.") + @GetMapping + public R> list( + @RequestParam(required = false) Long kbId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (kbId != null) { + verifyKBWorkspace(kbId, wsId); + return R.ok(transformationService.listForKb(kbId, wsId)); + } + return R.ok(transformationService.listByWorkspace(wsId)); + } + + @RequireWorkspaceRole("viewer") + @GetMapping("/{id}") + public R get(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity t = transformationService.getById(id); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, workspaceId); + return R.ok(t); + } + + @RequireWorkspaceRole("member") + @PostMapping + public R create(@RequestBody WikiTransformationEntity body, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (body.getKbId() != null) { + verifyKBWorkspace(body.getKbId(), wsId); + } + body.setWorkspaceId(wsId); + WikiTransformationEntity created = transformationService.create(body); + return R.ok(created); + } + + @RequireWorkspaceRole("member") + @PutMapping("/{id}") + public R update(@PathVariable Long id, + @RequestBody WikiTransformationEntity body, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity existing = transformationService.getById(id); + if (existing == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(existing, workspaceId); + return R.ok(transformationService.update(id, body)); + } + + @RequireWorkspaceRole("member") + @DeleteMapping("/{id}") + public R delete(@PathVariable Long id, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity existing = transformationService.getById(id); + if (existing != null) { + verifyTemplateWorkspace(existing, workspaceId); + transformationService.delete(id); + } + return R.ok(); + } + + // ==================== Apply ==================== + + @RequireWorkspaceRole("member") + @Operation(summary = "Run a transformation against a raw material or wiki page", + description = "Body accepts exactly one of {rawId, pageId}. Set sync=true to block " + + "until the LLM call returns; when false (default) the call returns " + + "immediately with the pending run row.") + @PostMapping("/{id}/apply") + public R apply(@PathVariable Long id, + @RequestBody Map body, + @RequestParam(defaultValue = "false") boolean sync, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity t = transformationService.getById(id); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, workspaceId); + + Object rawIdRaw = body == null ? null : body.get("rawId"); + Object pageIdRaw = body == null ? null : body.get("pageId"); + if (rawIdRaw == null && pageIdRaw == null) { + return R.fail("One of rawId / pageId is required"); + } + if (rawIdRaw != null && pageIdRaw != null) { + return R.fail("Pass only one of rawId / pageId, not both"); + } + + if (rawIdRaw != null) { + Long rawId = Long.valueOf(rawIdRaw.toString()); + if (sync) return R.ok(executor.runOnRawSync(t, rawId, "manual")); + executor.runOnRawAsync(t, rawId, "manual"); + } else { + Long pageId = Long.valueOf(pageIdRaw.toString()); + if (sync) return R.ok(executor.runOnPageSync(t, pageId, "manual")); + executor.runOnPageAsync(t, pageId, "manual"); + } + return R.ok(); + } + + @RequireWorkspaceRole("member") + @Operation(summary = "Aggregate all completed runs of a template into one KB-level synthesis page", + description = "Map-reduces across every completed run of the template within the given KB. " + + "Upserts the merged document at slug '-aggregate'.") + @PostMapping("/{id}/aggregate") + public R> aggregate(@PathVariable Long id, + @RequestParam Long kbId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationEntity t = transformationService.getById(id); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, workspaceId); + verifyKBWorkspace(kbId, workspaceId != null ? workspaceId : 1L); + + try { + WikiTransformationAggregator.Result res = aggregator.aggregate(t, kbId, "manual"); + if (res.pageId() == null) { + return R.fail(res.title()); // when sources are empty we put the reason in title field + } + return R.ok(Map.of( + "pageId", res.pageId(), + "slug", res.slug(), + "title", res.title(), + "sourcesUsed", res.sourcesUsed(), + "charsFed", res.charsFed(), + "created", res.created())); + } catch (IllegalStateException | IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + + // ==================== Runs ==================== + + @RequireWorkspaceRole("viewer") + @GetMapping("/runs/{runId}") + public R getRun(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return R.fail("Run not found"); + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + return R.ok(run); + } + + @RequireWorkspaceRole("viewer") + @GetMapping("/runs") + public R> listRuns( + @RequestParam(required = false) Long rawId, + @RequestParam(required = false) Long kbId, + @RequestParam(required = false) Long transformationId, + @RequestParam(defaultValue = "50") int limit, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (rawId != null) { + return R.ok(transformationService.listRunsByRaw(rawId, limit)); + } + if (transformationId != null) { + WikiTransformationEntity t = transformationService.getById(transformationId); + if (t == null) return R.fail("Transformation not found"); + verifyTemplateWorkspace(t, wsId); + return R.ok(transformationService.listRunsByTransformation(transformationId, limit)); + } + if (kbId != null) { + verifyKBWorkspace(kbId, wsId); + return R.ok(transformationService.listRunsByKb(kbId, limit)); + } + return R.fail("One of rawId / kbId / transformationId is required"); + } + + @RequireWorkspaceRole("member") + @Operation(summary = "Cancel a still-running transformation run", + description = "Marks the run as cancelled so the executor drops the eventual LLM output. " + + "The HTTP request to the model continues server-side because most providers " + + "do not support cancellation; this endpoint affects bookkeeping only.") + @PostMapping("/runs/{runId}/cancel") + public R cancelRun(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return R.fail("Run not found"); + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + boolean cancelled = executor.cancelRun(runId); + if (!cancelled) return R.fail("Run is not running"); + return R.ok(); + } + + @RequireWorkspaceRole("member") + @Operation(summary = "Save a completed run's output as a synthesis wiki page", + description = "Idempotent: re-saving an already-saved run updates the same page slug.") + @PostMapping("/runs/{runId}/save-as-page") + public R> saveRunAsPage(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return R.fail("Run not found"); + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + try { + var page = executor.manualSaveRunAsPage(runId); + if (page == null) return R.fail("Page service unavailable"); + return R.ok(Map.of( + "pageId", page.getId(), + "slug", page.getSlug(), + "title", page.getTitle())); + } catch (IllegalStateException | IllegalArgumentException e) { + return R.fail(e.getMessage()); + } + } + + @RequireWorkspaceRole("member") + @DeleteMapping("/runs/{runId}") + public R deleteRun(@PathVariable Long runId, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run != null) { + verifyKBWorkspace(run.getKbId(), workspaceId != null ? workspaceId : 1L); + transformationService.deleteRun(runId); + } + return R.ok(); + } + + // ==================== helpers ==================== + + private void verifyKBWorkspace(Long kbId, Long workspaceId) { + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + if (kb == null) { + throw new MateClawException("Knowledge base not found"); + } + long wsId = workspaceId != null ? workspaceId : 1L; + if (kb.getWorkspaceId() != null && !kb.getWorkspaceId().equals(wsId)) { + throw new MateClawException("err.common.wrong_workspace", "Resource does not belong to current workspace"); + } + } + + private void verifyTemplateWorkspace(WikiTransformationEntity t, Long workspaceId) { + long wsId = workspaceId != null ? workspaceId : 1L; + if (t.getWorkspaceId() != null && !t.getWorkspaceId().equals(wsId)) { + throw new MateClawException("err.common.wrong_workspace", "Resource does not belong to current workspace"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiEmbeddingProviderFailingException.java b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiEmbeddingProviderFailingException.java new file mode 100644 index 00000000..48964bee --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/job/WikiEmbeddingProviderFailingException.java @@ -0,0 +1,40 @@ +package vip.mate.wiki.job; + +/** + * Thrown by {@link vip.mate.wiki.service.WikiEmbeddingService#embedMissingChunks(Long)} + * when the embedding provider has failed N batches in a row, where N is + * controlled by {@code mate.wiki.embedding-consecutive-failure-threshold}. + * + *

Without this circuit, a misconfigured or unavailable provider (out + * of credits, wrong API key, network partition) silently churns through + * every pending chunk one batch at a time — producing log noise but no + * actual progress, and consuming wall-clock time the user sees as a + * stuck "task in loop". The circuit lets the embedding pass abort fast + * so the user can fix configuration and retry. + * + *

This is a soft failure: the next call into {@code embedMissingChunks} + * starts a fresh counter and will retry the provider, so once the user + * has corrected the configuration the embedding pass picks up where it + * left off without manual intervention. + */ +public class WikiEmbeddingProviderFailingException extends RuntimeException { + + private final int consecutiveFailures; + private final int remainingChunks; + + public WikiEmbeddingProviderFailingException(String message, + int consecutiveFailures, + int remainingChunks) { + super(message); + this.consecutiveFailures = consecutiveFailures; + this.remainingChunks = remainingChunks; + } + + public int getConsecutiveFailures() { + return consecutiveFailures; + } + + public int getRemainingChunks() { + return remainingChunks; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java index aef31ce6..6c7abf0f 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiChunkEntity.java @@ -51,6 +51,15 @@ public class WikiChunkEntity { /** RFC-011:生成该 embedding 的模型名称(切模型时需全量重嵌) */ private String embeddingModel; + /** + * Identifies the input format used to produce the stored embedding. + *

+ * Set to the embedding input builder's current version on every write. + * NULL signals a legacy content-only embedding from before the builder + * existed and is treated as stale on the next re-embed pass. + */ + private String embeddingTextVersion; + /** RFC-051: source page number (PDF/PPTX) when known; null otherwise. */ private Integer pageNumber; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java index 78641cc6..ba57e887 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiPageEntity.java @@ -73,6 +73,20 @@ public class WikiPageEntity { */ private Integer archived; + /** + * Page-level embedding (float32 little-endian) used by the semantic + * retriever to surface pages whose generated content does not appear + * in any source raw's chunks — typically synthesis pages produced by + * a transformation. {@code null} = not yet embedded. + */ + private byte[] embedding; + + /** Model name that produced {@link #embedding}; used for re-embed detection. */ + private String embeddingModel; + + /** Input-format version for {@link #embedding}; bumped when the embedding builder changes. */ + private String embeddingTextVersion; + @TableField(fill = FieldFill.INSERT) private LocalDateTime createTime; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java index 06bcdbba..f623ccdf 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiRawMaterialEntity.java @@ -46,9 +46,18 @@ public class WikiRawMaterialEntity { /** 文件大小(字节) */ private Long fileSize; - /** 处理状态:pending / processing / completed / failed */ + /** 处理状态:pending / processing / completed / failed / partial / cancelled */ private String processingStatus; + /** + * User-requested cancellation flag. Set to {@code true} via the cancel + * endpoint while a raw material is in {@code processing}. The pipeline + * observes the flag at its abort checkpoints and exits early with + * {@code processingStatus = "cancelled"}; the flag is cleared on the + * next successful claim for processing. + */ + private Boolean cancelRequested; + /** 上次处理时间 */ private LocalDateTime lastProcessedAt; diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java new file mode 100644 index 00000000..7acc5ec5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationEntity.java @@ -0,0 +1,95 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * User-defined prompt template applied to a knowledge base's raw materials + * (and, eventually, pages). One template + one source = one + * {@link WikiTransformationRunEntity}. + * + *

Template body supports the placeholders {@code {input_text}} and + * {@code {title}}, replaced by the executor before the LLM call. + */ +@Data +@TableName("mate_wiki_transformation") +public class WikiTransformationEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + /** + * Pinned KB. {@code null} means the template is available to every KB + * in the workspace. + */ + private Long kbId; + + private Long workspaceId; + + /** Stable short identifier; unique per {@code kbId}. */ + private String name; + + private String title; + + private String description; + + /** Prompt body with {@code {input_text}} / {@code {title}} placeholders. */ + private String promptTemplate; + + /** + * When true, the ingestion pipeline fires this template automatically + * for every raw material that lands in {@code completed} for a matching + * KB. + */ + private Boolean applyDefault; + + /** Optional explicit model override; {@code null} = use KB default. */ + private Long modelId; + + private Boolean enabled; + + /** + * Where the output of a successful run lands. + *

    + *
  • {@code none} — output stays in the run history only (default).
  • + *
  • {@code page} — output is upserted as a synthesis wiki page on the + * same KB; subsequent runs against the same source raw material + * update the same page rather than spawning duplicates.
  • + *
+ */ + private String outputTarget; + + /** + * Declared shape of the LLM output. {@code markdown} (default) accepts + * any text and stores it verbatim. {@code json} asks the LLM for a + * single JSON document; the executor parses it, retries once on parse + * failure, and marks the run failed if both attempts fail. JSON output + * is stored as a fenced ```json block in the run row so the existing + * markdown rendering path stays compatible. + */ + private String outputFormat; + + /** + * Optional JSON Schema text describing the expected shape when + * {@code outputFormat == 'json'}. Injected into the prompt verbatim + * so the LLM has explicit field expectations; the executor also runs + * a lightweight required-fields check after parsing. + */ + private String outputSchema; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java new file mode 100644 index 00000000..2f7dc63b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/model/WikiTransformationRunEntity.java @@ -0,0 +1,78 @@ +package vip.mate.wiki.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableLogic; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * One execution of a {@link WikiTransformationEntity} against a source + * (raw material today; pages in a follow-up). Output is stored inline so + * the UI can render the result without re-running the LLM. + */ +@Data +@TableName("mate_wiki_transformation_run") +public class WikiTransformationRunEntity { + + @TableId(type = IdType.AUTO) + private Long id; + + private Long transformationId; + private Long kbId; + private Long workspaceId; + + /** {@code raw} | {@code page} | {@code text}. */ + private String inputKind; + + private Long rawId; + private Long pageId; + + /** {@code pending} | {@code running} | {@code completed} | {@code failed}. */ + private String status; + + /** LLM output; treat as Markdown unless the prompt asked for JSON. */ + private String output; + + private String error; + + /** Model that actually produced the output after routing fallback. */ + private Long modelId; + + /** {@code apply_default} | {@code manual} | {@code agent_tool}. */ + private String triggeredBy; + + private LocalDateTime startedAt; + private LocalDateTime completedAt; + private Long durationMs; + + /** + * Set when the run was persisted as a synthesis wiki page (either + * automatically because the template's {@code outputTarget} is {@code page}, + * or manually via the save-as-page endpoint). Points at + * {@code mate_wiki_page.id}. + */ + private Long outputPageId; + + /** Prompt-side tokens reported by the provider (Spring AI Usage). */ + private Long inputTokens; + + /** Completion-side tokens reported by the provider. */ + private Long outputTokens; + + /** Provider's own total (usually input + output, but providers vary). */ + private Long totalTokens; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + @TableLogic + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java new file mode 100644 index 00000000..667b8c3c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationMapper.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiTransformationEntity; + +@Mapper +public interface WikiTransformationMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java new file mode 100644 index 00000000..e08bdc7e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/repository/WikiTransformationRunMapper.java @@ -0,0 +1,9 @@ +package vip.mate.wiki.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.wiki.model.WikiTransformationRunEntity; + +@Mapper +public interface WikiTransformationRunMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java index e63cf404..ac4f5eef 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/HybridRetriever.java @@ -211,13 +211,15 @@ public class HybridRetriever { // ==================== Internal methods ==================== - /** Semantic search: chunk cosine → aggregate to page level */ + /** Semantic search: chunk cosine → aggregate to page level, then merged + * with direct page-level cosine when a page has its own embedding. + * The page-level signal covers synthesis pages whose vocabulary doesn't + * appear in any source raw's chunks. */ private List semanticSearch(Long kbId, String query, int limit) { float[] queryVec = embeddingService.embedQuery(kbId, query); if (queryVec == null) return List.of(); List allChunks = chunkService.listByKbId(kbId); - if (allChunks.isEmpty()) return List.of(); Map chunkScores = new HashMap<>(); for (WikiChunkEntity chunk : allChunks) { @@ -230,6 +232,14 @@ public class HybridRetriever { List allPages = pageService.listByKbId(kbId); Map pageScores = new HashMap<>(); for (WikiPageEntity page : allPages) { + // Direct page-level signal: the page carries its own embedding + // (typical for transformation synthesis pages). + if (page.getEmbedding() != null) { + float[] pageVec = WikiEmbeddingService.bytesToFloats(page.getEmbedding()); + float pageScore = WikiEmbeddingService.cosine(queryVec, pageVec); + pageScores.merge(page.getId(), (double) pageScore, Math::max); + } + // Transitive signal: chunks of any source raw this page references. String rawIds = page.getSourceRawIds(); if (rawIds == null) continue; for (String rawIdStr : rawIds.replaceAll("[\\[\\]\\s]", "").split(",")) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java new file mode 100644 index 00000000..1ffe5f3b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookup.java @@ -0,0 +1,15 @@ +package vip.mate.wiki.service; + +/** + * Resolves {@code rawId -> rawTitle} for embedding-time enrichment. + *

+ * Implementations may be naive (one DB hit per call), batch-preloaded for a + * given job, or backed by an in-memory snapshot shared with an ingest-scope + * page index. Callers must tolerate {@code null} for unknown / deleted ids. + */ +@FunctionalInterface +public interface RawTitleLookup { + + /** @return raw material title, or {@code null} when the id is unknown */ + String titleFor(Long rawId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java new file mode 100644 index 00000000..57aef55b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/RawTitleLookups.java @@ -0,0 +1,42 @@ +package vip.mate.wiki.service; + +import vip.mate.wiki.dto.RawTitleRef; +import vip.mate.wiki.repository.WikiRawMaterialMapper; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; + +/** + * Factory helpers for {@link RawTitleLookup}. + */ +public final class RawTitleLookups { + + private RawTitleLookups() {} + + /** Lookup that always returns {@code null}; useful for callers without raw context. */ + public static RawTitleLookup empty() { + return id -> null; + } + + /** Lookup backed by a pre-resolved map (e.g. from an ingest-scope snapshot). */ + public static RawTitleLookup of(Map titlesById) { + Map snapshot = titlesById == null ? Map.of() : Map.copyOf(titlesById); + return snapshot::get; + } + + /** + * Preload titles for the given ids in a single batch query and return a + * lookup over the resulting map. Unknown ids resolve to {@code null}. + */ + public static RawTitleLookup preload(WikiRawMaterialMapper mapper, Collection rawIds) { + if (mapper == null || rawIds == null || rawIds.isEmpty()) { + return empty(); + } + Map titles = new HashMap<>(rawIds.size()); + for (RawTitleRef ref : mapper.selectBatchTitles(rawIds)) { + titles.put(ref.id(), ref.title()); + } + return of(titles); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java new file mode 100644 index 00000000..53428923 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingInputBuilder.java @@ -0,0 +1,82 @@ +package vip.mate.wiki.service; + +import org.springframework.stereotype.Component; +import vip.mate.wiki.model.WikiChunkEntity; + +/** + * Produces the text fed to the embedding model for a chunk. + *

+ * Naive content-only embeddings make short or context-poor chunks (e.g. a + * standalone sentence like "accuracy improved by 12%") near-indistinguishable + * in vector space. Prefixing the model input with already-available metadata — + * source title, header breadcrumb, source section, page number — preserves + * the semantic neighborhood the chunk came from without changing the storage + * model. + *

+ * Bump {@link #CURRENT_INPUT_VERSION} whenever the prefix format changes. The + * embedding pass treats any chunk whose stored {@code embedding_text_version} + * differs from the current value as stale and re-embeds it. + */ +@Component +public class WikiEmbeddingInputBuilder { + + /** + * Version tag stamped onto every chunk that this builder embeds. + * Increment when the prefix format below changes in a way that should + * trigger a re-embed pass. The string is opaque; "v1", "v2", ... is fine. + */ + public static final String CURRENT_INPUT_VERSION = "v1"; + + /** + * Build the embedding input string for a chunk. Metadata fields that are + * null or blank are skipped so empty values never produce stray headers. + * Falls back to the chunk content alone when no metadata is available. + */ + public String build(WikiChunkEntity chunk, RawTitleLookup lookup) { + if (chunk == null) { + return ""; + } + String content = chunk.getContent() == null ? "" : chunk.getContent(); + String prefix = buildPrefix(chunk, lookup); + return prefix.isEmpty() ? content : prefix + content; + } + + /** + * Build only the metadata prefix for a chunk. Useful when callers need to + * split content into sub-segments and prepend the prefix to each one so + * the metadata participates in every per-segment embedding before pooling. + * Returns an empty string when no metadata is available; otherwise ends + * with a blank line so the content reads as a separate paragraph. + */ + public String buildPrefix(WikiChunkEntity chunk, RawTitleLookup lookup) { + if (chunk == null) { + return ""; + } + StringBuilder sb = new StringBuilder(); + String rawTitle = (lookup == null || chunk.getRawId() == null) + ? null : lookup.titleFor(chunk.getRawId()); + appendLine(sb, "Source", rawTitle); + appendLine(sb, "Section", chunk.getHeaderBreadcrumb()); + appendLine(sb, "Subsection", chunk.getSourceSection()); + if (chunk.getPageNumber() != null) { + appendLine(sb, "Page", String.valueOf(chunk.getPageNumber())); + } + if (sb.length() == 0) { + return ""; + } + sb.append('\n'); + return sb.toString(); + } + + /** @return the version tag this builder stamps onto each chunk it embeds */ + public String currentVersion() { + return CURRENT_INPUT_VERSION; + } + + private static void appendLine(StringBuilder sb, String label, String value) { + if (value == null || value.isBlank()) { + return; + } + sb.append(label).append(": ").append(value.strip()).append('\n'); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java index 74aec2c7..b91458d9 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiEmbeddingService.java @@ -2,6 +2,7 @@ package vip.mate.wiki.service; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; +import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.ai.embedding.EmbeddingModel; @@ -16,12 +17,17 @@ import vip.mate.system.repository.SystemSettingMapper; import vip.mate.wiki.WikiProperties; import vip.mate.wiki.model.WikiChunkEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.repository.WikiChunkMapper; +import vip.mate.wiki.repository.WikiPageMapper; +import vip.mate.wiki.repository.WikiRawMaterialMapper; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; /** * RFC-011 + Embedding-UI-Config: Wiki 嵌入服务 @@ -42,16 +48,87 @@ import java.util.List; public class WikiEmbeddingService { private final WikiChunkMapper chunkMapper; + private final WikiPageMapper pageMapper; + private final WikiRawMaterialMapper rawMaterialMapper; private final WikiProperties properties; private final EmbeddingModelFactory factory; private final ModelConfigService modelConfigService; private final WikiKnowledgeBaseService kbService; private final SystemSettingMapper systemSettingMapper; private final vip.mate.llm.service.ModelProviderService modelProviderService; + private final WikiEmbeddingInputBuilder inputBuilder; /** 系统默认 embedding 模型的 mate_system_setting key */ public static final String SYSTEM_SETTING_DEFAULT_EMBEDDING_ID = "embedding.default.model.id"; + /** + * Returns the embedding input format version this service stamps onto + * each chunk. The builder's constant is the source of truth; the + * {@code mate.wiki.embedding-text-version-current} property exists only + * to support ops overrides and is validated against the builder at + * startup ({@link #verifyConfiguredInputVersion()}). + */ + public String currentInputVersion() { + String configured = properties.getEmbeddingTextVersionCurrent(); + return (configured == null || configured.isBlank()) ? inputBuilder.currentVersion() : configured.trim(); + } + + /** + * Validate the configured embedding input version against the builder + * constant on startup. A blank config is normal (the builder version is + * used). A config below the builder is allowed with a WARN so a KB can + * be embedded against an older format during a gradual rollback. A + * config above the builder fails fast — it almost always means the + * config was deployed ahead of the code. + */ + @PostConstruct + void verifyConfiguredInputVersion() { + String configured = properties.getEmbeddingTextVersionCurrent(); + if (configured == null || configured.isBlank()) { + log.info("[WikiEmbedding] Embedding input version: {} (from builder)", inputBuilder.currentVersion()); + return; + } + String builderVersion = inputBuilder.currentVersion(); + int cmp = compareInputVersions(configured.trim(), builderVersion); + if (cmp == 0) { + log.info("[WikiEmbedding] Embedding input version: {} (matches builder)", configured); + } else if (cmp < 0) { + log.warn("[WikiEmbedding] Configured embedding input version {} is older than builder {}; " + + "new embeddings will still be stamped with the configured value. " + + "Clear mate.wiki.embedding-text-version-current to use the builder default.", + configured, builderVersion); + } else { + throw new IllegalStateException( + "Configured embedding input version " + configured + " is newer than builder version " + + builderVersion + ". The builder code is older than the deployment config; " + + "upgrade the application or clear mate.wiki.embedding-text-version-current."); + } + } + + /** + * Compare version tags of the form {@code v\d+} numerically (so v2 > v10 + * does not happen). Falls back to case-insensitive string compare when + * either side does not match the expected pattern. + */ + static int compareInputVersions(String a, String b) { + Integer ai = parseNumericVersion(a); + Integer bi = parseNumericVersion(b); + if (ai != null && bi != null) { + return Integer.compare(ai, bi); + } + return a.compareToIgnoreCase(b); + } + + private static Integer parseNumericVersion(String tag) { + if (tag == null || tag.length() < 2) return null; + if (tag.charAt(0) != 'v' && tag.charAt(0) != 'V') return null; + try { + return Integer.parseInt(tag.substring(1)); + } catch (NumberFormatException e) { + return null; + } + } + /** * 判断全局是否有可用的 embedding 能力(任何 enabled 的 embedding 模型配置) */ @@ -116,8 +193,10 @@ public class WikiEmbeddingService { /** * 批量嵌入指定 KB 中缺失 embedding 的 chunk。 *

- * 只嵌入 embedding 为 NULL 或 embeddingModel 与当前解析出的模型不一致的 chunk。 - * 模型切换时自动触发全量重嵌(通过 embedding_model 字段比对)。 + * Pending criteria: embedding is NULL, the stored embedding_model differs + * from the currently-resolved model, or the stored embedding_text_version + * differs from the active builder version. Switching the embedding model + * or bumping the input format both trigger a full re-embed pass. */ public int embedMissingChunks(Long kbId) { Resolved r = resolveForKb(kbId); @@ -127,20 +206,29 @@ public class WikiEmbeddingService { } String modelName = r.modelName(); + String inputVersion = currentInputVersion(); List pending = chunkMapper.selectList( new LambdaQueryWrapper() .eq(WikiChunkEntity::getKbId, kbId) .and(w -> w.isNull(WikiChunkEntity::getEmbedding) - .or().ne(WikiChunkEntity::getEmbeddingModel, modelName))); + .or().ne(WikiChunkEntity::getEmbeddingModel, modelName) + .or().isNull(WikiChunkEntity::getEmbeddingTextVersion) + .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion))); if (pending.isEmpty()) { log.debug("[WikiEmbedding] No chunks need embedding for kbId={}", kbId); return 0; } + RawTitleLookup titleLookup = preloadTitlesFor(pending); int batchSize = Math.max(1, properties.getEmbeddingBatchSize()); int maxChars = Math.max(500, properties.getEmbeddingMaxChars()); + int threshold = Math.max(1, properties.getEmbeddingConsecutiveFailureThreshold()); int total = 0; + // Consecutive failure counter: resets on any successful batch / long + // chunk, increments when a unit returns zero progress. Crossing the + // threshold trips the circuit and aborts the rest of this pass. + int consecutiveFailures = 0; for (int offset = 0; offset < pending.size(); offset += batchSize) { List batch = pending.subList(offset, Math.min(offset + batchSize, pending.size())); @@ -160,13 +248,30 @@ public class WikiEmbeddingService { // Short chunks: existing batch path if (!shortBatch.isEmpty()) { - total += embedShortBatch(shortBatch, r.model(), modelName, kbId); + int embedded = embedShortBatch(shortBatch, r.model(), modelName, kbId, inputVersion, titleLookup); + total += embedded; + if (embedded == 0) { + consecutiveFailures++; + if (consecutiveFailures >= threshold) { + int remaining = pending.size() - (offset + batch.size()); + throw circuitOpen(kbId, modelName, consecutiveFailures, remaining); + } + } else { + consecutiveFailures = 0; + } } // Long chunks: each goes through sub-segment split + mean pool for (WikiChunkEntity longChunk : longChunks) { - if (embedLongChunk(longChunk, r.model(), modelName, maxChars)) { + if (embedLongChunk(longChunk, r.model(), modelName, maxChars, inputVersion, titleLookup)) { total++; + consecutiveFailures = 0; + } else { + consecutiveFailures++; + if (consecutiveFailures >= threshold) { + int remaining = pending.size() - (offset + batch.size()); + throw circuitOpen(kbId, modelName, consecutiveFailures, remaining); + } } } } @@ -181,21 +286,34 @@ public class WikiEmbeddingService { return total; } + private vip.mate.wiki.job.WikiEmbeddingProviderFailingException circuitOpen( + Long kbId, String modelName, int failures, int remaining) { + String message = "Embedding provider unavailable: " + failures + + " consecutive batch failures (kbId=" + kbId + ", model=" + modelName + + "). Aborted with " + remaining + " chunk(s) still pending."; + log.warn("[WikiEmbedding] Circuit opened — {}", message); + return new vip.mate.wiki.job.WikiEmbeddingProviderFailingException(message, failures, remaining); + } + /** * Embed a batch of chunks whose content fits within the per-segment char limit. * One API call per batch; individual results are persisted independently. * Returns the number of chunks that were successfully embedded and persisted. */ private int embedShortBatch(List batch, EmbeddingModel model, - String modelName, Long kbId) { + String modelName, Long kbId, + String inputVersion, RawTitleLookup titleLookup) { try { - List inputs = batch.stream().map(WikiChunkEntity::getContent).toList(); + List inputs = batch.stream() + .map(c -> inputBuilder.build(c, titleLookup)) + .toList(); EmbeddingResponse resp = model.call(new EmbeddingRequest(inputs, null)); for (int i = 0; i < batch.size(); i++) { float[] vec = resp.getResults().get(i).getOutput(); WikiChunkEntity chunk = batch.get(i); chunk.setEmbedding(floatsToBytes(vec)); chunk.setEmbeddingModel(modelName); + chunk.setEmbeddingTextVersion(inputVersion); chunkMapper.updateById(chunk); } return batch.size(); @@ -217,12 +335,24 @@ public class WikiEmbeddingService { * Returns true if at least one sub-segment succeeded and the chunk was persisted. */ private boolean embedLongChunk(WikiChunkEntity chunk, EmbeddingModel model, - String modelName, int maxChars) { - List segments = splitForEmbedding(chunk.getContent(), maxChars); - if (segments.isEmpty()) { + String modelName, int maxChars, + String inputVersion, RawTitleLookup titleLookup) { + // Prepend the metadata prefix to every sub-segment so the per-segment + // embeddings carry the same context before mean-pooling. The split + // budget is reduced by the prefix length to keep each enriched segment + // under the provider's per-input cap; the floor of 500 keeps the + // splitter from collapsing to single-char windows when a pathological + // metadata prefix appears. + String prefix = inputBuilder.buildPrefix(chunk, titleLookup); + int segmentBudget = Math.max(500, maxChars - prefix.length()); + List rawSegments = splitForEmbedding(chunk.getContent(), segmentBudget); + if (rawSegments.isEmpty()) { log.warn("[WikiEmbedding] Chunk {} produced no embeddable segments after split", chunk.getId()); return false; } + List segments = prefix.isEmpty() + ? rawSegments + : rawSegments.stream().map(s -> prefix + s).toList(); log.info("[WikiEmbedding] Chunk {} ({} chars) split into {} sub-segments", chunk.getId(), chunk.getContent().length(), segments.size()); @@ -255,6 +385,7 @@ public class WikiEmbeddingService { float[] pooled = averageAndNormalize(vectors); chunk.setEmbedding(floatsToBytes(pooled)); chunk.setEmbeddingModel(modelName); + chunk.setEmbeddingTextVersion(inputVersion); chunkMapper.updateById(chunk); return true; } @@ -323,6 +454,74 @@ public class WikiEmbeddingService { return end; // hard cut } + /** + * Embed a wiki page's content directly so the semantic retriever can match + * vocabulary that exists in the synthesised page but not in any source + * raw's chunks (typical for transformation-generated synthesis pages). + * Idempotent: skips when the stored embedding is already current for the + * resolved model + input version. + * + * @return {@code true} when the page row was updated with a fresh embedding + */ + public boolean embedPage(Long pageId) { + if (pageId == null) return false; + WikiPageEntity page = pageMapper.selectById(pageId); + if (page == null) { + log.warn("[WikiEmbedding] embedPage: page not found id={}", pageId); + return false; + } + Resolved r = resolveForKb(page.getKbId()); + if (r == null) { + log.debug("[WikiEmbedding] embedPage: no embedding model for kbId={}", page.getKbId()); + return false; + } + String inputVersion = currentInputVersion(); + // Short-circuit when this page is already embedded against the same + // model + input format — nothing to do. + if (page.getEmbedding() != null + && r.modelName().equals(page.getEmbeddingModel()) + && inputVersion.equals(page.getEmbeddingTextVersion())) { + return false; + } + + String input = buildPageEmbeddingInput(page); + if (input.isBlank()) return false; + int maxChars = Math.max(500, properties.getEmbeddingMaxChars()); + if (input.length() > maxChars) input = input.substring(0, maxChars); + + try { + EmbeddingResponse resp = r.model().call(new EmbeddingRequest(List.of(input), null)); + float[] vec = resp.getResults().get(0).getOutput(); + page.setEmbedding(floatsToBytes(vec)); + page.setEmbeddingModel(r.modelName()); + page.setEmbeddingTextVersion(inputVersion); + pageMapper.updateById(page); + log.info("[WikiEmbedding] Embedded page id={} kbId={} model={} ({} chars)", + pageId, page.getKbId(), r.modelName(), input.length()); + return true; + } catch (Exception e) { + log.warn("[WikiEmbedding] embedPage failed id={}: {}", pageId, e.getMessage()); + return false; + } + } + + /** Concatenates the fields that best capture a page's topic — title + + * summary + content prefix — so the embedding picks up both the + * vocabulary the LLM authored and the source-derived material. */ + private String buildPageEmbeddingInput(WikiPageEntity page) { + StringBuilder sb = new StringBuilder(); + if (page.getTitle() != null && !page.getTitle().isBlank()) { + sb.append("# ").append(page.getTitle()).append("\n\n"); + } + if (page.getSummary() != null && !page.getSummary().isBlank()) { + sb.append(page.getSummary()).append("\n\n"); + } + if (page.getContent() != null && !page.getContent().isBlank()) { + sb.append(page.getContent()); + } + return sb.toString(); + } + /** * 查询向量化(混合搜索时调用,需指定 KB 以便解析对应模型) */ @@ -343,6 +542,51 @@ public class WikiEmbeddingService { } } + /** + * Snapshot of how many chunks in a KB still need to be re-embedded + * against the current model + input version. Powers the admin "embedding + * drift" indicator without exposing internal pending logic. + */ + public EmbeddingDrift describeDrift(Long kbId) { + String inputVersion = currentInputVersion(); + Resolved r = resolveForKb(kbId); + String modelName = r == null ? null : r.modelName(); + + long totalEmbedded = chunkMapper.selectCount( + new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId) + .isNotNull(WikiChunkEntity::getEmbedding)); + + LambdaQueryWrapper pendingQ = new LambdaQueryWrapper() + .eq(WikiChunkEntity::getKbId, kbId) + .and(w -> { + w.isNull(WikiChunkEntity::getEmbedding) + .or().isNull(WikiChunkEntity::getEmbeddingTextVersion) + .or().ne(WikiChunkEntity::getEmbeddingTextVersion, inputVersion); + if (modelName != null) { + w.or().ne(WikiChunkEntity::getEmbeddingModel, modelName); + } + }); + List pending = chunkMapper.selectList(pendingQ); + + long pendingChars = 0; + for (WikiChunkEntity c : pending) { + if (c.getContent() != null) pendingChars += c.getContent().length(); + } + // Provider-agnostic token approximation; ~4 chars per token covers + // English and is conservative for Chinese (which is denser per token). + long pendingTokens = pendingChars / 4; + + return new EmbeddingDrift(inputVersion, pending.size(), totalEmbedded, pendingTokens); + } + + /** Result of {@link #describeDrift(Long)}; serialized into KB stats. */ + public record EmbeddingDrift( + String currentEmbeddingTextVersion, + int pendingReembedChunks, + long totalEmbeddedChunks, + long pendingReembedEstimatedTokens) {} + /** * 清空指定 KB 的所有 embedding(模型切换时调用) */ @@ -350,10 +594,22 @@ public class WikiEmbeddingService { chunkMapper.update(null, new LambdaUpdateWrapper() .eq(WikiChunkEntity::getKbId, kbId) .set(WikiChunkEntity::getEmbedding, null) - .set(WikiChunkEntity::getEmbeddingModel, null)); + .set(WikiChunkEntity::getEmbeddingModel, null) + .set(WikiChunkEntity::getEmbeddingTextVersion, null)); log.info("[WikiEmbedding] Cleared all embeddings for kbId={}", kbId); } + private RawTitleLookup preloadTitlesFor(List chunks) { + if (chunks == null || chunks.isEmpty()) { + return RawTitleLookups.empty(); + } + Set rawIds = new HashSet<>(); + for (WikiChunkEntity c : chunks) { + if (c.getRawId() != null) rawIds.add(c.getRawId()); + } + return RawTitleLookups.preload(rawMaterialMapper, rawIds); + } + // ==================== 私有 helper ==================== private ModelConfigEntity safeGetModel(Long id) { diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java index 9ea5bf25..996e7e4c 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiProcessingService.java @@ -114,6 +114,15 @@ public class WikiProcessingService { @org.springframework.beans.factory.annotation.Autowired(required = false) private WikiLogService logService; + /** + * Optional. When present, every successful ingest triggers an async sweep + * of the KB's apply-default transformation templates. Missing in the + * legacy unit tests that wire this service directly. + */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + @org.springframework.context.annotation.Lazy + private WikiTransformationExecutor transformationExecutor; + /** Parallel chunk / material processing executor (JDK 21 virtual threads) */ public static final ExecutorService WIKI_EXECUTOR = Executors.newVirtualThreadPerTaskExecutor(); @@ -303,7 +312,17 @@ public class WikiProcessingService { String finalStatus; String finalDetail = null; - if (totalPages == 0) { + // Cancellation takes precedence over the normal terminal-state logic: + // chunks that observed the cancel flag returned early as "failed", but + // those aren't real failures — the user asked to stop. Surface that + // intent explicitly so the UI can show "cancelled" instead of "failed" + // or "partial". + if (rawService.isCancelRequested(rawId)) { + finalDetail = "Cancelled by user (" + totalPages + " page(s) generated, " + + (totalChunks - failedChunks) + "/" + totalChunks + " chunks completed before stop)."; + rawService.updateProcessingStatus(rawId, "cancelled", finalDetail); + finalStatus = "cancelled"; + } else if (totalPages == 0) { // RFC-051 follow-up: previously this was an unconditional "failed". // But chunks were already persisted (and the materials are searchable // via wiki_semantic_search) — the only thing that actually went wrong @@ -371,34 +390,45 @@ public class WikiProcessingService { var terminalStage = switch (finalStatus) { case "failed" -> vip.mate.wiki.job.WikiJobStage.FAILED; case "partial" -> vip.mate.wiki.job.WikiJobStage.PARTIAL; + case "cancelled" -> vip.mate.wiki.job.WikiJobStage.CANCELLED; default -> vip.mate.wiki.job.WikiJobStage.COMPLETED; }; wikiJobService.transition(jobId, terminalStage); } catch (Exception ignored) {} } - // RFC-051 PR-2c: log every non-failed eager ingest. Failures already get a - // RAW_FAILED broadcast and an error message in the raw row. Title goes first - // so the log reads as "what just landed" instead of an opaque raw id. - if (logService != null && !"failed".equals(finalStatus)) { + // Skip the post-terminal side effects (log line, overview rebuild, + // KB-dirty event) for cancelled and failed runs. A cancelled run + // means the user explicitly stopped — don't burn LLM tokens on + // overview regeneration over an unstable partial state. + boolean nonTerminalSideEffects = !"failed".equals(finalStatus) && !"cancelled".equals(finalStatus); + if (logService != null && nonTerminalSideEffects) { String title = (raw.getTitle() == null || raw.getTitle().isBlank()) ? ("raw#" + rawId) : raw.getTitle(); logService.append(kb.getId(), WikiLogService.EventType.INGEST, "eager " + finalStatus + " · " + title + " · " + totalPages + " pages · " + totalChunks + " chunks"); } - // RFC-051 PR-2b: refresh overview stats whenever a raw lands in a terminal state - // (completed or partial). Failures don't shift the stats meaningfully. - if (overviewService != null && !"failed".equals(finalStatus)) { + // Refresh overview stats whenever a raw lands in a terminal state + // (completed or partial). Failures and cancellations don't shift the stats meaningfully. + if (overviewService != null && nonTerminalSideEffects) { overviewService.rebuild(kb.getId()); } // Tier 2: signal "KB content is dirty" so WikiNarrativeService can // schedule (debounced) an LLM-generated overview narrative refresh. // Stats rebuild above is sync; narrative regen runs after-commit. - if (!"failed".equals(finalStatus)) { + if (nonTerminalSideEffects) { eventPublisher.publishEvent(new vip.mate.wiki.event.WikiKbDirtyEvent(this, kb.getId())); } + // Run apply-default transformation templates against the newly + // ingested raw material. Fire-and-forget; failures are logged + // inside the executor and do not affect the ingest outcome. + if (transformationExecutor != null && nonTerminalSideEffects) { + Long wsId = kb.getWorkspaceId() == null ? 1L : kb.getWorkspaceId(); + transformationExecutor.runDefaultsAsync(kb.getId(), wsId, rawId, "apply_default"); + } + log.info("[Wiki] Processing completed for raw={}, kbId={}, generatedPages={}, totalPages={}", rawId, kb.getId(), totalPages, pageCount); @@ -410,7 +440,12 @@ public class WikiProcessingService { // RFC-051 follow-up: trigger embedding whenever chunks landed, not only when // pages were produced. Otherwise the partial-with-no-pages case above ends up // with chunks in DB but never embedded, so semantic search silently misses them. - if (totalChunks > 0) { + // Skip the post-ingest embedding sweep when this run was cancelled. + // The user almost certainly stopped because the embedding provider + // is failing (out of credits, wrong key, etc.); kicking off another + // embedding pass on the same provider would just churn through + // every pending chunk and produce more "all chunks failed" noise. + if (totalChunks > 0 && !"cancelled".equals(finalStatus)) { final Long fKbId = kb.getId(); WIKI_EXECUTOR.submit(() -> { try { @@ -418,6 +453,12 @@ public class WikiProcessingService { if (embedded > 0) { log.info("[Wiki] Async embedding completed: kbId={}, embedded={}", fKbId, embedded); } + } catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) { + // Circuit-breaker tripped — the provider has consistently failed. + // The exception's own log line in WikiEmbeddingService is enough; + // emit a calmer notice here instead of a generic failure log. + log.warn("[Wiki] Async embedding aborted by circuit-breaker for kbId={}: {}", + fKbId, ex.getMessage()); } catch (Exception ex) { log.warn("[Wiki] Async embedding failed for kbId={}: {}", fKbId, ex.getMessage()); } @@ -425,16 +466,39 @@ public class WikiProcessingService { } } catch (Exception e) { - log.error("[Wiki] Processing failed for raw={}: {}", rawId, e.getMessage(), e); - rawService.updateProcessingStatus(rawId, "failed", e.getMessage()); - kbService.updateStatus(kb.getId(), "active"); - // Transition job to failed - if (wikiJobService != null && jobId != null) { - try { wikiJobService.transition(jobId, vip.mate.wiki.job.WikiJobStage.FAILED); } catch (Exception ignored) {} + // If the user requested cancellation while this run was in flight, + // surface the abort as 'cancelled' rather than 'failed' even when + // the exception bubbled up from somewhere mid-pipeline (e.g. a + // checkpoint rejected between chunks). + boolean cancelled = rawService.isCancelRequested(rawId); + String terminalStatus = cancelled ? "cancelled" : "failed"; + String detail = cancelled + ? "Cancelled by user (interrupted: " + (e.getMessage() == null ? "unknown" : e.getMessage()) + ")" + : e.getMessage(); + if (cancelled) { + log.info("[Wiki] Processing cancelled for raw={}: {}", rawId, e.getMessage()); + } else { + log.error("[Wiki] Processing failed for raw={}: {}", rawId, e.getMessage(), e); + } + rawService.updateProcessingStatus(rawId, terminalStatus, detail); + kbService.updateStatus(kb.getId(), "active"); + if (wikiJobService != null && jobId != null) { + try { + wikiJobService.transition(jobId, cancelled + ? vip.mate.wiki.job.WikiJobStage.CANCELLED + : vip.mate.wiki.job.WikiJobStage.FAILED); + } catch (Exception ignored) {} + } + // Broadcast: cancelled rows reuse the COMPLETED event with status="cancelled" + // so subscribers can render the terminal-but-not-error UI; only true failures + // go through RAW_FAILED (which the UI surfaces as a red banner). + if (cancelled) { + progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_COMPLETED, + java.util.Map.of("rawId", rawId, "status", "cancelled")); + } else { + progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED, + java.util.Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage())); } - // RFC-012 M3:广播异常终态 - progressBus.broadcast(kb.getId(), WikiProgressBus.EVENT_RAW_FAILED, - java.util.Map.of("rawId", rawId, "error", e.getMessage() == null ? "unknown" : e.getMessage())); } finally { // RFC-012 M2 v2 UI v2:写入最终进度并清理共享计数器 ProgressCounter pc = progressCounters.remove(rawId); @@ -2164,10 +2228,15 @@ public class WikiProcessingService { * @return {@code true} if the raw is gone; caller should stop work */ private boolean isAborted(Long rawId, String ctx) { - if (rawService.getById(rawId) == null) { + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw == null) { log.info("[Wiki] Aborting {} for raw={}: raw was deleted mid-processing", ctx, rawId); return true; } + if (Boolean.TRUE.equals(raw.getCancelRequested())) { + log.info("[Wiki] Aborting {} for raw={}: cancellation requested by user", ctx, rawId); + return true; + } return false; } @@ -2260,6 +2329,9 @@ public class WikiProcessingService { if (embedded > 0) { log.info("[Wiki] Lazy async embedding completed: kbId={}, embedded={}", fKbId, embedded); } + } catch (vip.mate.wiki.job.WikiEmbeddingProviderFailingException ex) { + log.warn("[Wiki] Lazy async embedding aborted by circuit-breaker for kbId={}: {}", + fKbId, ex.getMessage()); } catch (Exception ex) { log.warn("[Wiki] Lazy async embedding failed for kbId={}: {}", fKbId, ex.getMessage()); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java index f71f158b..ba2ea7b8 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiRawMaterialService.java @@ -232,10 +232,47 @@ public class WikiRawMaterialService { entity.setProgressPhase(null); entity.setProgressTotal(0); entity.setProgressDone(0); + // Fresh start clears any stale cancel request from a previous run. + entity.setCancelRequested(Boolean.FALSE); rawMapper.updateById(entity); return true; } + /** + * Mark a raw material for cancellation. Only valid while it is currently + * being processed; for any other status this is a no-op so the call is + * idempotent and safe to retry from the UI. + * + * @return {@code true} if the flag was set, {@code false} otherwise + */ + @Transactional + public boolean requestCancel(Long id) { + WikiRawMaterialEntity entity = rawMapper.selectById(id); + if (entity == null) { + return false; + } + if (!"processing".equals(entity.getProcessingStatus())) { + return false; + } + if (Boolean.TRUE.equals(entity.getCancelRequested())) { + // Already requested; treat as success without redundant write. + return true; + } + entity.setCancelRequested(Boolean.TRUE); + rawMapper.updateById(entity); + return true; + } + + /** + * Returns {@code true} if the user has asked to cancel this raw material's + * current processing run. Used by abort checkpoints inside the processing + * pipeline to bail out early. + */ + public boolean isCancelRequested(Long id) { + WikiRawMaterialEntity entity = rawMapper.selectById(id); + return entity != null && Boolean.TRUE.equals(entity.getCancelRequested()); + } + /** * RFC-012 M2 v2 UI:更新 wiki 两阶段消化的进度字段。 *

@@ -266,6 +303,12 @@ public class WikiRawMaterialService { if ("completed".equals(status)) { entity.setLastProcessedAt(java.time.LocalDateTime.now()); } + // Cancellation flag is only meaningful while a row is being processed. + // Any transition out of 'processing' clears it so the field reflects + // an idle row's true state and the next reprocess starts clean. + if (!"processing".equals(status)) { + entity.setCancelRequested(Boolean.FALSE); + } rawMapper.updateById(entity); } @@ -368,9 +411,13 @@ public class WikiRawMaterialService { log.warn("[Wiki] Failed to cascade-delete chunks for raw={}: {}", id, e.getMessage()); } - // Source file last — DB pointer is gone, no other row references this - // path (each upload gets a timestamp-prefixed unique name), so - // leaving it on disk would just accumulate as the upload tree grows. + // Source file last. cleanupFile is sandboxed to the upload dir, so: + // - uploaded raws (server-managed copy under uploadDir) are removed — + // each upload has a timestamp-prefixed unique name, no other row + // references it, leaving it would just accumulate disk garbage. + // - directory-scanned raws (sourcePath points at the user's own file + // outside uploadDir) are left untouched — the scanner references + // the original in place; the user's file is theirs to keep. // Failure here is soft-logged and non-blocking — operator can run a // sweep later if disk usage matters more than the delete RTT. if (entity != null) { @@ -593,16 +640,37 @@ public class WikiRawMaterialService { } /** - * Best-effort delete of an upload-tree file. Used both when a fresh - * upload turns out to be a duplicate (the new file is redundant) and - * when a raw material row is deleted (its source file becomes a - * disk orphan with no DB pointer to it). Idempotent — silently - * succeeds when the path is null or the file is already gone. + * Best-effort delete of a raw material's source file on disk. Used both + * when a fresh upload turns out to be a duplicate (the new file is + * redundant) and when a raw material row is deleted (its source file + * becomes a disk orphan with no DB pointer to it). + *

+ * Sandboxed to {@link WikiProperties#getUploadDir()}: only deletes files + * that live under the configured upload directory — i.e. files this + * service is responsible for (uploaded raws + KB pipeline outputs). + * Files outside the upload tree are left alone, because the directory + * scanner imports raws by referencing the user's local file in place + * (no copy); deleting them would wipe the user's original document, not + * just our internal cache. See {@code WikiDirectoryScanService}. + *

+ * Idempotent — silently succeeds when the path is null, the file is + * already gone, or the path is outside the upload sandbox. */ private void cleanupFile(String path) { if (path == null || path.isBlank()) return; try { - java.nio.file.Files.deleteIfExists(java.nio.file.Paths.get(path)); + java.nio.file.Path target = java.nio.file.Paths.get(path).toAbsolutePath().normalize(); + java.nio.file.Path uploadRoot = java.nio.file.Paths.get(properties.getUploadDir()) + .toAbsolutePath().normalize(); + if (!target.startsWith(uploadRoot)) { + // User-owned file (imported by directory scan in place). The DB row is + // gone but the file on the user's disk must stay — that's their data, + // not ours. + log.info("[Wiki] Skip file delete (outside upload dir, user-owned): path={} uploadDir={}", + target, uploadRoot); + return; + } + java.nio.file.Files.deleteIfExists(target); } catch (Exception e) { log.warn("[Wiki] Failed to clean up upload file {}: {}", path, e.getMessage()); } diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationAggregator.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationAggregator.java new file mode 100644 index 00000000..5e21a5f4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationAggregator.java @@ -0,0 +1,193 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Map-reduce a single transformation template across all completed runs in + * a KB: produces one synthesis wiki page that unifies the per-source + * outputs. The map step is already done by the executor — each run carries + * its per-source output. This service is the reduce step: load the runs, + * stack them with source labels, ask an LLM to merge + dedupe, and persist + * the merged document as a synthesis page slugged + * {@code -aggregate}. + * + *

Idempotent: re-running upserts the same slug, so the aggregate page + * stays current as new runs land. Page-level embedding + reverse-citation + * extraction run the same way they do for single-source synthesis pages. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationAggregator { + + /** Hard cap on combined input chars fed to the LLM merge call. */ + private static final int MAX_AGG_INPUT_CHARS = 80_000; + + /** Per-source output is truncated to keep the merge prompt within the cap when many sources exist. */ + private static final int PER_SOURCE_SOFT_CAP = 12_000; + + private final WikiTransformationService transformationService; + private final WikiRawMaterialService rawService; + private final WikiPageService pageService; + + @Autowired(required = false) + private WikiModelRoutingService modelRoutingService; + + @Autowired(required = false) + private WikiEmbeddingService embeddingService; + + private final com.fasterxml.jackson.databind.ObjectMapper objectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public record Result(Long pageId, String slug, String title, + int sourcesUsed, int charsFed, boolean created) { + public static Result empty(String reason) { + return new Result(null, null, reason, 0, 0, false); + } + } + + public Result aggregate(WikiTransformationEntity template, Long kbId, String triggeredBy) { + if (template == null) throw new IllegalArgumentException("template is required"); + if (kbId == null) throw new IllegalArgumentException("kbId is required"); + if (modelRoutingService == null) throw new IllegalStateException("ModelRoutingService unavailable"); + + // Load every completed run for this template against this KB. Cap at + // 100 sources so a degenerate KB doesn't push past the input window. + List runs = transformationService + .listRunsByTransformation(template.getId(), 200) + .stream() + .filter(r -> "completed".equalsIgnoreCase(r.getStatus()) + && kbId.equals(r.getKbId()) + && r.getOutput() != null && !r.getOutput().isBlank() + && r.getRawId() != null) + .limit(100) + .toList(); + if (runs.isEmpty()) { + return Result.empty("no completed runs to aggregate"); + } + + // Deduplicate by rawId — only the most recent completed run per raw + // contributes, so a template that's been re-run several times against + // the same source doesn't get its old outputs included. + java.util.Map latestByRaw = new java.util.LinkedHashMap<>(); + for (WikiTransformationRunEntity r : runs) { + latestByRaw.putIfAbsent(r.getRawId(), r); // listRunsByTransformation orders DESC by createTime + } + List distinct = new ArrayList<>(latestByRaw.values()); + + // Build the per-source block, truncating each section to keep the + // merged prompt within the model's context window. + StringBuilder outputs = new StringBuilder(); + Set sourceRawIds = new LinkedHashSet<>(); + int totalChars = 0; + int sourcesIncluded = 0; + for (WikiTransformationRunEntity run : distinct) { + WikiRawMaterialEntity raw = rawService.getById(run.getRawId()); + String sourceTitle = raw != null && raw.getTitle() != null && !raw.getTitle().isBlank() + ? raw.getTitle() : ("raw#" + run.getRawId()); + String body = run.getOutput().length() > PER_SOURCE_SOFT_CAP + ? run.getOutput().substring(0, PER_SOURCE_SOFT_CAP) + "\n…(truncated for merge)" + : run.getOutput(); + String block = "### From: " + sourceTitle + "\n\n" + body + "\n\n---\n\n"; + if (totalChars + block.length() > MAX_AGG_INPUT_CHARS) { + log.warn("[WikiAggregator] template={} kb={} stopping merge at {} sources to stay under {} chars", + template.getName(), kbId, sourcesIncluded, MAX_AGG_INPUT_CHARS); + break; + } + outputs.append(block); + sourceRawIds.add(run.getRawId()); + totalChars += block.length(); + sourcesIncluded++; + } + if (sourcesIncluded == 0) return Result.empty("all source outputs were empty after dedup"); + + // LLM call + String systemPrompt = PromptLoader.loadPrompt("wiki/transformation-aggregate-system"); + String userPrompt = PromptLoader.loadPrompt("wiki/transformation-aggregate-user") + .replace("{template_title}", template.getTitle() == null ? template.getName() : template.getTitle()) + .replace("{template_description}", template.getDescription() == null ? "" : template.getDescription()) + .replace("{outputs}", outputs.toString()); + + Long modelId = template.getModelId() != null + ? template.getModelId() + : modelRoutingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.CREATE_PAGE); + ChatModel chatModel = modelRoutingService.buildChatModel(modelId); + + ChatResponse resp = chatModel.call(new Prompt(List.of( + new SystemMessage(systemPrompt), new UserMessage(userPrompt)))); + String mergedOutput = (resp == null || resp.getResult() == null + || resp.getResult().getOutput() == null) ? null : resp.getResult().getOutput().getText(); + if (mergedOutput == null || mergedOutput.isBlank()) { + throw new IllegalStateException("Aggregator LLM returned empty output"); + } + mergedOutput = WikiTransformationExecutor.cleanLlmOutput(mergedOutput); + + // Upsert the aggregate page on a deterministic slug so re-aggregation + // refreshes it in place instead of spawning duplicates. + String slug = template.getName() + "-aggregate"; + String title = (template.getTitle() == null ? template.getName() : template.getTitle()) + + "(KB 聚合)"; + String summary = sourcesIncluded + " 个原始材料合并 · " + + (triggeredBy == null ? "manual" : triggeredBy); + String sourceRawIdsJson = toJsonArray(new ArrayList<>(sourceRawIds)); + + WikiPageEntity existing = pageService.getBySlug(kbId, slug); + WikiPageEntity persisted; + boolean created; + if (existing == null) { + persisted = pageService.createPage(kbId, slug, title, mergedOutput, summary, + sourceRawIdsJson, "synthesis"); + created = true; + } else { + persisted = pageService.updatePageByAi(kbId, slug, mergedOutput, summary, + sourceRawIds.iterator().next()); + if (persisted == null) persisted = existing; + created = false; + } + log.info("[WikiAggregator] {} aggregate page slug={} for template={} kb={} ({} sources, {} chars in)", + created ? "created" : "updated", slug, template.getName(), kbId, + sourcesIncluded, totalChars); + + // Fire-and-forget page embed so the aggregate joins semantic search. + if (embeddingService != null) { + final Long pid = persisted.getId(); + Thread.startVirtualThread(() -> { + try { embeddingService.embedPage(pid); } + catch (Exception ee) { + log.warn("[WikiAggregator] post-aggregate embed failed pageId={}: {}", + pid, ee.getMessage()); + } + }); + } + + return new Result(persisted.getId(), slug, title, sourcesIncluded, totalChars, created); + } + + private String toJsonArray(List ids) { + try { return objectMapper.writeValueAsString(ids); } + catch (Exception e) { + return ids.toString().replace(" ", ""); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationCitationExtractor.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationCitationExtractor.java new file mode 100644 index 00000000..6c8a77ac --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationCitationExtractor.java @@ -0,0 +1,139 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.wiki.model.WikiChunkEntity; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Reverse-citation parser: scans the markdown output of a transformation run + * for references that point back into the source raw material (e.g. + * {@code 第 14 页}, {@code page 14}, {@code 示例题号 1, 5}, {@code 第 5 题}) + * and resolves each reference to a chunk in the source raw. + * + *

The resolved chunk IDs are passed to + * {@link WikiCitationService#buildCitations(Long, Long, List)} so the + * synthesis page cites only the specific chunks the LLM said it relied on + * rather than every chunk of the source raw — this keeps the citation + * graph (and the relation signals derived from shared citations) clean. + * + *

When no parseable references are found the extractor returns {@code 0} + * without touching existing citations; the caller can decide whether to + * fall back to the raw-level default citation build. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationCitationExtractor { + + private final WikiChunkService chunkService; + private final WikiCitationService citationService; + + /** {@code 第 N 页} / {@code 页 N} / {@code page N} / {@code p.N} / {@code p N} */ + private static final Pattern PAGE_REF = Pattern.compile( + "(?:第\\s*(\\d+)\\s*页|页\\s+(\\d+)|[Pp]age\\s+(\\d+)|p\\.\\s*(\\d+)|p\\s+(\\d+))"); + + /** {@code 第 N 题} / {@code 例(题)? N} / {@code 题 N} / {@code Problem N} / {@code Example N} */ + private static final Pattern PROBLEM_REF = Pattern.compile( + "(?:第\\s*(\\d+)\\s*题|例(?:题)?\\s*(\\d+)|题\\s+(\\d+)|[Pp]roblem\\s+(\\d+)|[Ee]xample\\s+(\\d+))"); + + /** + * Run extract → resolve → write. Returns the count of chunk citations + * actually written. Best-effort: any internal exception is logged and + * the call returns 0 so callers can fall back without surfacing the + * error to the user. + */ + public int extractAndApply(Long pageId, Long kbId, Long sourceRawId, String output) { + if (pageId == null || kbId == null || sourceRawId == null) return 0; + if (output == null || output.isBlank()) return 0; + try { + Set pageRefs = parseNumeric(output, PAGE_REF); + Set problemRefs = parseNumeric(output, PROBLEM_REF); + if (pageRefs.isEmpty() && problemRefs.isEmpty()) return 0; + + List chunks = chunkService.listByRawId(sourceRawId); + if (chunks.isEmpty()) return 0; + + Set hitIds = new LinkedHashSet<>(); + for (WikiChunkEntity chunk : chunks) { + if (chunkMatches(chunk, pageRefs, problemRefs)) { + hitIds.add(chunk.getId()); + } + } + if (hitIds.isEmpty()) return 0; + + citationService.buildCitations(pageId, kbId, new ArrayList<>(hitIds)); + log.info("[WikiCitationExtractor] page={} kb={} cited {} chunks " + + "(pageRefs={}, problemRefs={})", + pageId, kbId, hitIds.size(), pageRefs, problemRefs); + return hitIds.size(); + } catch (Exception e) { + log.warn("[WikiCitationExtractor] extract failed pageId={}: {}", pageId, e.getMessage()); + return 0; + } + } + + /** Collect every integer captured by any group of the supplied pattern. */ + private static Set parseNumeric(String text, Pattern pattern) { + Set out = new LinkedHashSet<>(); + Matcher m = pattern.matcher(text); + while (m.find()) { + for (int i = 1; i <= m.groupCount(); i++) { + String g = m.group(i); + if (g != null) { + try { out.add(Integer.parseInt(g)); break; } + catch (NumberFormatException ignored) {} + } + } + } + return out; + } + + /** + * A chunk is a citation hit when: + *

    + *
  • its {@code pageNumber} matches one of the {@code pageRefs}, OR
  • + *
  • its {@code content} contains a problem marker matching one of + * {@code problemRefs} (e.g. "第 5 题" / "5." / "Problem 5").
  • + *
+ */ + private boolean chunkMatches(WikiChunkEntity chunk, Set pageRefs, Set problemRefs) { + if (!pageRefs.isEmpty() + && chunk.getPageNumber() != null + && pageRefs.contains(chunk.getPageNumber())) { + return true; + } + if (!problemRefs.isEmpty() && chunk.getContent() != null) { + String content = chunk.getContent(); + for (Integer n : problemRefs) { + if (containsProblemMarker(content, n)) return true; + } + } + return false; + } + + /** Match any of the conventional problem-number forms in the chunk content. */ + private static boolean containsProblemMarker(String content, int n) { + if (content == null) return false; + return content.contains("第 " + n + " 题") + || content.contains("第" + n + "题") + || content.contains("例 " + n) + || content.contains("例" + n) + || content.contains("题 " + n) + || content.contains("Problem " + n) + || content.contains("Example " + n) + // Common "1.", "2." problem-number markers at line start. Cheap + // contains-check rather than a regex anchor — the false-positive + // rate is low because we only match when problemRefs is non-empty, + // i.e. the LLM explicitly cited a numbered example. + || content.contains("\n" + n + ". ") + || content.startsWith(n + ". "); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java new file mode 100644 index 00000000..c6961dd2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationExecutor.java @@ -0,0 +1,723 @@ +package vip.mate.wiki.service; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.wiki.job.WikiJobStep; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.metrics.WikiMetrics; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * Runs a single {@link WikiTransformationEntity} against a source raw + * material: substitutes placeholders into the user-defined prompt, calls + * the configured chat model, and persists the run row with the output. + * + *

Sync entry point: {@link #runOnRawSync}. Async fire-and-forget + * helpers (used by the ingest-pipeline hook and the controller's "apply" + * endpoint when the caller doesn't want to block) wrap that on a virtual + * thread executor. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationExecutor { + + /** Virtual-thread pool — matches the WIKI_EXECUTOR pattern used elsewhere in the module. */ + private static final ExecutorService WORKER = Executors.newVirtualThreadPerTaskExecutor(); + + /** Hard cap on input text fed into the prompt (defensive against multi-MB extracted PDFs). */ + private static final int MAX_INPUT_CHARS = 60_000; + + private final WikiTransformationService transformationService; + private final WikiRawMaterialService rawService; + private final WikiMetrics metrics; + + @Autowired(required = false) + private WikiModelRoutingService modelRoutingService; + + /** Optional. When wired, completed runs whose template has + * {@code outputTarget=page} are persisted as a synthesis wiki page. */ + @Autowired(required = false) + private WikiPageService pageService; + + /** Optional. When wired, every persisted synthesis page is embedded so + * the semantic retriever can surface it on terms that exist only in the + * transformation output (not in any source raw's chunks). */ + @Autowired(required = false) + private WikiEmbeddingService embeddingService; + + /** Optional. When wired, the executor parses references like + * {@code 第 5 题 / 第 14 页 / page 14} out of the output and writes + * chunk-level citations binding the synthesis page back to the source + * chunks the LLM said it relied on. */ + @Autowired(required = false) + private WikiTransformationCitationExtractor citationExtractor; + + private final com.fasterxml.jackson.databind.ObjectMapper objectMapper = + new com.fasterxml.jackson.databind.ObjectMapper(); + + public CompletableFuture runDefaultsAsync(Long kbId, Long workspaceId, Long rawId, String triggeredBy) { + return CompletableFuture.runAsync(() -> { + try { + List defaults = + transformationService.listApplyDefaultsForKb(kbId, workspaceId); + for (WikiTransformationEntity t : defaults) { + try { + runOnRawSync(t, rawId, triggeredBy); + } catch (Exception e) { + log.warn("[WikiTransformation] default run failed transformation={} rawId={}: {}", + t.getName(), rawId, e.getMessage()); + } + } + } catch (Exception e) { + log.warn("[WikiTransformation] default sweep failed kbId={} rawId={}: {}", + kbId, rawId, e.getMessage()); + } + }, WORKER); + } + + public CompletableFuture runOnRawAsync( + WikiTransformationEntity transformation, Long rawId, String triggeredBy) { + return CompletableFuture.supplyAsync( + () -> runOnRawSync(transformation, rawId, triggeredBy), WORKER); + } + + public CompletableFuture runOnPageAsync( + WikiTransformationEntity transformation, Long pageId, String triggeredBy) { + return CompletableFuture.supplyAsync( + () -> runOnPageSync(transformation, pageId, triggeredBy), WORKER); + } + + /** + * Run the transformation against an existing wiki page (e.g. a previous + * synthesis page or a manually-authored page). Mirrors + * {@link #runOnRawSync} but uses page content as the source. Output is + * not auto-saved as a wiki page even when the template has + * {@code outputTarget=page} — overwriting the input page would be + * surprising; users can still save manually from the run history. + */ + public WikiTransformationRunEntity runOnPageSync( + WikiTransformationEntity transformation, Long pageId, String triggeredBy) { + if (transformation == null) { + throw new IllegalArgumentException("transformation is required"); + } + if (pageId == null) { + throw new IllegalArgumentException("pageId is required"); + } + if (pageService == null) { + throw new IllegalStateException("Page service unavailable"); + } + WikiPageEntity page = pageService.getById(pageId); + if (page == null) { + throw new IllegalArgumentException("Page not found: " + pageId); + } + if (Boolean.FALSE.equals(transformation.getEnabled())) { + log.debug("[WikiTransformation] skipping disabled template id={} name={}", + transformation.getId(), transformation.getName()); + return null; + } + + long startNanos = System.nanoTime(); + WikiTransformationRunEntity run = new WikiTransformationRunEntity(); + run.setTransformationId(transformation.getId()); + run.setKbId(page.getKbId()); + run.setWorkspaceId(transformation.getWorkspaceId()); + run.setInputKind("page"); + run.setPageId(pageId); + run.setStatus("running"); + run.setTriggeredBy(triggeredBy == null ? "manual" : triggeredBy); + run.setStartedAt(LocalDateTime.now()); + transformationService.insertRun(run); + + try { + String inputText = page.getContent(); + if (inputText == null || inputText.isBlank()) { + throw new IllegalStateException("Page has no content"); + } + String output = renderAndCallLlm(transformation, page.getKbId(), + page.getTitle() == null ? ("page#" + pageId) : page.getTitle(), + inputText, run); + + WikiTransformationRunEntity current = transformationService.getRun(run.getId()); + if (current != null && "cancelled".equalsIgnoreCase(current.getStatus())) { + log.info("[WikiTransformation] run={} was cancelled mid-flight; discarding {} chars of LLM output", + run.getId(), output.length()); + return current; + } + + run.setOutput(output); + run.setStatus("completed"); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + // For page input we never auto-save back to a page — the call site + // can use the manual save-as-page endpoint with a derived slug. + transformationService.updateRun(run); + + metrics.recordCompileStage("transformation_run", page.getKbId(), + Duration.ofNanos(System.nanoTime() - startNanos)); + log.info("[WikiTransformation] ok run={} transformation={} pageId={} kbId={} ({} ms)", + run.getId(), transformation.getName(), pageId, page.getKbId(), + run.getDurationMs()); + } catch (Exception e) { + WikiTransformationRunEntity current = transformationService.getRun(run.getId()); + if (current != null && "cancelled".equalsIgnoreCase(current.getStatus())) { + log.info("[WikiTransformation] run={} was cancelled before failure could be recorded ({})", + run.getId(), e.getMessage()); + return current; + } + run.setStatus("failed"); + String msg = e.getMessage(); + run.setError(msg == null ? e.getClass().getSimpleName() : msg); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + transformationService.updateRun(run); + log.warn("[WikiTransformation] failed run={} transformation={} pageId={}: {}", + run.getId(), transformation.getName(), pageId, msg); + } + return run; + } + + /** + * Shared prompt-render + LLM-call + output-cleanup stage used by both + * raw-input and page-input entry points. Sets {@code run.modelId} as a + * side-effect so the run row reflects which model produced the output. + *

+ * When the template declares {@code outputFormat=json}, the response is + * parsed as JSON; on parse failure the LLM is asked once more with a + * stricter "return only JSON" reminder before the run is failed. + */ + private String renderAndCallLlm(WikiTransformationEntity transformation, Long kbId, + String sourceTitle, String sourceText, + WikiTransformationRunEntity run) { + String trimmedInput = sourceText.length() > MAX_INPUT_CHARS + ? sourceText.substring(0, MAX_INPUT_CHARS) + "\n…(truncated)" + : sourceText; + + boolean wantJson = "json".equalsIgnoreCase(transformation.getOutputFormat()); + String schema = transformation.getOutputSchema(); + boolean hasSchema = wantJson && schema != null && !schema.isBlank(); + + String systemPrompt = PromptLoader.loadPrompt( + wantJson ? "wiki/transformation-system-json" : "wiki/transformation-system"); + String instruction = (transformation.getPromptTemplate() == null ? "" : transformation.getPromptTemplate()) + .replace("{input_text}", trimmedInput) + .replace("{title}", sourceTitle); + if (hasSchema) { + instruction = instruction + + "\n\n---\n\n输出必须严格符合下面这个 JSON Schema:\n```json\n" + + schema + "\n```"; + } + String userPrompt = PromptLoader.loadPrompt("wiki/transformation-user") + .replace("{instruction}", instruction) + .replace("{source_title}", sourceTitle) + .replace("{source_text}", trimmedInput); + + Long resolvedModelId = resolveModelId(transformation, kbId); + ChatModel chatModel = buildChatModel(resolvedModelId); + run.setModelId(resolvedModelId); + + CallResult first = callOnce(chatModel, systemPrompt, userPrompt); + accumulateUsage(run, first); + if (wantJson) { + String coerced = coerceToJson(first.text()); + String validationError = coerced != null ? validateAgainstSchema(coerced, schema) : "not valid JSON"; + if (coerced != null && validationError == null) { + // Wrap in a fenced block so UI rendering and save-as-page + // keep the existing markdown contract. The raw JSON is the + // first thing inside the block, so downstream tools can grep. + return "```json\n" + coerced + "\n```"; + } + // One retry with an explicit nudge about what failed. + log.info("[WikiTransformation] JSON validation failed for template={} ({}); retrying with stricter reminder", + transformation.getName(), validationError); + String reminder = "上一次回复无效:" + validationError + "。请只返回一个合法 JSON 文档," + + "前后不要有任何文字或代码块标记" + + (hasSchema ? ",并严格匹配上面给出的 JSON Schema。" : "。"); + String retryUserPrompt = userPrompt + "\n\n---\n\n" + reminder; + CallResult retry = callOnce(chatModel, systemPrompt, retryUserPrompt); + accumulateUsage(run, retry); + String coercedRetry = coerceToJson(retry.text()); + String retryError = coercedRetry != null ? validateAgainstSchema(coercedRetry, schema) : "not valid JSON"; + if (coercedRetry != null && retryError == null) { + return "```json\n" + coercedRetry + "\n```"; + } + throw new IllegalStateException("LLM output failed JSON validation after one retry: " + retryError); + } + return first.text(); + } + + /** + * Lightweight JSON Schema check — verifies the parsed value is the + * declared top-level type and contains every entry in the + * {@code required} array. Deep validation (per-field types, enums, + * patterns) is out of scope; the prompt-time schema injection does + * most of the work and this check just guards the obvious failures. + * + * @return {@code null} when valid, otherwise a short failure description + */ + private static String validateAgainstSchema(String jsonText, String schemaText) { + if (schemaText == null || schemaText.isBlank()) return null; + try { + com.fasterxml.jackson.databind.JsonNode value = JSON_MAPPER.readTree(jsonText); + com.fasterxml.jackson.databind.JsonNode schema = JSON_MAPPER.readTree(schemaText); + + String type = schema.path("type").asText(""); + if ("object".equals(type) && !value.isObject()) { + return "expected object at top level, got " + value.getNodeType().name().toLowerCase(); + } + if ("array".equals(type) && !value.isArray()) { + return "expected array at top level, got " + value.getNodeType().name().toLowerCase(); + } + + com.fasterxml.jackson.databind.JsonNode required = schema.get("required"); + if (required != null && required.isArray() && value.isObject()) { + List missing = new java.util.ArrayList<>(); + for (com.fasterxml.jackson.databind.JsonNode req : required) { + String field = req.asText(); + if (!field.isBlank() && !value.has(field)) missing.add(field); + } + if (!missing.isEmpty()) { + return "missing required field(s): " + String.join(", ", missing); + } + } + return null; + } catch (Exception e) { + return "schema check error: " + e.getMessage(); + } + } + + /** Tuple returned from a single LLM call: cleaned text + usage (null when provider didn't surface usage). */ + private record CallResult(String text, Long inputTokens, Long outputTokens, Long totalTokens) {} + + /** One LLM call, returns the cleaned output + provider usage. Throws when the call yields blank. */ + private CallResult callOnce(ChatModel chatModel, String systemPrompt, String userPrompt) { + ChatResponse resp = chatModel.call(new Prompt(List.of( + new SystemMessage(systemPrompt), new UserMessage(userPrompt)))); + String rawOutput = (resp == null || resp.getResult() == null + || resp.getResult().getOutput() == null) + ? null : resp.getResult().getOutput().getText(); + if (rawOutput == null || rawOutput.isBlank()) { + throw new IllegalStateException("LLM returned empty output"); + } + String output = cleanLlmOutput(rawOutput); + if (output.isBlank()) { + throw new IllegalStateException("LLM output was empty after cleanup"); + } + Long in = null, out = null, total = null; + try { + if (resp.getMetadata() != null && resp.getMetadata().getUsage() != null) { + var u = resp.getMetadata().getUsage(); + in = u.getPromptTokens() == null ? null : u.getPromptTokens().longValue(); + out = u.getCompletionTokens() == null ? null : u.getCompletionTokens().longValue(); + total = u.getTotalTokens() == null ? null : u.getTotalTokens().longValue(); + } + } catch (Exception ignored) { + // Usage extraction is best-effort — different providers expose it differently. + } + return new CallResult(output, in, out, total); + } + + /** Add provider-reported usage onto the run row (accumulates across retries). */ + private static void accumulateUsage(WikiTransformationRunEntity run, CallResult call) { + if (call.inputTokens() != null) { + run.setInputTokens((run.getInputTokens() == null ? 0L : run.getInputTokens()) + call.inputTokens()); + } + if (call.outputTokens() != null) { + run.setOutputTokens((run.getOutputTokens() == null ? 0L : run.getOutputTokens()) + call.outputTokens()); + } + if (call.totalTokens() != null) { + run.setTotalTokens((run.getTotalTokens() == null ? 0L : run.getTotalTokens()) + call.totalTokens()); + } + } + + private static final com.fasterxml.jackson.databind.ObjectMapper JSON_MAPPER = + new com.fasterxml.jackson.databind.ObjectMapper(); + + /** + * Try to parse the output as JSON. If the LLM wrapped it in a fenced + * block or sprinkled prose around it, fall back to finding the outer + * '{' / '[' brackets and try again. Returns the normalized JSON string + * on success, {@code null} on failure. + */ + private static String coerceToJson(String text) { + if (text == null || text.isBlank()) return null; + String candidate = text.trim(); + try { + JSON_MAPPER.readTree(candidate); + return candidate; + } catch (Exception ignored) { + // fall through to bracket-trim attempt + } + int objStart = candidate.indexOf('{'); + int arrStart = candidate.indexOf('['); + int start; + char open; + if (objStart < 0) { start = arrStart; open = '['; } + else if (arrStart < 0) { start = objStart; open = '{'; } + else { start = Math.min(objStart, arrStart); open = candidate.charAt(start); } + if (start < 0) return null; + char close = open == '{' ? '}' : ']'; + int end = candidate.lastIndexOf(close); + if (end <= start) return null; + String trimmed = candidate.substring(start, end + 1); + try { + JSON_MAPPER.readTree(trimmed); + return trimmed; + } catch (Exception e) { + return null; + } + } + + /** + * Run the transformation against the given raw material and persist + * the outcome. The returned entity is the persisted run row, regardless + * of success or failure (failure leaves {@code status=failed} and + * {@code error} populated). + */ + public WikiTransformationRunEntity runOnRawSync( + WikiTransformationEntity transformation, Long rawId, String triggeredBy) { + if (transformation == null) { + throw new IllegalArgumentException("transformation is required"); + } + if (rawId == null) { + throw new IllegalArgumentException("rawId is required"); + } + WikiRawMaterialEntity raw = rawService.getById(rawId); + if (raw == null) { + throw new IllegalArgumentException("Raw material not found: " + rawId); + } + if (Boolean.FALSE.equals(transformation.getEnabled())) { + log.debug("[WikiTransformation] skipping disabled template id={} name={}", + transformation.getId(), transformation.getName()); + return null; + } + + long startNanos = System.nanoTime(); + WikiTransformationRunEntity run = new WikiTransformationRunEntity(); + run.setTransformationId(transformation.getId()); + run.setKbId(raw.getKbId()); + run.setWorkspaceId(transformation.getWorkspaceId()); + run.setInputKind("raw"); + run.setRawId(rawId); + run.setStatus("running"); + run.setTriggeredBy(triggeredBy == null ? "manual" : triggeredBy); + run.setStartedAt(LocalDateTime.now()); + transformationService.insertRun(run); + + try { + String inputText = rawService.getTextContent(raw); + if (inputText == null || inputText.isBlank()) { + throw new IllegalStateException("Raw material has no extractable text yet"); + } + String output = renderAndCallLlm(transformation, raw.getKbId(), + safeTitle(raw), inputText, run); + // Honour a mid-flight cancel: the cancel endpoint flipped the run + // row to 'cancelled' while the LLM was still working. Drop the + // output and stop here rather than overwrite the cancelled state. + WikiTransformationRunEntity current = transformationService.getRun(run.getId()); + if (current != null && "cancelled".equalsIgnoreCase(current.getStatus())) { + log.info("[WikiTransformation] run={} was cancelled mid-flight; discarding {} chars of LLM output", + run.getId(), output.length()); + return current; + } + + run.setOutput(output); + run.setStatus("completed"); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + + // Persist as a synthesis wiki page when the template asks for it. + // Failures here are logged but do not flip the run back to failed: + // the LLM output is already valid, the page-write is best-effort. + if ("page".equalsIgnoreCase(transformation.getOutputTarget())) { + try { + WikiPageEntity page = saveRunAsPage(run, transformation, raw, output); + if (page != null) run.setOutputPageId(page.getId()); + } catch (Exception pe) { + log.warn("[WikiTransformation] auto-save as page failed run={}: {}", + run.getId(), pe.getMessage()); + } + } + transformationService.updateRun(run); + + metrics.recordCompileStage("transformation_run", raw.getKbId(), + Duration.ofNanos(System.nanoTime() - startNanos)); + log.info("[WikiTransformation] ok run={} transformation={} rawId={} kbId={} ({} ms, pageId={})", + run.getId(), transformation.getName(), rawId, raw.getKbId(), + run.getDurationMs(), run.getOutputPageId()); + } catch (Exception e) { + WikiTransformationRunEntity current = transformationService.getRun(run.getId()); + if (current != null && "cancelled".equalsIgnoreCase(current.getStatus())) { + log.info("[WikiTransformation] run={} was cancelled before failure could be recorded ({})", + run.getId(), e.getMessage()); + return current; + } + run.setStatus("failed"); + String msg = e.getMessage(); + run.setError(msg == null ? e.getClass().getSimpleName() : msg); + run.setCompletedAt(LocalDateTime.now()); + run.setDurationMs(Duration.ofNanos(System.nanoTime() - startNanos).toMillis()); + transformationService.updateRun(run); + log.warn("[WikiTransformation] failed run={} transformation={} rawId={}: {}", + run.getId(), transformation.getName(), rawId, msg); + } + return run; + } + + /** + * Mark a still-active run as cancelled. The blocked LLM call (if any) + * continues server-side but its eventual output is dropped by the + * post-call check in {@link #runOnRawSync}. + */ + public boolean cancelRun(Long runId) { + if (runId == null) return false; + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) return false; + String status = run.getStatus(); + if (!"pending".equalsIgnoreCase(status) && !"running".equalsIgnoreCase(status)) { + return false; + } + run.setStatus("cancelled"); + run.setCompletedAt(LocalDateTime.now()); + if (run.getError() == null) run.setError("Cancelled by user"); + transformationService.updateRun(run); + log.info("[WikiTransformation] run={} cancelled by user", runId); + return true; + } + + private Long resolveModelId(WikiTransformationEntity transformation, Long kbId) { + if (transformation.getModelId() != null) return transformation.getModelId(); + if (modelRoutingService == null) { + throw new IllegalStateException("No model bound on transformation and ModelRoutingService unavailable"); + } + return modelRoutingService.selectModelId(kbId, "heavy_ingest", WikiJobStep.CREATE_PAGE); + } + + private ChatModel buildChatModel(Long modelId) { + if (modelRoutingService == null) { + throw new IllegalStateException("ModelRoutingService unavailable; cannot run transformation"); + } + return modelRoutingService.buildChatModel(modelId); + } + + private static String safeTitle(WikiRawMaterialEntity raw) { + String t = raw.getTitle(); + return (t == null || t.isBlank()) ? ("raw#" + raw.getId()) : t; + } + + /** Recognises the conversational openers LLMs sometimes prepend even when + * the system prompt told them not to. Lines matching this pattern at the + * very start of the output are dropped. */ + private static final java.util.regex.Pattern PREAMBLE_PATTERN = + java.util.regex.Pattern.compile( + "^\\s*(以下是|下面是|这是|根据您的要求|Sure(?:!|,)?|Of course[!,]?|Here(?:'s| is| are)|Certainly[!,]?|Got it[!,]?)[^\\n]*[::][^\\n]*\\n+", + java.util.regex.Pattern.CASE_INSENSITIVE); + + /** + * Normalise raw LLM output before persisting: + *

    + *
  • strip an outer markdown / language code fence (```markdown ... ``` or ``` ... ```)
  • + *
  • drop a conversational opener line ending with a colon
  • + *
  • trim whitespace
  • + *
+ * Conservative — only trims when the heuristic match is unambiguous, + * because over-trimming on a structured response would corrupt content. + */ + static String cleanLlmOutput(String text) { + if (text == null) return ""; + String result = text.trim(); + + // Outer code fence ```lang? ... ``` + if (result.startsWith("```")) { + int firstNewline = result.indexOf('\n'); + if (firstNewline > 0 && result.endsWith("```")) { + String header = result.substring(3, firstNewline).trim(); + // Only strip when the header is empty or looks like a language tag + // (markdown / md / json / yaml / text / plaintext) — never strip + // when the LLM used ``` as actual fenced content inside. + if (header.isEmpty() || header.matches("(?i)markdown|md|text|plaintext|json|yaml|yml|html?")) { + result = result.substring(firstNewline + 1, result.length() - 3).trim(); + } + } + } + + // Conversational opener line ending with a colon, followed by content. + java.util.regex.Matcher m = PREAMBLE_PATTERN.matcher(result); + if (m.find()) { + result = result.substring(m.end()).trim(); + } + + return result; + } + + // ==================== Save-as-page ==================== + + /** + * Manual entry point used by the {@code POST /runs/{runId}/save-as-page} + * endpoint. Loads the run + its template + its source raw material, + * delegates to {@link #saveRunAsPage}, and updates the run row with the + * resulting page id so the UI can render a "saved as: …" affordance. + * + * @return the persisted page; never {@code null} on success + * @throws IllegalArgumentException when the run / raw / template is missing + * @throws IllegalStateException when the run is not completed or no output + */ + public WikiPageEntity manualSaveRunAsPage(Long runId) { + if (runId == null) throw new IllegalArgumentException("runId is required"); + WikiTransformationRunEntity run = transformationService.getRun(runId); + if (run == null) throw new IllegalArgumentException("Run not found: " + runId); + if (!"completed".equalsIgnoreCase(run.getStatus())) { + throw new IllegalStateException("Run is not completed (status=" + run.getStatus() + ")"); + } + if (run.getOutput() == null || run.getOutput().isBlank()) { + throw new IllegalStateException("Run has no output to save"); + } + if (run.getRawId() == null) { + throw new IllegalStateException("Run is not bound to a raw material"); + } + WikiTransformationEntity template = transformationService.getById(run.getTransformationId()); + if (template == null) { + throw new IllegalStateException("Transformation template no longer exists"); + } + WikiRawMaterialEntity raw = rawService.getById(run.getRawId()); + if (raw == null) { + throw new IllegalStateException("Source raw material no longer exists"); + } + WikiPageEntity page = saveRunAsPage(run, template, raw, run.getOutput()); + if (page != null) { + run.setOutputPageId(page.getId()); + transformationService.updateRun(run); + } + return page; + } + + /** + * Upsert the transformation output as a synthesis wiki page on the same + * KB. Slug is deterministic — {@code -} — + * so re-running an apply_default template against the same raw material + * updates the existing page in place rather than spawning duplicates. + */ + private WikiPageEntity saveRunAsPage(WikiTransformationRunEntity run, + WikiTransformationEntity template, + WikiRawMaterialEntity raw, + String output) { + if (pageService == null) { + log.warn("[WikiTransformation] save-as-page requested but WikiPageService not available"); + return null; + } + Long kbId = raw.getKbId(); + String slug = buildSlug(template, raw); + String title = template.getTitle() + " · " + safeTitle(raw); + String summary = deriveSummary(output); + String sourceRawIdsJson = toJsonArray(raw.getId()); + + WikiPageEntity existing = pageService.getBySlug(kbId, slug); + WikiPageEntity persisted; + if (existing == null) { + persisted = pageService.createPage(kbId, slug, title, output, summary, + sourceRawIdsJson, "synthesis"); + log.info("[WikiTransformation] saved run={} as new page slug={} pageId={}", + run.getId(), slug, persisted.getId()); + } else { + persisted = pageService.updatePageByAi(kbId, slug, output, summary, raw.getId()); + if (persisted == null) persisted = existing; + log.info("[WikiTransformation] updated existing synthesis page slug={} pageId={} from run={}", + slug, persisted.getId(), run.getId()); + } + + // Fire-and-forget page-level embedding so semantic search can match + // vocabulary the LLM authored which isn't present in the source raw's + // chunks (e.g. "AM-GM", "柯西不等式" derived from a garbled OCR PDF). + if (embeddingService != null) { + final Long pid = persisted.getId(); + WORKER.submit(() -> { + try { embeddingService.embedPage(pid); } + catch (Exception ee) { + log.warn("[WikiTransformation] post-save embedPage failed pageId={}: {}", + pid, ee.getMessage()); + } + }); + } + + // Fire-and-forget reverse-citation extraction. If the LLM cited + // specific page numbers / problem numbers, write precise chunk + // citations binding the synthesis page back to those source chunks. + if (citationExtractor != null) { + final Long pid = persisted.getId(); + final Long kid = kbId; + final Long rid = raw.getId(); + final String out = output; + WORKER.submit(() -> { + try { citationExtractor.extractAndApply(pid, kid, rid, out); } + catch (Exception ee) { + log.warn("[WikiTransformation] post-save citation extract failed pageId={}: {}", + pid, ee.getMessage()); + } + }); + } + return persisted; + } + + /** + * Common document / image extensions that we don't want leaking into the + * slug. The pattern matches a trailing dotted extension and is case- + * insensitive so both {@code foo.PDF} and {@code foo.pdf} are stripped. + */ + private static final java.util.regex.Pattern FILE_EXT_PATTERN = + java.util.regex.Pattern.compile( + "\\.(pdf|docx?|pptx?|xlsx?|csv|tsv|txt|md|markdown|rtf|odt|epub|html?|json|xml|yaml|yml|jpe?g|png|gif|bmp|tiff?|webp|svg|mp3|wav|mp4|mov|webm)$", + java.util.regex.Pattern.CASE_INSENSITIVE); + + private static String buildSlug(WikiTransformationEntity template, WikiRawMaterialEntity raw) { + String trimmedTitle = stripFileExtension(raw.getTitle()); + String rawPart = WikiPageService.toSlug(trimmedTitle); + if (rawPart == null || rawPart.isBlank()) rawPart = "r" + raw.getId(); + return template.getName() + "-" + rawPart; + } + + private static String stripFileExtension(String title) { + if (title == null) return null; + return FILE_EXT_PATTERN.matcher(title.trim()).replaceFirst(""); + } + + /** First non-empty line of the output, capped to ~280 chars, used as page summary. */ + private static String deriveSummary(String output) { + if (output == null) return ""; + for (String line : output.split("\\n")) { + String trimmed = line.trim(); + if (trimmed.isEmpty()) continue; + if (trimmed.startsWith("#")) { + trimmed = trimmed.replaceAll("^#+\\s*", ""); + if (trimmed.isEmpty()) continue; + } + return trimmed.length() > 280 ? trimmed.substring(0, 280) + "…" : trimmed; + } + return ""; + } + + private String toJsonArray(Long rawId) { + try { + return objectMapper.writeValueAsString(java.util.List.of(rawId)); + } catch (Exception e) { + return "[" + rawId + "]"; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java new file mode 100644 index 00000000..90ed1606 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/wiki/service/WikiTransformationService.java @@ -0,0 +1,263 @@ +package vip.mate.wiki.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; +import vip.mate.wiki.repository.WikiTransformationMapper; +import vip.mate.wiki.repository.WikiTransformationRunMapper; + +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; + +/** + * CRUD + lookups for wiki transformation templates and their execution + * history. Pure persistence — the LLM call lives in + * {@link WikiTransformationExecutor}. + */ +@Slf4j +@Service +@RequiredArgsConstructor +public class WikiTransformationService { + + private static final Pattern NAME_PATTERN = Pattern.compile("^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$"); + + private final WikiTransformationMapper transformationMapper; + private final WikiTransformationRunMapper runMapper; + + /** Templates visible to a KB: pinned to this KB plus workspace-wide ones (kb_id NULL). */ + public List listForKb(Long kbId, Long workspaceId) { + if (kbId == null) { + return List.of(); + } + return transformationMapper.selectList( + new LambdaQueryWrapper() + .and(w -> w.eq(WikiTransformationEntity::getKbId, kbId) + .or(g -> g.isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId))) + .orderByDesc(WikiTransformationEntity::getUpdateTime)); + } + + public List listByWorkspace(Long workspaceId) { + return transformationMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId) + .orderByDesc(WikiTransformationEntity::getUpdateTime)); + } + + public WikiTransformationEntity getById(Long id) { + return transformationMapper.selectById(id); + } + + public Optional findByName(Long kbId, Long workspaceId, String name) { + if (name == null || name.isBlank()) return Optional.empty(); + // Prefer the KB-pinned record over a workspace-wide one of the same name. + WikiTransformationEntity pinned = transformationMapper.selectOne( + new LambdaQueryWrapper() + .eq(WikiTransformationEntity::getKbId, kbId) + .eq(WikiTransformationEntity::getName, name) + .last("LIMIT 1")); + if (pinned != null) return Optional.of(pinned); + WikiTransformationEntity global = transformationMapper.selectOne( + new LambdaQueryWrapper() + .isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId) + .eq(WikiTransformationEntity::getName, name) + .last("LIMIT 1")); + return Optional.ofNullable(global); + } + + /** Default-apply templates that should run for a raw material in {@code kbId}. */ + public List listApplyDefaultsForKb(Long kbId, Long workspaceId) { + return listForKb(kbId, workspaceId).stream() + .filter(t -> Boolean.TRUE.equals(t.getApplyDefault())) + .filter(t -> !Boolean.FALSE.equals(t.getEnabled())) + .toList(); + } + + @Transactional + public WikiTransformationEntity create(WikiTransformationEntity input) { + validateName(input.getName()); + if (input.getTitle() == null || input.getTitle().isBlank()) { + throw new IllegalArgumentException("title is required"); + } + if (input.getPromptTemplate() == null || input.getPromptTemplate().isBlank()) { + throw new IllegalArgumentException("promptTemplate is required"); + } + Long workspaceId = input.getWorkspaceId() == null ? 1L : input.getWorkspaceId(); + + // Enforce uniqueness on (kbId, name) — including the NULL-kbId case + // where MySQL would otherwise allow duplicates. + findByExactScopeAndName(input.getKbId(), workspaceId, input.getName()) + .ifPresent(existing -> { + throw new IllegalArgumentException("Transformation already exists: " + input.getName()); + }); + + WikiTransformationEntity entity = new WikiTransformationEntity(); + entity.setKbId(input.getKbId()); + entity.setWorkspaceId(workspaceId); + entity.setName(input.getName()); + entity.setTitle(input.getTitle()); + entity.setDescription(input.getDescription()); + entity.setPromptTemplate(input.getPromptTemplate()); + entity.setApplyDefault(Boolean.TRUE.equals(input.getApplyDefault())); + entity.setEnabled(input.getEnabled() == null ? Boolean.TRUE : input.getEnabled()); + // Treat negative values as the "clear / use default" sentinel so the + // create and update paths accept the same payload from the UI. + entity.setModelId(input.getModelId() != null && input.getModelId() < 0 ? null : input.getModelId()); + entity.setOutputTarget(normalizeOutputTarget(input.getOutputTarget())); + entity.setOutputFormat(normalizeOutputFormat(input.getOutputFormat())); + entity.setOutputSchema(sanitizeOutputSchema(input.getOutputSchema())); + transformationMapper.insert(entity); + log.info("[WikiTransformation] created id={} name={} kbId={}", + entity.getId(), entity.getName(), entity.getKbId()); + return entity; + } + + @Transactional + public WikiTransformationEntity update(Long id, WikiTransformationEntity patch) { + WikiTransformationEntity entity = transformationMapper.selectById(id); + if (entity == null) { + throw new IllegalArgumentException("Transformation not found: " + id); + } + if (patch.getTitle() != null) entity.setTitle(patch.getTitle()); + if (patch.getDescription() != null) entity.setDescription(patch.getDescription()); + if (patch.getPromptTemplate() != null) entity.setPromptTemplate(patch.getPromptTemplate()); + if (patch.getApplyDefault() != null) entity.setApplyDefault(patch.getApplyDefault()); + if (patch.getEnabled() != null) entity.setEnabled(patch.getEnabled()); + // modelId is allowed to be cleared via explicit -1 sentinel handled by controller; + // here we only honour non-null assignments. + if (patch.getModelId() != null) { + entity.setModelId(patch.getModelId() < 0 ? null : patch.getModelId()); + } + if (patch.getOutputTarget() != null) { + entity.setOutputTarget(normalizeOutputTarget(patch.getOutputTarget())); + } + if (patch.getOutputFormat() != null) { + entity.setOutputFormat(normalizeOutputFormat(patch.getOutputFormat())); + } + if (patch.getOutputSchema() != null) { + // Empty string clears the schema; non-blank gets stored after a parse check. + entity.setOutputSchema(sanitizeOutputSchema(patch.getOutputSchema())); + } + transformationMapper.updateById(entity); + return entity; + } + + /** Whitelist incoming outputTarget; unknown / null = "none". */ + private static String normalizeOutputTarget(String raw) { + if (raw == null) return "none"; + String trimmed = raw.trim().toLowerCase(); + return switch (trimmed) { + case "page" -> "page"; + default -> "none"; + }; + } + + /** Whitelist incoming outputFormat; unknown / null = "markdown". */ + private static String normalizeOutputFormat(String raw) { + if (raw == null) return "markdown"; + String trimmed = raw.trim().toLowerCase(); + return switch (trimmed) { + case "json" -> "json"; + default -> "markdown"; + }; + } + + /** + * Sanitises the user-supplied JSON Schema text. Empty / blank values + * clear the column. Non-parseable values are rejected at the API + * boundary so the executor doesn't have to defend against garbage + * stored on the template. + */ + private static final com.fasterxml.jackson.databind.ObjectMapper SCHEMA_MAPPER = + new com.fasterxml.jackson.databind.ObjectMapper(); + + private static String sanitizeOutputSchema(String raw) { + if (raw == null) return null; + String trimmed = raw.trim(); + if (trimmed.isEmpty()) return null; + try { + SCHEMA_MAPPER.readTree(trimmed); + } catch (Exception e) { + throw new IllegalArgumentException("output_schema is not valid JSON: " + e.getMessage()); + } + return trimmed; + } + + @Transactional + public void delete(Long id) { + transformationMapper.deleteById(id); + } + + // ==================== Runs ==================== + + public WikiTransformationRunEntity getRun(Long runId) { + return runMapper.selectById(runId); + } + + public List listRunsByRaw(Long rawId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getRawId, rawId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + public List listRunsByKb(Long kbId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getKbId, kbId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + public List listRunsByTransformation(Long transformationId, int limit) { + return runMapper.selectList( + new LambdaQueryWrapper() + .eq(WikiTransformationRunEntity::getTransformationId, transformationId) + .orderByDesc(WikiTransformationRunEntity::getCreateTime) + .last("LIMIT " + Math.max(1, Math.min(limit, 200)))); + } + + @Transactional + public WikiTransformationRunEntity insertRun(WikiTransformationRunEntity run) { + runMapper.insert(run); + return run; + } + + @Transactional + public void updateRun(WikiTransformationRunEntity run) { + runMapper.updateById(run); + } + + @Transactional + public void deleteRun(Long runId) { + runMapper.deleteById(runId); + } + + // ==================== helpers ==================== + + private Optional findByExactScopeAndName(Long kbId, Long workspaceId, String name) { + LambdaQueryWrapper q = new LambdaQueryWrapper<>(); + if (kbId == null) { + q.isNull(WikiTransformationEntity::getKbId) + .eq(WikiTransformationEntity::getWorkspaceId, workspaceId); + } else { + q.eq(WikiTransformationEntity::getKbId, kbId); + } + q.eq(WikiTransformationEntity::getName, name).last("LIMIT 1"); + return Optional.ofNullable(transformationMapper.selectOne(q)); + } + + private static void validateName(String name) { + if (name == null || !NAME_PATTERN.matcher(name).matches()) { + throw new IllegalArgumentException( + "name must be 3-64 chars, lowercase letters / digits / hyphens (start and end alphanumeric)"); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java index 3d8e7d5f..b1502472 100644 --- a/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java +++ b/mateclaw-server/src/main/java/vip/mate/wiki/tool/WikiTool.java @@ -18,6 +18,8 @@ import vip.mate.wiki.job.model.WikiProcessingJobEntity; import vip.mate.wiki.model.WikiKnowledgeBaseEntity; import vip.mate.wiki.model.WikiPageEntity; import vip.mate.wiki.model.WikiRawMaterialEntity; +import vip.mate.wiki.model.WikiTransformationEntity; +import vip.mate.wiki.model.WikiTransformationRunEntity; import vip.mate.wiki.repository.WikiRawMaterialMapper; import vip.mate.wiki.service.*; @@ -59,6 +61,16 @@ public class WikiTool { @Autowired(required = false) private WikiCompileService compileService; + /** Optional transformation engine. Tools degrade with a clear error when missing. */ + @Autowired(required = false) + private WikiTransformationService transformationService; + + @Autowired(required = false) + private WikiTransformationExecutor transformationExecutor; + + @Autowired(required = false) + private WikiTransformationAggregator transformationAggregator; + public WikiTool(WikiPageService pageService, WikiKnowledgeBaseService kbService, WikiRawMaterialService rawService, @@ -632,6 +644,176 @@ public class WikiTool { return "Wikilink enrichment queued for: " + slug; } + // ==================== Transformations ==================== + + @Tool(description = """ + List the transformation templates available to this agent's wiki KB. + Each result has a name (use it with wiki_apply_transformation), a + human title, and a description of what the prompt produces. + """) + public String wiki_list_transformations( + @ToolParam(description = "Agent ID") Long agentId) { + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null) return error("Transformations not available"); + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + List templates = transformationService.listForKb(kbId, wsId); + JSONArray arr = new JSONArray(); + for (WikiTransformationEntity t : templates) { + if (Boolean.FALSE.equals(t.getEnabled())) continue; + arr.add(JSONUtil.createObj() + .set("name", t.getName()) + .set("title", t.getTitle()) + .set("description", t.getDescription()) + .set("applyDefault", Boolean.TRUE.equals(t.getApplyDefault()))); + } + return JSONUtil.createObj().set("kbId", kbId).set("transformations", arr).toString(); + } + + @Tool(description = """ + Run a transformation template against one raw material and return the + generated text. Use wiki_list_transformations first to discover names. + The run is also persisted so the result is visible in the wiki UI. + """) + public String wiki_apply_transformation( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Transformation name (from wiki_list_transformations)") String name, + @ToolParam(description = "Raw material ID to run the transformation against") Long rawId) { + if (name == null || name.isBlank()) return error("name is required"); + if (rawId == null) return error("rawId is required"); + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null || transformationExecutor == null) { + return error("Transformations not available"); + } + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + WikiTransformationEntity template = transformationService.findByName(kbId, wsId, name).orElse(null); + if (template == null) return error("Transformation not found: " + name); + + try { + WikiTransformationRunEntity run = transformationExecutor.runOnRawSync(template, rawId, "agent_tool"); + if (run == null) return error("Transformation is disabled: " + name); + if ("failed".equals(run.getStatus())) { + return error("Transformation failed: " + run.getError()); + } + return JSONUtil.createObj() + .set("ok", true) + .set("runId", run.getId()) + .set("transformation", template.getName()) + .set("output", run.getOutput()) + .toString(); + } catch (IllegalStateException | IllegalArgumentException e) { + return error(e.getMessage()); + } catch (Exception e) { + log.warn("[WikiTool] wiki_apply_transformation failed: {}", e.getMessage()); + return error("Apply failed: " + e.getMessage()); + } + } + + @Tool(description = """ + Run a transformation template against an existing wiki page and return + the generated text. Use this when you want to derive a new artifact + from an existing page — e.g. "summarize the contract-review page", + "extract action items from this meeting-notes page". The run output + is persisted in the wiki UI; pass slug (not page id) for convenience. + """) + public String wiki_apply_transformation_to_page( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Transformation name (from wiki_list_transformations)") String name, + @ToolParam(description = "Source wiki page slug to run the transformation against") String slug) { + if (name == null || name.isBlank()) return error("name is required"); + if (slug == null || slug.isBlank()) return error("slug is required"); + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null || transformationExecutor == null) { + return error("Transformations not available"); + } + + WikiPageEntity page = pageService.getBySlug(kbId, slug); + if (page == null) return error("Page not found: " + slug); + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + WikiTransformationEntity template = transformationService.findByName(kbId, wsId, name).orElse(null); + if (template == null) return error("Transformation not found: " + name); + + try { + WikiTransformationRunEntity run = transformationExecutor.runOnPageSync(template, page.getId(), "agent_tool"); + if (run == null) return error("Transformation is disabled: " + name); + if ("failed".equals(run.getStatus())) { + return error("Transformation failed: " + run.getError()); + } + return JSONUtil.createObj() + .set("ok", true) + .set("runId", run.getId()) + .set("transformation", template.getName()) + .set("inputPage", slug) + .set("output", run.getOutput()) + .toString(); + } catch (IllegalStateException | IllegalArgumentException e) { + return error(e.getMessage()); + } catch (Exception e) { + log.warn("[WikiTool] wiki_apply_transformation_to_page failed: {}", e.getMessage()); + return error("Apply failed: " + e.getMessage()); + } + } + + @Tool(description = """ + Aggregate all completed runs of a transformation template across every + raw material in this KB into a single synthesis wiki page. Use this + after running a template against multiple sources to get a KB-level + unified document (e.g. one consolidated 题型库 across 5 different + mock exam PDFs, one customer-account brief across all sources for an + account). Idempotent — re-running upserts the same slug. + """) + public String wiki_aggregate_transformation( + @ToolParam(description = "Agent ID") Long agentId, + @ToolParam(description = "Transformation name (from wiki_list_transformations)") String name) { + if (name == null || name.isBlank()) return error("name is required"); + Long kbId = resolveKbId(agentId); + if (kbId == null) return error("No wiki knowledge base found for this agent"); + if (transformationService == null || transformationAggregator == null) { + return error("Transformations not available"); + } + + WikiKnowledgeBaseEntity kb = kbService.getById(kbId); + Long wsId = (kb == null || kb.getWorkspaceId() == null) ? 1L : kb.getWorkspaceId(); + + WikiTransformationEntity template = transformationService.findByName(kbId, wsId, name).orElse(null); + if (template == null) return error("Transformation not found: " + name); + + try { + var res = transformationAggregator.aggregate(template, kbId, "agent_tool"); + if (res.pageId() == null) { + return JSONUtil.createObj() + .set("ok", true) + .set("aggregated", false) + .set("reason", res.title()) + .toString(); + } + return JSONUtil.createObj() + .set("ok", true) + .set("aggregated", true) + .set("pageSlug", res.slug()) + .set("pageTitle", res.title()) + .set("sourcesUsed", res.sourcesUsed()) + .set("created", res.created()) + .toString(); + } catch (IllegalStateException | IllegalArgumentException e) { + return error(e.getMessage()); + } catch (Exception e) { + log.warn("[WikiTool] wiki_aggregate_transformation failed: {}", e.getMessage()); + return error("Aggregate failed: " + e.getMessage()); + } + } + // ==================== Helpers ==================== private Long resolveKbId(Long agentId) { diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/api/CompileErrorResponse.java b/mateclaw-server/src/main/java/vip/mate/workflow/api/CompileErrorResponse.java new file mode 100644 index 00000000..f7fba3a6 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/api/CompileErrorResponse.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.api; + +import vip.mate.workflow.compiler.CompileError; + +import java.util.List; + +/** + * Response shape for compile failures returned from publish / preview-compile + * endpoints. Surfaces every diagnostic at once so the front-end editor can + * highlight all offending fields in a single round trip; mirroring + * {@link CompileError} preserves the path / code / message tuple the editor + * expects. + */ +public record CompileErrorResponse(int errorCount, List errors) { + + public record Item(String code, String path, String message) {} + + public static CompileErrorResponse of(List errors) { + List items = errors.stream() + .map(e -> new Item(e.code(), e.path(), e.message())) + .toList(); + return new CompileErrorResponse(items.size(), items); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowController.java b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowController.java new file mode 100644 index 00000000..177b42b8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowController.java @@ -0,0 +1,308 @@ +package vip.mate.workflow.api; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.workflow.compiler.PublishContext; +import vip.mate.workflow.compiler.WorkflowAclPort; +import vip.mate.workflow.compiler.WorkflowCompileFailedException; +import vip.mate.workflow.compiler.WorkflowCompiler; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.model.WorkflowRunPauseEntity; +import vip.mate.workflow.model.WorkflowRunStepEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.repository.WorkflowRunPauseMapper; +import vip.mate.workflow.repository.WorkflowRunStepMapper; +import vip.mate.workflow.service.WorkflowService; + +import java.util.List; + +/** + * REST surface for workflow CRUD + draft / publish / run inspection. + * Endpoints follow the project convention of a single workspace id passed + * via query param (production deploys read it from {@code X-Workspace-Id} + * via the workspace interceptor; the param fallback keeps tests simple). + */ +@Tag(name = "工作流管理") +@RestController +@RequestMapping("/api/v1/workflows") +@RequiredArgsConstructor +public class WorkflowController { + + private final WorkflowService workflowService; + private final WorkflowRunMapper runMapper; + private final WorkflowRunStepMapper stepMapper; + private final WorkflowRunPauseMapper pauseMapper; + private final WorkflowCompiler compiler; + private final WorkflowAclPort aclPort; + /** Optional — only present when the LLM module is wired (production). + * Tests that don't boot the chat-model factory get a null and the + * /draft/generate endpoint returns 503 instead of crashing. */ + @org.springframework.beans.factory.annotation.Autowired(required = false) + private vip.mate.workflow.draftgen.WorkflowDraftGenerator draftGenerator; + @org.springframework.beans.factory.annotation.Autowired(required = false) + private vip.mate.workflow.draftgen.WorkflowDraftTemplateLibrary draftTemplates; + + @Operation(summary = "List workflows in the workspace") + @GetMapping + public R> list(@RequestHeader("X-Workspace-Id") long workspaceId) { + return R.ok(workflowService.listByWorkspace(workspaceId)); + } + + @Operation(summary = "Get a workflow by id (includes inline draft).") + @GetMapping("/{id}") + public R get(@PathVariable long id, + @RequestHeader("X-Workspace-Id") long workspaceId) { + WorkflowEntity row = workflowService.get(id, workspaceId); + if (row == null) return R.fail("workflow not found: " + id); + return R.ok(row); + } + + @Operation(summary = "Create a workflow row (draft starts empty).") + @PostMapping + public R create(@RequestBody WorkflowEntity workflow, + @RequestHeader("X-Workspace-Id") long workspaceId) { + // Force the workspace from the trusted header — the request body + // can't choose a workspace for the new row, otherwise a caller + // could plant rows into another tenant. + workflow.setWorkspaceId(workspaceId); + return R.ok(workflowService.create(workflow)); + } + + @Operation(summary = "Update workflow metadata (name / description / enabled).") + @PutMapping("/{id}") + public R update(@PathVariable long id, + @RequestBody WorkflowMetadataRequest body, + @RequestHeader("X-Workspace-Id") long workspaceId) { + return R.ok(workflowService.updateMetadata(id, workspaceId, + body.name(), body.description(), body.enabled())); + } + + @Operation(summary = "Save the inline draft graph_json without compiling.") + @PutMapping("/{id}/draft") + public R saveDraft(@PathVariable long id, + @RequestBody WorkflowDraftRequest body, + @RequestParam(value = "userId", required = false) Long userId, + @RequestHeader("X-Workspace-Id") long workspaceId) { + return R.ok(workflowService.saveDraft(id, workspaceId, body.draftJson(), userId)); + } + + @Operation(summary = "Compile the draft and surface diagnostics without persisting a revision.") + @PostMapping("/{id}/compile") + public ResponseEntity compileDraft(@PathVariable long id, + @RequestHeader("X-Workspace-Id") long workspaceId) { + WorkflowEntity row = workflowService.get(id, workspaceId); + if (row == null) { + return ResponseEntity.badRequest().body(R.fail("workflow not found: " + id)); + } + // The parser throws WorkflowParseException for null/blank/whitespace + // input, which would otherwise bubble up to the global handler as a + // 500. A blank draft is a normal user state ("just created, nothing + // typed yet"), so we surface a friendly 400 here. + if (row.getDraftJson() == null || row.getDraftJson().trim().isEmpty()) { + return ResponseEntity.badRequest() + .body(R.fail("workflow has no draft to compile: " + id)); + } + WorkflowCompiler.Result result; + try { + // PublishContext is (workspaceId, publisherId) — mind the order. + result = compiler.compile(row.getDraftJson(), + new PublishContext(row.getWorkspaceId(), 0L), aclPort); + } catch (vip.mate.workflow.compiler.WorkflowParseException e) { + // Malformed JSON / structurally invalid graph → render as a + // single-error compile failure so the UI's existing errors + // panel handles it without a stack trace dialog. + return ResponseEntity.unprocessableEntity().body(buildCompileFailure(List.of( + new vip.mate.workflow.compiler.CompileError( + "graph.parse_failed", "/", e.getMessage())))); + } + if (!result.ok()) { + return ResponseEntity.unprocessableEntity() + .body(buildCompileFailure(result.errors())); + } + return ResponseEntity.ok(R.ok()); + } + + @Operation(summary = "Compile the draft and persist a new revision pointed at by latest_revision_id.") + @PostMapping("/{id}/publish") + public ResponseEntity publish(@PathVariable long id, + @RequestBody(required = false) WorkflowPublishRequest body, + @RequestParam(value = "userId", required = false) Long userId, + @RequestHeader("X-Workspace-Id") long workspaceId) { + try { + WorkflowService.PublishOutcome outcome = workflowService.publish(id, workspaceId, userId, + body == null ? null : body.note()); + return ResponseEntity.ok(R.ok(outcome)); + } catch (WorkflowCompileFailedException e) { + return ResponseEntity.unprocessableEntity().body(buildCompileFailure(e.errors())); + } catch (vip.mate.workflow.compiler.WorkflowParseException e) { + // Same surface as a compile error so the UI errors panel + // handles a malformed / blank draft without a 500 dialog. + return ResponseEntity.unprocessableEntity().body(buildCompileFailure(List.of( + new vip.mate.workflow.compiler.CompileError( + "graph.parse_failed", "/", e.getMessage())))); + } catch (IllegalArgumentException | IllegalStateException e) { + return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(R.fail(e.getMessage())); + } + } + + @Operation(summary = "Soft-delete a workflow row.") + @DeleteMapping("/{id}") + public R delete(@PathVariable long id, + @RequestHeader("X-Workspace-Id") long workspaceId) { + workflowService.delete(id, workspaceId); + return R.ok(); + } + + @Operation(summary = "List the most recent runs for a workflow.") + @GetMapping("/{id}/runs") + public R> listRuns(@PathVariable long id, + @RequestParam(value = "limit", defaultValue = "50") int limit, + @RequestHeader("X-Workspace-Id") long workspaceId) { + // Verify the parent workflow belongs to the caller's workspace + // before listing run rows, otherwise a caller could enumerate + // every workspace's runs by guessing workflow ids. + if (workflowService.get(id, workspaceId) == null) { + return R.fail("workflow not found: " + id); + } + int capped = Math.min(Math.max(limit, 1), 200); + List rows = runMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, id) + .eq(WorkflowRunEntity::getWorkspaceId, workspaceId) + .orderByDesc(WorkflowRunEntity::getStartedAt) + .last("LIMIT " + capped)); + return R.ok(rows); + } + + @Operation(summary = "List paused runs across the workspace so operators can resume them.") + @GetMapping("/runs/paused") + public R> listPausedRuns(@RequestParam(value = "limit", defaultValue = "50") int limit, + @RequestHeader("X-Workspace-Id") long workspaceId) { + // Without this listing surface, an await_approval pause is only + // recoverable by a caller that already happens to know the runId + // and pauseToken — i.e. orphaned for any human operator. The + // shape is small (run + active pause token) because operator UIs + // primarily need to know "which runs are blocked, and how do I + // resume them". + int capped = Math.min(Math.max(limit, 1), 200); + List paused = runMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkspaceId, workspaceId) + .eq(WorkflowRunEntity::getState, "paused") + .orderByDesc(WorkflowRunEntity::getStartedAt) + .last("LIMIT " + capped)); + if (paused.isEmpty()) return R.ok(List.of()); + List out = new java.util.ArrayList<>(paused.size()); + for (WorkflowRunEntity run : paused) { + WorkflowRunPauseEntity pause = pauseMapper.selectOne( + new LambdaQueryWrapper() + .eq(WorkflowRunPauseEntity::getRunId, run.getId()) + .isNull(WorkflowRunPauseEntity::getResumedAt) + .orderByDesc(WorkflowRunPauseEntity::getPausedAt) + .last("LIMIT 1")); + out.add(new PausedRunSummary(run, pause)); + } + return R.ok(out); + } + + @Operation(summary = "Inspect a single run with its step rows for replay / debugging.") + @GetMapping("/runs/{runId}") + public R getRun(@PathVariable long runId, + @RequestHeader("X-Workspace-Id") long workspaceId) { + WorkflowRunEntity run = runMapper.selectById(runId); + if (run == null) return R.fail("run not found: " + runId); + if (run.getWorkspaceId() == null || run.getWorkspaceId() != workspaceId) { + // Same surface as "not found" — don't leak run id existence + // to non-owning workspaces. + return R.fail("run not found: " + runId); + } + List steps = stepMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunStepEntity::getRunId, runId) + .orderByAsc(WorkflowRunStepEntity::getStepIndex) + .orderByAsc(WorkflowRunStepEntity::getIterationIndex)); + // Include the most recent unresolved pause so the caller can wire + // a "resume" button without a second roundtrip. + WorkflowRunPauseEntity activePause = pauseMapper.selectOne( + new LambdaQueryWrapper() + .eq(WorkflowRunPauseEntity::getRunId, runId) + .isNull(WorkflowRunPauseEntity::getResumedAt) + .orderByDesc(WorkflowRunPauseEntity::getPausedAt) + .last("LIMIT 1")); + return R.ok(new RunDetail(run, steps, activePause)); + } + + @Operation(summary = "Generate a workflow draft from a natural-language description.") + @PostMapping("/draft/generate") + public ResponseEntity generateDraft(@RequestBody DraftGenerateRequest body, + @RequestHeader("X-Workspace-Id") long workspaceId) { + if (draftGenerator == null) { + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE) + .body(R.fail("workflow draft generator is not configured on this deployment")); + } + if (body == null || body.description() == null || body.description().isBlank()) { + return ResponseEntity.badRequest().body(R.fail("description is required")); + } + try { + return ResponseEntity.ok(R.ok(draftGenerator.generate(body.description(), workspaceId))); + } catch (IllegalArgumentException e) { + return ResponseEntity.badRequest().body(R.fail(e.getMessage())); + } catch (IllegalStateException e) { + return ResponseEntity.status(HttpStatus.BAD_GATEWAY).body(R.fail(e.getMessage())); + } + } + + @Operation(summary = "List the canonical workflow templates the generator can apply directly.") + @GetMapping("/draft/templates") + public R> listDraftTemplates() { + if (draftTemplates == null) return R.ok(List.of()); + return R.ok(draftTemplates.all()); + } + + @Operation(summary = "Compile arbitrary draft JSON without persisting — used by the template picker / generator preview to surface real ACL + schema diagnostics before a workflow row exists.") + @PostMapping("/draft/preview-compile") + public ResponseEntity previewCompile(@RequestBody WorkflowDraftRequest body, + @RequestHeader("X-Workspace-Id") long workspaceId) { + if (body == null || body.draftJson() == null || body.draftJson().isBlank()) { + return ResponseEntity.badRequest().body(R.fail("draftJson is required")); + } + WorkflowCompiler.Result result; + try { + result = compiler.compile(body.draftJson(), + new PublishContext(workspaceId, 0L), aclPort); + } catch (vip.mate.workflow.compiler.WorkflowParseException e) { + return ResponseEntity.unprocessableEntity().body(buildCompileFailure(List.of( + new vip.mate.workflow.compiler.CompileError( + "graph.parse_failed", "/", e.getMessage())))); + } + if (!result.ok()) { + return ResponseEntity.unprocessableEntity() + .body(buildCompileFailure(result.errors())); + } + return ResponseEntity.ok(R.ok()); + } + + public record DraftGenerateRequest(String description) {} + + /** Narrow patch shape for {@link #update}; keeps the metadata path + * from accepting fields that would clobber the draft. */ + public record WorkflowMetadataRequest(String name, String description, Boolean enabled) {} + + public record RunDetail(WorkflowRunEntity run, + List steps, + WorkflowRunPauseEntity activePause) {} + + public record PausedRunSummary(WorkflowRunEntity run, WorkflowRunPauseEntity pause) {} + + private static R buildCompileFailure(List errors) { + R r = new R<>(); + r.setCode(422); + r.setMsg("compile failed"); + r.setData(CompileErrorResponse.of(errors)); + return r; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowDraftRequest.java b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowDraftRequest.java new file mode 100644 index 00000000..d0d5f30c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowDraftRequest.java @@ -0,0 +1,10 @@ +package vip.mate.workflow.api; + +/** + * Request body for {@code PUT /api/v1/workflows/{id}/draft}. The wire format + * matches {@code mate_workflow.draft_json} verbatim — the controller does + * not reshape this before persisting, so the editor / API caller owns the + * exact JSON the publish-time compiler will see. + */ +public record WorkflowDraftRequest(String draftJson) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowPublishRequest.java b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowPublishRequest.java new file mode 100644 index 00000000..50fe103b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowPublishRequest.java @@ -0,0 +1,8 @@ +package vip.mate.workflow.api; + +/** + * Request body for {@code POST /api/v1/workflows/{id}/publish}. {@code note} + * is the human-friendly publish note recorded on the new revision row. + */ +public record WorkflowPublishRequest(String note) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowResumeController.java b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowResumeController.java new file mode 100644 index 00000000..55df9356 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/api/WorkflowResumeController.java @@ -0,0 +1,137 @@ +package vip.mate.workflow.api; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.tags.Tag; +import lombok.RequiredArgsConstructor; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; +import vip.mate.common.result.R; +import vip.mate.workflow.compiler.PublishContext; +import vip.mate.workflow.compiler.WorkflowAclPort; +import vip.mate.workflow.compiler.WorkflowCompiler; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.model.WorkflowRevisionEntity; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.model.WorkflowRunPauseEntity; +import vip.mate.workflow.repository.WorkflowRevisionMapper; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.repository.WorkflowRunPauseMapper; +import vip.mate.workflow.runtime.WorkflowResumer; +import vip.mate.workflow.service.WorkflowService; + +import java.nio.charset.StandardCharsets; + +/** + * HTTP surface for resuming an {@code await_approval} pause. + * + *

The pause itself is opened by {@code AwaitApprovalStepAdapter} when a + * step transitions to PAUSED; this controller is what advances the run + * once a human (operator UI / approval webhook / timeout sweeper) + * decides the outcome. Without a public endpoint here, every paused run + * would be stuck until someone called {@code WorkflowResumer} from + * inside the JVM — exactly the gap the design called out as + * "v0 functionally broken". + * + *

v0 supports the operator-driven path: an authorised user in the + * owning workspace POSTs the pauseToken and an outcome. v1 will add the + * webhook callback that {@code ApprovalWorkflowService.requestWorkflowApproval} + * fires once the platform has a real workflow-approval pending row. + */ +@Tag(name = "工作流恢复") +@RestController +@RequestMapping("/api/v1/workflows/runs") +@RequiredArgsConstructor +public class WorkflowResumeController { + + private final WorkflowResumer resumer; + private final WorkflowRunMapper runMapper; + private final WorkflowRunPauseMapper pauseMapper; + private final WorkflowRevisionMapper revisionMapper; + private final WorkflowService workflowService; + private final WorkflowCompiler compiler; + private final WorkflowAclPort aclPort; + + @Operation(summary = "Resume a paused workflow run with the given outcome.") + @PostMapping("/{runId}/resume") + public ResponseEntity resume(@PathVariable long runId, + @RequestBody ResumeRequest body, + @RequestHeader("X-Workspace-Id") long workspaceId) { + if (body == null || body.pauseToken() == null || body.pauseToken().isBlank()) { + return ResponseEntity.badRequest().body(R.fail("pauseToken is required")); + } + WorkflowResumer.ResumeOutcome outcome = parseOutcome(body.outcome()); + if (outcome == null) { + return ResponseEntity.badRequest() + .body(R.fail("outcome must be one of: approved / rejected / timeout / cancelled")); + } + + WorkflowRunEntity run = runMapper.selectById(runId); + if (run == null + || run.getWorkspaceId() == null + || run.getWorkspaceId() != workspaceId) { + // Same surface as "not found" so tenants can't probe foreign run ids. + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(R.fail("run not found: " + runId)); + } + + // Validate the pause token belongs to this run before doing anything. + // Without this, a token leaked from one workspace could resume a run + // in another workspace just because the resumer doesn't itself check + // the workspace-vs-token coupling. + WorkflowRunPauseEntity pause = pauseMapper.selectOne( + new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .eq(WorkflowRunPauseEntity::getPauseToken, body.pauseToken()) + .last("LIMIT 1")); + if (pause == null || pause.getRunId() == null || pause.getRunId() != runId) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(R.fail("pause not found for run " + runId)); + } + + // Re-compile the locked revision to materialize a graph the resumer + // can walk. Compile errors here would mean a published revision is + // unparseable — should never happen in practice but we surface 500 + // explicitly rather than crashing inside the resumer. + WorkflowEntity workflow = workflowService.get(run.getWorkflowId(), workspaceId); + if (workflow == null) { + return ResponseEntity.status(HttpStatus.NOT_FOUND) + .body(R.fail("workflow not found: " + run.getWorkflowId())); + } + WorkflowRevisionEntity revision = revisionMapper.selectById(run.getRevisionId()); + if (revision == null) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(R.fail("revision " + run.getRevisionId() + " missing for run " + runId)); + } + // PublishContext is (workspaceId, publisherId) — mind the order; + // ACL resolution scopes by workspace. + WorkflowCompiler.Result compiled = compiler.compile(revision.getGraphJson(), + new PublishContext(run.getWorkspaceId(), 0L), aclPort); + if (!compiled.ok()) { + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) + .body(R.fail("revision graph failed to recompile on resume")); + } + + byte[] payload = (body.payload() == null || body.payload().isEmpty()) + ? null + : body.payload().getBytes(StandardCharsets.UTF_8); + + WorkflowResumer.Outcome result = resumer.resume(compiled.graph(), body.pauseToken(), outcome, payload); + return ResponseEntity.ok(R.ok(new ResumeResponse(result.kind().name(), + result.runId(), result.errorMessage()))); + } + + private static WorkflowResumer.ResumeOutcome parseOutcome(String token) { + if (token == null) return null; + String t = token.trim().toLowerCase(); + return switch (t) { + case "approved" -> WorkflowResumer.ResumeOutcome.APPROVED; + case "rejected" -> WorkflowResumer.ResumeOutcome.REJECTED; + case "timeout" -> WorkflowResumer.ResumeOutcome.TIMEOUT; + case "cancelled" -> WorkflowResumer.ResumeOutcome.CANCELLED; + default -> null; + }; + } + + public record ResumeRequest(String pauseToken, String outcome, String payload) {} + public record ResumeResponse(String kind, Long runId, String errorMessage) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/CompileError.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/CompileError.java new file mode 100644 index 00000000..d4ab52ff --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/CompileError.java @@ -0,0 +1,19 @@ +package vip.mate.workflow.compiler; + +/** + * Single workflow compile-time diagnostic. {@code path} points at the offending + * field using a JSONPath-ish notation rooted at the workflow definition (e.g. + * {@code steps[2].mode.expression} or {@code steps[5]}). + */ +public record CompileError(String code, String path, String message) { + + /** Convenience for step-rooted errors. */ + public static CompileError step(int index, String code, String message) { + return new CompileError(code, "steps[" + index + "]", message); + } + + /** Step-rooted error pointing at a specific sub-field. */ + public static CompileError stepField(int index, String field, String code, String message) { + return new CompileError(code, "steps[" + index + "]." + field, message); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ExpressionException.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ExpressionException.java new file mode 100644 index 00000000..ff94933a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ExpressionException.java @@ -0,0 +1,11 @@ +package vip.mate.workflow.compiler; + +/** + * Raised by {@link PebbleSubsetEvaluator} on parse or evaluate failures so + * callers (the schema validator, output-content-type checker, and runtime) + * see a single exception type for all expression-language errors. + */ +public class ExpressionException extends RuntimeException { + public ExpressionException(String message) { super(message); } + public ExpressionException(String message, Throwable cause) { super(message, cause); } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/OutputContentTypeChecker.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/OutputContentTypeChecker.java new file mode 100644 index 00000000..e3228845 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/OutputContentTypeChecker.java @@ -0,0 +1,99 @@ +package vip.mate.workflow.compiler; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.compiler.ir.WorkflowStep; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Compile-time guard against accessing a sub-field on a step output whose + * content type is plain text. The rule: + *

    + *
  • {@code outputs.X} is always allowed — the value is always defined as + * a string for text outputs and as a parsed JSON for json outputs.
  • + *
  • {@code outputs.X.field} is only allowed when step X has + * {@code outputContentType: json}; on a text output the access raises + * a compile-time error.
  • + *
+ * + *

The check uses a regex over the expression / template source rather + * than a full Pebble AST walk. This is good enough for v0 — the only + * sub-field reads that matter are the literal {@code outputs..} + * pattern; users who genuinely need richer JSON paths use the {@code | jq} + * filter (added in Lane 2) instead of dotted access. + */ +@Component +public class OutputContentTypeChecker { + + private static final Pattern OUTPUT_FIELD_REF = Pattern.compile( + "\\boutputs\\.([A-Za-z_][A-Za-z0-9_]*)\\.([A-Za-z_][A-Za-z0-9_.]*)"); + + public List check(WorkflowGraph graph) { + if (graph == null || graph.steps().isEmpty()) { + return List.of(); + } + Map outputVarToContentType = collectOutputVars(graph); + + List errors = new ArrayList<>(); + for (int i = 0; i < graph.steps().size(); i++) { + WorkflowStep s = graph.steps().get(i); + // Each step contributes a few sources that may carry expressions: + // promptTemplate, conditional.expression, dispatch_channel.content, + // write_memory.content. Walk them all. + checkSource(i, "promptTemplate", s.promptTemplate(), outputVarToContentType, errors); + if (s.mode() instanceof StepMode.Conditional c) { + checkSource(i, "mode.expression", c.expression(), outputVarToContentType, errors); + } else if (s.mode() instanceof StepMode.DispatchChannel d) { + checkSource(i, "mode.content", d.content(), outputVarToContentType, errors); + } else if (s.mode() instanceof StepMode.WriteMemory w) { + checkSource(i, "mode.content", w.content(), outputVarToContentType, errors); + } + } + return errors; + } + + private static Map collectOutputVars(WorkflowGraph graph) { + Map out = new HashMap<>(); + for (WorkflowStep s : graph.steps()) { + String var = s.outputVar(); + if (var != null && !var.isBlank()) { + out.put(var, s.effectiveOutputContentType()); + } + } + return out; + } + + private static void checkSource(int stepIndex, String fieldPath, String source, + Map outputContentTypes, + List errors) { + if (source == null || source.isEmpty()) { + return; + } + Matcher m = OUTPUT_FIELD_REF.matcher(source); + while (m.find()) { + String varName = m.group(1); + String fieldRest = m.group(2); + String contentType = outputContentTypes.get(varName); + if (contentType == null) { + errors.add(CompileError.stepField(stepIndex, fieldPath, + "expression.unknown_output_var", + "expression references unknown outputVar '" + varName + "'")); + continue; + } + if (!"json".equals(contentType)) { + errors.add(CompileError.stepField(stepIndex, fieldPath, + "expression.field_on_text_output", + "cannot access '." + fieldRest + "' on output '" + varName + + "' because its outputContentType is text — " + + "set outputContentType: json on the producing step")); + } + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PebbleSubsetEvaluator.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PebbleSubsetEvaluator.java new file mode 100644 index 00000000..d0aface2 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PebbleSubsetEvaluator.java @@ -0,0 +1,141 @@ +package vip.mate.workflow.compiler; + +import io.pebbletemplates.pebble.PebbleEngine; +import io.pebbletemplates.pebble.template.PebbleTemplate; +import org.springframework.stereotype.Component; + +import java.io.StringWriter; +import java.io.Writer; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Wraps Pebble with a v0 expression-language subset suitable for workflow + * conditionals and string templates. The wrapper: + *

    + *
  • Pre-screens the source for blocked tags ({@code {% include %}}, + * {@code {% extends %}}, {@code {% import %}}, {@code {% from %}}, + * {@code {% set %}}, {@code {% macro %}}, {@code {% block %}}). These + * reach beyond the expression sandbox and are never required for a + * workflow expression.
  • + *
  • Disables auto-escaping (workflow content is not HTML), turns the + * template cache off (each compile is one-shot), and runs in + * non-strict variable mode so {@code default('x')} and missing-field + * access remain ergonomic.
  • + *
  • Treats expressions and full string templates as the same engine + * artifact — {@link #parseExpression(String)} accepts either the bare + * expression ({@code outputs.x.tier == 'enterprise'}) or the wrapped + * form ({@code "{{ outputs.x.tier == 'enterprise' }}"}).
  • + *
+ * + *

JSONPath-style filtering (the {@code | jq('.foo')} syntax in the design + * doc) is intentionally not yet wired here — Day 2-3 ships only the engine + * wrapper plus parse / evaluate; the {@code jq} filter will be added in + * Lane 2 alongside its runtime tests so we can exercise it against real + * step outputs. + */ +@Component +public class PebbleSubsetEvaluator { + + private static final Pattern BLOCKED_TAG_PATTERN = Pattern.compile( + "\\{%\\s*(include|extends|import|from|set|macro|block)\\b", + Pattern.CASE_INSENSITIVE); + + /** Wrapping form recognized for bare conditional expressions. */ + private static final Pattern WRAPPED_EXPRESSION = Pattern.compile( + "^\\s*\\{\\{(.*)\\}\\}\\s*$", Pattern.DOTALL); + + private final PebbleEngine engine; + + public PebbleSubsetEvaluator() { + this.engine = new PebbleEngine.Builder() + .strictVariables(false) + .cacheActive(false) + .autoEscaping(false) + .build(); + } + + /** + * Parse a conditional expression into a compiled artifact ready for + * repeated evaluation. Accepts either {@code expr} or {@code "{{ expr }}"}. + */ + public Compiled parseExpression(String expression) { + if (expression == null || expression.isBlank()) { + throw new ExpressionException("expression is empty"); + } + rejectBlockedTags(expression); + + String inner = stripWrapping(expression); + String source = "{{ " + inner + " }}"; + return compile(source, expression); + } + + /** + * Parse a multi-segment string template (prompt template, dispatch_channel + * content, write_memory content). The whole string is treated as a Pebble + * template body. + */ + public Compiled parseTemplate(String template) { + if (template == null) { + throw new ExpressionException("template is null"); + } + rejectBlockedTags(template); + return compile(template, template); + } + + public String evaluateAsString(Compiled compiled, Map context) { + StringWriter writer = new StringWriter(); + evaluate(compiled, context, writer); + return writer.toString(); + } + + public boolean evaluateAsBoolean(Compiled compiled, Map context) { + String rendered = evaluateAsString(compiled, context).trim(); + return "true".equalsIgnoreCase(rendered); + } + + private void evaluate(Compiled compiled, Map context, Writer writer) { + try { + compiled.template.evaluate(writer, context == null ? Map.of() : context); + } catch (Exception e) { + throw new ExpressionException( + "expression evaluation failed: " + e.getMessage() + + " (source: " + compiled.originalSource + ")", + e); + } + } + + private Compiled compile(String pebbleSource, String originalSource) { + try { + // getLiteralTemplate uses the source string itself as the template + // body, bypassing the Loader (which is the right call here — we + // never want to read templates from the filesystem or classpath). + PebbleTemplate template = engine.getLiteralTemplate(pebbleSource); + return new Compiled(template, originalSource); + } catch (Exception e) { + throw new ExpressionException( + "expression parse failed: " + e.getMessage() + + " (source: " + originalSource + ")", + e); + } + } + + private static void rejectBlockedTags(String source) { + Matcher m = BLOCKED_TAG_PATTERN.matcher(source); + if (m.find()) { + throw new ExpressionException( + "expression uses blocked tag '" + m.group(1) + + "' — workflow expressions only allow {{ ... }} substitutions"); + } + } + + private static String stripWrapping(String expression) { + Matcher m = WRAPPED_EXPRESSION.matcher(expression); + return m.matches() ? m.group(1).trim() : expression.trim(); + } + + /** Compiled, reusable expression. */ + public record Compiled(PebbleTemplate template, String originalSource) { + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PublishContext.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PublishContext.java new file mode 100644 index 00000000..4c950065 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/PublishContext.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.compiler; + +/** + * Immutable scope passed to publish-time validators: the workspace the + * workflow lives in plus the user attempting to publish. ACL checks compare + * these against the resolvable agent / channel / employee scope. + */ +public record PublishContext(long workspaceId, Long publisherId) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclPort.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclPort.java new file mode 100644 index 00000000..86bd2cb9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclPort.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.compiler; + +/** + * Pluggable ACL probe used by {@link WorkflowAclValidator}. The validator + * stays free of Spring-bean dependencies (mapper / service injection) so its + * unit tests can stub a port directly. The runtime wiring sits in + * {@code vip.mate.workflow.runtime} where this port is implemented in terms + * of {@code AgentBindingService}, the workspace channel allowlist, and the + * mate_skill.enabled view. + */ +public interface WorkflowAclPort { + + /** True if the named agent exists, is enabled, and lives in the workspace. */ + boolean agentExists(long workspaceId, String agentName); + + /** True if the agentId resolves to an enabled agent in the workspace. */ + boolean agentIdExists(long workspaceId, long agentId); + + /** True if the channel is on the workspace allowlist. */ + boolean channelAllowed(long workspaceId, String channelName); + + /** True if employeeId is a member of the workspace. */ + boolean employeeInWorkspace(long workspaceId, String employeeId); +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclValidator.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclValidator.java new file mode 100644 index 00000000..40b2ff29 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowAclValidator.java @@ -0,0 +1,106 @@ +package vip.mate.workflow.compiler; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.compiler.ir.WorkflowStep; + +import java.util.ArrayList; +import java.util.List; + +/** + * Publish-time access-control validator. For each step that touches an + * external scope (agent / channel / employee memory), the validator asks + * the {@link WorkflowAclPort} whether the reference resolves inside the + * publishing workspace. Any negative answer is recorded as a + * {@link CompileError}; downstream the publish flow refuses to write a new + * revision when the error list is non-empty. + * + *

Pure structural ACL — workflow-level actor identity (the publisher + * versus the runtime acting agent) is enforced separately when steps are + * registered with the runtime, where {@code AgentBindingService.getEffectiveToolNames} + * applies the per-agent tool ACL. + */ +@Component +public class WorkflowAclValidator { + + public List validate(WorkflowGraph graph, PublishContext ctx, WorkflowAclPort port) { + if (graph == null || graph.steps().isEmpty()) { + return List.of(); + } + List errors = new ArrayList<>(); + for (int i = 0; i < graph.steps().size(); i++) { + WorkflowStep s = graph.steps().get(i); + checkAgent(i, s, ctx, port, errors); + checkChannels(i, s, ctx, port, errors); + checkEmployee(i, s, ctx, port, errors); + } + return errors; + } + + private static void checkAgent(int i, WorkflowStep s, PublishContext ctx, + WorkflowAclPort port, List errors) { + if (s.mode() instanceof StepMode.AwaitApproval + || s.mode() instanceof StepMode.Collect + || s.mode() instanceof StepMode.DispatchChannel + || s.mode() instanceof StepMode.WriteMemory) { + return; // these modes do not invoke an agent at runtime + } + if (s.agentId() != null) { + if (!port.agentIdExists(ctx.workspaceId(), s.agentId())) { + errors.add(CompileError.stepField(i, "agentId", + "acl.agent_not_resolvable", + "agentId " + s.agentId() + " does not resolve to an enabled agent in this workspace")); + } + return; + } + if (s.agentName() != null && !s.agentName().isBlank() + && !port.agentExists(ctx.workspaceId(), s.agentName())) { + errors.add(CompileError.stepField(i, "agentName", + "acl.agent_not_resolvable", + "agent '" + s.agentName() + "' does not resolve to an enabled agent in this workspace")); + } + } + + private static void checkChannels(int i, WorkflowStep s, PublishContext ctx, + WorkflowAclPort port, List errors) { + if (!(s.mode() instanceof StepMode.DispatchChannel d)) { + return; + } + if (d.channels() == null) return; + for (int c = 0; c < d.channels().size(); c++) { + String ch = d.channels().get(c); + if (ch == null || ch.isBlank()) continue; + if (!port.channelAllowed(ctx.workspaceId(), ch)) { + errors.add(CompileError.stepField(i, "mode.channels[" + c + "]", + "acl.channel_not_allowed", + "channel '" + ch + "' is not on the workspace allowlist")); + } + } + } + + private static void checkEmployee(int i, WorkflowStep s, PublishContext ctx, + WorkflowAclPort port, List errors) { + if (!(s.mode() instanceof StepMode.WriteMemory w)) { + return; + } + // Pebble templates resolve at runtime — do not ACL-check expressions + // that aren't a literal employee id. Literal forms are the safe + // common case worth guarding. + if (w.employeeId() == null || w.employeeId().isBlank()) { + return; + } + if (containsTemplate(w.employeeId())) { + return; + } + if (!port.employeeInWorkspace(ctx.workspaceId(), w.employeeId())) { + errors.add(CompileError.stepField(i, "mode.employeeId", + "acl.employee_not_in_workspace", + "employeeId '" + w.employeeId() + "' is not a member of this workspace")); + } + } + + private static boolean containsTemplate(String s) { + return s != null && s.contains("{{"); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompileFailedException.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompileFailedException.java new file mode 100644 index 00000000..670e1848 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompileFailedException.java @@ -0,0 +1,36 @@ +package vip.mate.workflow.compiler; + +import java.util.List; + +/** + * Thrown by {@link WorkflowCompiler.Result#requireOk()} when at least one + * compile error was raised. The error list is preserved on the exception so + * callers (REST endpoints, persistence layers) can surface every problem + * back to the publishing user without losing diagnostic context. + */ +public class WorkflowCompileFailedException extends RuntimeException { + + private final List errors; + + public WorkflowCompileFailedException(List errors) { + super(buildMessage(errors)); + this.errors = List.copyOf(errors); + } + + public List errors() { + return errors; + } + + private static String buildMessage(List errors) { + if (errors == null || errors.isEmpty()) { + return "workflow compile failed"; + } + StringBuilder sb = new StringBuilder(); + sb.append("workflow compile failed with ").append(errors.size()).append(" error(s):"); + for (CompileError e : errors) { + sb.append("\n - [").append(e.code()).append("] ").append(e.path()) + .append(": ").append(e.message()); + } + return sb.toString(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompiler.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompiler.java new file mode 100644 index 00000000..d0ea6fad --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowCompiler.java @@ -0,0 +1,112 @@ +package vip.mate.workflow.compiler; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.WorkflowGraph; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Top-level compile entry point. Runs each pass in order and collects the + * resulting diagnostics into a single {@link Result}. Phases short-circuit + * on the kind of failure that would invalidate later passes: + *

    + *
  • Parse failure raises a {@link WorkflowParseException} immediately — + * structural validation needs an IR.
  • + *
  • Schema, expression, and ACL checks are independent and all run, so + * a single compile call surfaces every problem instead of + * error-by-error round-trips.
  • + *
+ */ +@Component +public class WorkflowCompiler { + + private final WorkflowParser parser; + private final WorkflowSchemaValidator schemaValidator; + private final OutputContentTypeChecker outputContentTypeChecker; + private final WorkflowAclValidator aclValidator; + private final PebbleSubsetEvaluator pebbleEvaluator; + + public WorkflowCompiler(WorkflowParser parser, + WorkflowSchemaValidator schemaValidator, + OutputContentTypeChecker outputContentTypeChecker, + WorkflowAclValidator aclValidator, + PebbleSubsetEvaluator pebbleEvaluator) { + this.parser = parser; + this.schemaValidator = schemaValidator; + this.outputContentTypeChecker = outputContentTypeChecker; + this.aclValidator = aclValidator; + this.pebbleEvaluator = pebbleEvaluator; + } + + public Result compile(String json, PublishContext ctx, WorkflowAclPort aclPort) { + WorkflowGraph graph = parser.parse(json); + List errors = new ArrayList<>(); + errors.addAll(schemaValidator.validate(graph)); + errors.addAll(checkExpressionSyntax(graph)); + errors.addAll(outputContentTypeChecker.check(graph)); + if (aclPort != null) { + errors.addAll(aclValidator.validate(graph, ctx, aclPort)); + } + return new Result(graph, Collections.unmodifiableList(errors)); + } + + private List checkExpressionSyntax(WorkflowGraph graph) { + List errors = new ArrayList<>(); + for (int i = 0; i < graph.steps().size(); i++) { + var step = graph.steps().get(i); + if (step.mode() instanceof vip.mate.workflow.compiler.ir.StepMode.Conditional c + && c.expression() != null && !c.expression().isBlank()) { + try { + pebbleEvaluator.parseExpression(c.expression()); + } catch (ExpressionException e) { + errors.add(CompileError.stepField(i, "mode.expression", + "expression.parse_failed", e.getMessage())); + } + } + if (step.promptTemplate() != null && !step.promptTemplate().isBlank()) { + try { + pebbleEvaluator.parseTemplate(step.promptTemplate()); + } catch (ExpressionException e) { + errors.add(CompileError.stepField(i, "promptTemplate", + "expression.parse_failed", e.getMessage())); + } + } + if (step.mode() instanceof vip.mate.workflow.compiler.ir.StepMode.DispatchChannel d + && d.content() != null && !d.content().isBlank()) { + try { + pebbleEvaluator.parseTemplate(d.content()); + } catch (ExpressionException e) { + errors.add(CompileError.stepField(i, "mode.content", + "expression.parse_failed", e.getMessage())); + } + } + if (step.mode() instanceof vip.mate.workflow.compiler.ir.StepMode.WriteMemory w + && w.content() != null && !w.content().isBlank()) { + try { + pebbleEvaluator.parseTemplate(w.content()); + } catch (ExpressionException e) { + errors.add(CompileError.stepField(i, "mode.content", + "expression.parse_failed", e.getMessage())); + } + } + } + return errors; + } + + /** + * Compile result. Callers that want strictness can do + * {@code result.requireOk()}; the publish flow uses that to refuse + * persisting a new revision row when there are errors. + */ + public record Result(WorkflowGraph graph, List errors) { + public boolean ok() { return errors.isEmpty(); } + + public void requireOk() { + if (!ok()) { + throw new WorkflowCompileFailedException(errors); + } + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParseException.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParseException.java new file mode 100644 index 00000000..2b870155 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParseException.java @@ -0,0 +1,12 @@ +package vip.mate.workflow.compiler; + +/** + * Thrown by {@link WorkflowParser} when the JSON wire format cannot be turned + * into a {@link vip.mate.workflow.compiler.ir.WorkflowGraph}. Distinct from + * {@link CompileError} so that wire-format problems never reach the validator + * passes — those operate exclusively on a syntactically valid IR. + */ +public class WorkflowParseException extends RuntimeException { + public WorkflowParseException(String message) { super(message); } + public WorkflowParseException(String message, Throwable cause) { super(message, cause); } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParser.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParser.java new file mode 100644 index 00000000..79244e13 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowParser.java @@ -0,0 +1,243 @@ +package vip.mate.workflow.compiler; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.ErrorMode; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.compiler.ir.WorkflowInput; +import vip.mate.workflow.compiler.ir.WorkflowStep; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +/** + * Parse the workflow JSON wire format into the immutable {@link WorkflowGraph} + * IR. The parser is structural-only: it surfaces malformed JSON and unknown + * mode types as {@link WorkflowParseException}s but does not run schema / + * expression / ACL validation — those passes consume the IR and emit + * {@link CompileError}s. + * + *

Field naming matches the wire format documented in the workflow design + * (see {@code mate_workflow_revision.graph_json}). + */ +@Component +public class WorkflowParser { + + private final ObjectMapper objectMapper; + + public WorkflowParser(ObjectMapper objectMapper) { + this.objectMapper = objectMapper; + } + + public WorkflowGraph parse(String json) { + if (json == null || json.isBlank()) { + throw new WorkflowParseException("workflow definition is empty"); + } + JsonNode root; + try { + root = objectMapper.readTree(json); + } catch (Exception e) { + throw new WorkflowParseException("workflow JSON is not parseable: " + e.getMessage(), e); + } + if (!root.isObject()) { + throw new WorkflowParseException("workflow definition root must be a JSON object"); + } + + String schemaVersion = textOrNull(root.get("schemaVersion")); + List inputs = parseInputs(root.get("inputs")); + List steps = parseSteps(root.get("steps")); + + return new WorkflowGraph(schemaVersion, inputs, steps); + } + + private List parseInputs(JsonNode node) { + if (node == null || node.isNull()) { + return List.of(); + } + if (!node.isArray()) { + throw new WorkflowParseException("inputs must be a JSON array"); + } + List out = new ArrayList<>(node.size()); + for (int i = 0; i < node.size(); i++) { + JsonNode entry = node.get(i); + if (!entry.isObject()) { + throw new WorkflowParseException("inputs[" + i + "] must be a JSON object"); + } + out.add(new WorkflowInput( + textOrNull(entry.get("name")), + textOrNull(entry.get("type")) + )); + } + return out; + } + + private List parseSteps(JsonNode node) { + if (node == null || node.isNull()) { + return List.of(); + } + if (!node.isArray()) { + throw new WorkflowParseException("steps must be a JSON array"); + } + List out = new ArrayList<>(node.size()); + for (int i = 0; i < node.size(); i++) { + JsonNode raw = node.get(i); + if (!raw.isObject()) { + throw new WorkflowParseException("steps[" + i + "] must be a JSON object"); + } + out.add(parseStep(raw, i)); + } + return out; + } + + private WorkflowStep parseStep(JsonNode raw, int index) { + Long agentId = null; + JsonNode agentIdNode = raw.get("agentId"); + if (agentIdNode != null && !agentIdNode.isNull()) { + if (agentIdNode.isNumber()) { + agentId = agentIdNode.asLong(); + } else if (agentIdNode.isTextual()) { + try { + agentId = Long.parseLong(agentIdNode.asText()); + } catch (NumberFormatException e) { + throw new WorkflowParseException("steps[" + index + "].agentId must be numeric"); + } + } else { + throw new WorkflowParseException("steps[" + index + "].agentId must be numeric"); + } + } + + Integer timeoutSecs = null; + JsonNode toNode = raw.get("timeoutSecs"); + if (toNode != null && !toNode.isNull()) { + if (!toNode.isInt() && !toNode.isLong()) { + throw new WorkflowParseException("steps[" + index + "].timeoutSecs must be an integer"); + } + timeoutSecs = toNode.asInt(); + } + + StepMode mode = parseMode(raw.get("mode"), index); + ErrorMode errorMode = parseErrorMode(raw.get("errorMode"), index); + + return new WorkflowStep( + textOrNull(raw.get("name")), + textOrNull(raw.get("agentName")), + agentId, + textOrNull(raw.get("promptTemplate")), + mode, + timeoutSecs, + errorMode, + textOrNull(raw.get("outputVar")), + textOrNull(raw.get("outputContentType")) + ); + } + + private StepMode parseMode(JsonNode raw, int stepIndex) { + if (raw == null || raw.isNull()) { + throw new WorkflowParseException("steps[" + stepIndex + "].mode is required"); + } + if (!raw.isObject()) { + throw new WorkflowParseException("steps[" + stepIndex + "].mode must be a JSON object"); + } + String type = textOrNull(raw.get("type")); + if (type == null || type.isBlank()) { + throw new WorkflowParseException("steps[" + stepIndex + "].mode.type is required"); + } + return switch (type) { + case "sequential" -> new StepMode.Sequential(); + case "fan_out" -> new StepMode.FanOut(); + case "collect" -> new StepMode.Collect(); + case "conditional" -> new StepMode.Conditional(textOrNull(raw.get("expression"))); + case "await_approval" -> new StepMode.AwaitApproval( + textOrNull(raw.get("approvalKind")), + parseStringList(raw.get("approverChannels")), + textOrNull(raw.get("approvalMessage")), + raw.has("timeoutSecs") && raw.get("timeoutSecs").isInt() ? raw.get("timeoutSecs").asInt() : null + ); + case "dispatch_channel" -> new StepMode.DispatchChannel( + parseStringList(raw.get("channels")), + parseStringMap(raw.get("targets")), + textOrNull(raw.get("content")) + ); + case "write_memory" -> new StepMode.WriteMemory( + textOrNull(raw.get("employeeId")), + textOrNull(raw.get("file")), + textOrNull(raw.get("mergeStrategy")), + textOrNull(raw.get("content")) + ); + default -> throw new WorkflowParseException( + "steps[" + stepIndex + "].mode.type '" + type + + "' is not supported in v0 (loop / invoke_skill are deferred)"); + }; + } + + private ErrorMode parseErrorMode(JsonNode raw, int stepIndex) { + if (raw == null || raw.isNull()) { + return null; + } + if (!raw.isObject()) { + throw new WorkflowParseException("steps[" + stepIndex + "].errorMode must be a JSON object"); + } + String type = textOrNull(raw.get("type")); + if (type == null) { + throw new WorkflowParseException("steps[" + stepIndex + "].errorMode.type is required"); + } + return switch (type) { + case "fail" -> new ErrorMode.Fail(); + case "skip" -> new ErrorMode.Skip(); + case "retry" -> { + JsonNode mr = raw.get("maxRetries"); + int max = (mr != null && mr.isInt()) ? mr.asInt() : 1; + yield new ErrorMode.Retry(max); + } + default -> throw new WorkflowParseException( + "steps[" + stepIndex + "].errorMode.type '" + type + "' is unknown"); + }; + } + + private static String textOrNull(JsonNode node) { + if (node == null || node.isNull()) { + return null; + } + return node.isTextual() ? node.asText() : node.asText(null); + } + + private static List parseStringList(JsonNode node) { + if (node == null || node.isNull()) { + return List.of(); + } + if (!node.isArray()) { + throw new WorkflowParseException("expected JSON array, got " + node.getNodeType()); + } + List out = new ArrayList<>(node.size()); + for (int i = 0; i < node.size(); i++) { + JsonNode v = node.get(i); + if (v == null || v.isNull()) { + continue; + } + out.add(v.asText()); + } + return out; + } + + private static Map parseStringMap(JsonNode node) { + if (node == null || node.isNull()) { + return Map.of(); + } + if (!node.isObject()) { + throw new WorkflowParseException("expected JSON object, got " + node.getNodeType()); + } + Map out = new HashMap<>(); + Iterator> it = node.fields(); + while (it.hasNext()) { + Map.Entry e = it.next(); + JsonNode v = e.getValue(); + out.put(e.getKey(), v == null || v.isNull() ? null : v.asText()); + } + return out; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowSchemaValidator.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowSchemaValidator.java new file mode 100644 index 00000000..9f27de4a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/WorkflowSchemaValidator.java @@ -0,0 +1,226 @@ +package vip.mate.workflow.compiler; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.compiler.ir.WorkflowStep; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * Structural validator. Ensures required fields are present per mode, names + * are unique, the step count is bounded, and the fan_out / collect grouping + * follows the workflow design rules: + *

    + *
  • A fan_out group must have at least two consecutive fan_out steps and + * must be terminated by a collect.
  • + *
  • A collect must follow a fan_out group.
  • + *
  • An await_approval step cannot live inside a fan_out group (multiple + * concurrent approvals have no aggregation UX).
  • + *
+ * + *

Expression-language and ACL checks live in dedicated validators so each + * pass has a single responsibility. + */ +@Component +public class WorkflowSchemaValidator { + + /** Default ceiling — flags runaway templates / config mistakes early. */ + public static final int DEFAULT_MAX_STEPS = 200; + + private final int maxSteps; + + public WorkflowSchemaValidator() { this(DEFAULT_MAX_STEPS); } + + public WorkflowSchemaValidator(int maxSteps) { + this.maxSteps = maxSteps; + } + + public List validate(WorkflowGraph graph) { + List errors = new ArrayList<>(); + if (graph == null) { + errors.add(new CompileError("workflow.null", "$", "workflow definition is null")); + return errors; + } + if (graph.steps().isEmpty()) { + errors.add(new CompileError("workflow.no_steps", "steps", + "workflow must declare at least one step")); + return errors; + } + if (graph.steps().size() > maxSteps) { + errors.add(new CompileError( + "workflow.too_many_steps", + "steps", + "workflow has " + graph.steps().size() + " steps; max is " + maxSteps)); + } + + validatePerStepFields(graph, errors); + validateUniqueNames(graph, errors); + validateFanOutCollectGrouping(graph, errors); + return errors; + } + + private void validatePerStepFields(WorkflowGraph graph, List errors) { + for (int i = 0; i < graph.steps().size(); i++) { + WorkflowStep s = graph.steps().get(i); + if (s.name() == null || s.name().isBlank()) { + errors.add(CompileError.stepField(i, "name", + "step.name_required", "step name is required")); + } + if (s.mode() == null) { + errors.add(CompileError.stepField(i, "mode", + "step.mode_required", "step mode is required")); + continue; + } + String oct = s.effectiveOutputContentType(); + if (!oct.equals("text") && !oct.equals("json")) { + errors.add(CompileError.stepField(i, "outputContentType", + "step.output_content_type_unsupported", + "outputContentType must be 'text' or 'json' (got '" + oct + "')")); + } + validateModeFields(i, s, errors); + } + } + + private void validateModeFields(int i, WorkflowStep s, List errors) { + StepMode m = s.mode(); + switch (m) { + case StepMode.Sequential ignored -> requireAgent(i, s, errors); + case StepMode.FanOut ignored -> requireAgent(i, s, errors); + case StepMode.Collect ignored -> { + // Agent invocation is optional on collect — the runtime can + // either feed the collected payload into the next step or + // run an agent at this step. Both are valid v0 shapes. + } + case StepMode.Conditional c -> { + if (c.expression() == null || c.expression().isBlank()) { + errors.add(CompileError.stepField(i, "mode.expression", + "step.conditional_expression_required", + "conditional mode requires an expression")); + } + requireAgent(i, s, errors); + } + case StepMode.AwaitApproval a -> { + if (a.approvalKind() == null || a.approvalKind().isBlank()) { + errors.add(CompileError.stepField(i, "mode.approvalKind", + "step.await_approval.kind_required", + "await_approval requires approvalKind")); + } + if (a.approverChannels() == null || a.approverChannels().isEmpty()) { + errors.add(CompileError.stepField(i, "mode.approverChannels", + "step.await_approval.channels_required", + "await_approval requires at least one approverChannel")); + } + } + case StepMode.DispatchChannel d -> { + if (d.channels() == null || d.channels().isEmpty()) { + errors.add(CompileError.stepField(i, "mode.channels", + "step.dispatch_channel.channels_required", + "dispatch_channel requires at least one channel")); + } + if (d.content() == null || d.content().isBlank()) { + errors.add(CompileError.stepField(i, "mode.content", + "step.dispatch_channel.content_required", + "dispatch_channel requires content")); + } + } + case StepMode.WriteMemory w -> { + if (w.employeeId() == null || w.employeeId().isBlank()) { + errors.add(CompileError.stepField(i, "mode.employeeId", + "step.write_memory.employee_required", + "write_memory requires employeeId")); + } + if (w.file() == null || w.file().isBlank()) { + errors.add(CompileError.stepField(i, "mode.file", + "step.write_memory.file_required", + "write_memory requires file")); + } + if (w.mergeStrategy() == null || w.mergeStrategy().isBlank()) { + errors.add(CompileError.stepField(i, "mode.mergeStrategy", + "step.write_memory.merge_required", + "write_memory requires mergeStrategy")); + } else if (!isKnownMergeStrategy(w.mergeStrategy())) { + errors.add(CompileError.stepField(i, "mode.mergeStrategy", + "step.write_memory.merge_unknown", + "mergeStrategy '" + w.mergeStrategy() + + "' must be one of append / replace_section / upsert_kv / overwrite")); + } + } + } + } + + private static boolean isKnownMergeStrategy(String s) { + return "append".equals(s) || "replace_section".equals(s) + || "upsert_kv".equals(s) || "overwrite".equals(s); + } + + private static void requireAgent(int i, WorkflowStep s, List errors) { + boolean hasName = s.agentName() != null && !s.agentName().isBlank(); + boolean hasId = s.agentId() != null; + if (!hasName && !hasId) { + errors.add(CompileError.step(i, "step.agent_required", + "step requires either agentName or agentId for mode '" + + s.mode().typeName() + "'")); + } + } + + private void validateUniqueNames(WorkflowGraph graph, List errors) { + Set seen = new HashSet<>(); + for (int i = 0; i < graph.steps().size(); i++) { + String name = graph.steps().get(i).name(); + if (name == null || name.isBlank()) continue; + if (!seen.add(name)) { + errors.add(CompileError.stepField(i, "name", + "step.name_duplicate", "step name '" + name + "' is duplicated")); + } + } + } + + private void validateFanOutCollectGrouping(WorkflowGraph graph, List errors) { + List steps = graph.steps(); + int i = 0; + while (i < steps.size()) { + StepMode m = steps.get(i).mode(); + if (m instanceof StepMode.FanOut) { + int groupStart = i; + int j = i; + while (j < steps.size() && steps.get(j).mode() instanceof StepMode.FanOut) { + if (containsAwaitApproval(steps.get(j))) { + // Defensive — fan_out with await_approval mode object + // can only appear if a single step had two modes, + // which the parser already rejects. Keeping the check + // costs nothing. + } + j++; + } + int groupSize = j - groupStart; + if (groupSize < 2) { + errors.add(CompileError.step(groupStart, "step.fan_out.singleton", + "fan_out groups must have at least 2 consecutive fan_out steps")); + } + if (j >= steps.size() || !(steps.get(j).mode() instanceof StepMode.Collect)) { + errors.add(CompileError.step(groupStart, "step.fan_out.no_terminating_collect", + "fan_out group starting at step '" + steps.get(groupStart).name() + + "' must be terminated by a collect step")); + } + i = j; + continue; + } + if (m instanceof StepMode.Collect) { + if (i == 0 || !(steps.get(i - 1).mode() instanceof StepMode.FanOut)) { + errors.add(CompileError.step(i, "step.collect.no_preceding_fan_out", + "collect step must follow a fan_out group")); + } + } + i++; + } + } + + /** Always false in v0 — placeholder for future composite-mode awareness. */ + private static boolean containsAwaitApproval(WorkflowStep step) { + return step.mode() instanceof StepMode.AwaitApproval; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/ErrorMode.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/ErrorMode.java new file mode 100644 index 00000000..67b4b8f1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/ErrorMode.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.compiler.ir; + +/** + * Per-step error policy. {@code Retry} carries the retry budget; {@code Fail} + * propagates the error to the run; {@code Skip} marks the step succeeded with + * no output (downstream steps that referenced its outputVar see the previous + * variable value, mirroring the conditional-false rule). + */ +public sealed interface ErrorMode { + + String typeName(); + + record Fail() implements ErrorMode { + @Override public String typeName() { return "fail"; } + } + + record Skip() implements ErrorMode { + @Override public String typeName() { return "skip"; } + } + + record Retry(int maxRetries) implements ErrorMode { + @Override public String typeName() { return "retry"; } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/StepMode.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/StepMode.java new file mode 100644 index 00000000..2ad53cee --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/StepMode.java @@ -0,0 +1,64 @@ +package vip.mate.workflow.compiler.ir; + +import java.util.List; +import java.util.Map; + +/** + * Tagged record describing the control-flow mode of a single workflow step. + * v0 supports four base modes (sequential / fan_out / collect / conditional) + * and three MateClaw-specific modes (await_approval / dispatch_channel / + * write_memory). loop and invoke_skill are deferred to v1. + */ +public sealed interface StepMode { + + String typeName(); + + /** Sequential — runs after the previous step, threads its output forward. */ + record Sequential() implements StepMode { + @Override public String typeName() { return "sequential"; } + } + + /** Fan-out — schedules in parallel with adjacent fan_out steps. */ + record FanOut() implements StepMode { + @Override public String typeName() { return "fan_out"; } + } + + /** Collect — joins the most recent fan_out group. */ + record Collect() implements StepMode { + @Override public String typeName() { return "collect"; } + } + + /** Conditional — runs only when the Pebble expression evaluates true. */ + record Conditional(String expression) implements StepMode { + @Override public String typeName() { return "conditional"; } + } + + /** Await approval — pauses the run until the approval row resolves. */ + record AwaitApproval( + String approvalKind, + List approverChannels, + String approvalMessage, + Integer timeoutSecs + ) implements StepMode { + @Override public String typeName() { return "await_approval"; } + } + + /** Dispatch channel — fan out a payload to one or more configured channels. */ + record DispatchChannel( + List channels, + Map targets, + String content + ) implements StepMode { + @Override public String typeName() { return "dispatch_channel"; } + } + + /** Write memory — apply a merge strategy to an employee's memory file. */ + record WriteMemory( + String employeeId, + String file, + String mergeStrategy, + String content + ) implements StepMode { + @Override public String typeName() { return "write_memory"; } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowGraph.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowGraph.java new file mode 100644 index 00000000..2a47f922 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowGraph.java @@ -0,0 +1,19 @@ +package vip.mate.workflow.compiler.ir; + +import java.util.List; + +/** + * Immutable in-memory representation of a parsed workflow definition. The + * compiler operates exclusively on this IR; the original JSON is the wire + * format and is not retained past the parse stage. + */ +public record WorkflowGraph( + String schemaVersion, + List inputs, + List steps +) { + public WorkflowGraph { + inputs = inputs == null ? List.of() : List.copyOf(inputs); + steps = steps == null ? List.of() : List.copyOf(steps); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowInput.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowInput.java new file mode 100644 index 00000000..06f2fdf7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowInput.java @@ -0,0 +1,5 @@ +package vip.mate.workflow.compiler.ir; + +/** Declared workflow input. Type values are advisory: {@code text|json|number|boolean}. */ +public record WorkflowInput(String name, String type) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowStep.java b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowStep.java new file mode 100644 index 00000000..363bdc9f --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/compiler/ir/WorkflowStep.java @@ -0,0 +1,26 @@ +package vip.mate.workflow.compiler.ir; + +/** + * Single step in a workflow's linear step array. {@code mode} holds the + * type-specific configuration; common fields like timeout / retry policy / + * outputVar live here so they apply to every mode without duplication. + */ +public record WorkflowStep( + String name, + String agentName, + Long agentId, + String promptTemplate, + StepMode mode, + Integer timeoutSecs, + ErrorMode errorMode, + String outputVar, + String outputContentType +) { + + /** Resolved content type, defaulting to {@code text} when unspecified. */ + public String effectiveOutputContentType() { + return outputContentType == null || outputContentType.isBlank() + ? "text" + : outputContentType; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/GeneratedWorkflowDraft.java b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/GeneratedWorkflowDraft.java new file mode 100644 index 00000000..2bef86d3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/GeneratedWorkflowDraft.java @@ -0,0 +1,38 @@ +package vip.mate.workflow.draftgen; + +import java.util.List; +import java.util.Map; + +/** + * Result of a natural-language → workflow draft generation. Crosses the + * REST boundary as JSON; the controller returns this verbatim. + * + *

{@code draftJson} is the {@code {"steps":[...]}} shape the + * runtime expects — same string the UI's JSON tab edits, same one + * {@link vip.mate.workflow.compiler.WorkflowCompiler} consumes. The + * generator pre-runs the compiler against it and reports compile + * failures via {@code compileErrors} without auto-publishing — v0 + * always lets the operator review before pushing the row to a + * revision. + * + *

{@code triggerDrafts} is a list of suggested triggers the user + * can choose to create alongside the workflow; they're NOT created + * automatically and arrive with {@code enabled=false} per the + * generator system prompt's contract. + * + *

{@code warnings} / {@code missingFields} surface anywhere the + * model had to hedge — unfilled {@code TODO_*} placeholders, ambiguous + * approval policy, missing channel target. The UI displays these + * inline so the operator can finish the draft. + */ +public record GeneratedWorkflowDraft( + String name, + String description, + String draftJson, + List> triggerDrafts, + List warnings, + List missingFields, + Double confidence, + boolean compileOk, + List compileErrors +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowAuthoringTool.java b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowAuthoringTool.java new file mode 100644 index 00000000..9ddaa5ff --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowAuthoringTool.java @@ -0,0 +1,112 @@ +package vip.mate.workflow.draftgen; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; +import org.springframework.lang.Nullable; +import org.springframework.stereotype.Component; +import vip.mate.agent.context.ChatOrigin; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.service.WorkflowService; + +/** + * Agent-callable workflow drafting tool. + * + *

Lets a user say in chat «帮我把每周一汇总销售这件事做成 workflow» + * and have the agent compose a draft + persist it as a fresh + * {@link WorkflowEntity} row + return a short natural-language summary + * the user can act on. The created workflow stays as a draft (no + * publish, no triggers wired) — same v0 safety contract as the + * controller endpoint. + * + *

Workspace is taken from {@link ChatOrigin} on the active + * {@link ToolContext}, so the tool can never write into a foreign + * workspace even if the agent prompt tried to forge one. + */ +@Slf4j +@Component +public class WorkflowAuthoringTool { + + private final WorkflowDraftGenerator generator; + private final WorkflowService workflowService; + + public WorkflowAuthoringTool(WorkflowDraftGenerator generator, + WorkflowService workflowService) { + this.generator = generator; + this.workflowService = workflowService; + } + + @Tool(description = "把用户描述的业务流程转换成一个 MateClaw workflow 草稿并保存到当前 workspace。" + + "适用场景:用户说「把 X 这件事做成 workflow / 自动化 / 流程」、「每周一让 X 员工 ...」、" + + "「客户消息进来时让 X 应对」。工具会输出 workflowId + 简短摘要,前端会自动在 workflow 编辑器里打开。" + + "不会自动发布,不会自动启用 trigger — 用户需要在编辑器里 review 后再 publish。") + public String workflow_draft_generate( + @ToolParam(description = "用户对业务流程的自然语言描述,越具体越好;可以包含触发条件、参与员工、是否要审批、要发到哪个渠道。") + String description, + // ChatOrigin-scoped workspace lookup; never trust the LLM to pass workspaceId. + @Nullable ToolContext ctx) { + + Long workspaceId = ctx == null ? null : ChatOrigin.from(ctx).workspaceId(); + if (workspaceId == null || workspaceId <= 0) { + return "无法确定当前 workspace,工具放弃执行。请在 workspace 上下文里调用我。"; + } + + GeneratedWorkflowDraft draft; + try { + draft = generator.generate(description, workspaceId); + } catch (Exception e) { + log.warn("[workflow_draft_generate] generation failed for ws={}: {}", + workspaceId, e.getMessage()); + return "生成失败:" + e.getMessage(); + } + + // Persist as a draft. No publish, no triggers — that's a separate + // user action via the editor / approve flow. We name it from the + // generator output so the editor surfaces something useful in + // the list immediately. + WorkflowEntity wf = new WorkflowEntity(); + wf.setName(draft.name()); + wf.setDescription(draft.description()); + wf.setEnabled(true); + wf.setWorkspaceId(workspaceId); + WorkflowEntity created; + try { + created = workflowService.create(wf); + workflowService.saveDraft(created.getId(), workspaceId, draft.draftJson(), null); + } catch (Exception e) { + log.warn("[workflow_draft_generate] persist failed: {}", e.getMessage()); + return "草稿生成成功但保存失败:" + e.getMessage(); + } + + StringBuilder out = new StringBuilder(); + out.append("已生成 workflow 草稿 ").append(draft.name()) + .append("(id=").append(created.getId()).append(")。\n"); + if (draft.compileOk()) { + out.append("✓ 编译预校验通过。\n"); + } else { + out.append("⚠ 编译预校验未通过 (").append(draft.compileErrors().size()).append(" 处),需在编辑器里修正。\n"); + } + if (draft.missingFields() != null && !draft.missingFields().isEmpty()) { + out.append("缺失字段:"); + for (int i = 0; i < draft.missingFields().size(); i++) { + if (i > 0) out.append(";"); + out.append(draft.missingFields().get(i)); + } + out.append("\n"); + } + if (draft.warnings() != null && !draft.warnings().isEmpty()) { + out.append("警告:"); + for (int i = 0; i < draft.warnings().size(); i++) { + if (i > 0) out.append(";"); + out.append(draft.warnings().get(i)); + } + out.append("\n"); + } + if (draft.triggerDrafts() != null && !draft.triggerDrafts().isEmpty()) { + out.append("建议触发器:").append(draft.triggerDrafts().size()).append(" 个 (默认未启用,需在编辑器里确认后创建)。\n"); + } + out.append("请到 workflow 编辑器查看并继续完善。"); + return out.toString(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftGenerator.java b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftGenerator.java new file mode 100644 index 00000000..f307dfd4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftGenerator.java @@ -0,0 +1,381 @@ +package vip.mate.workflow.draftgen; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.stereotype.Service; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.repository.ChannelMapper; +import vip.mate.llm.chatmodel.ProviderChatModelFactory; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.workflow.compiler.PublishContext; +import vip.mate.workflow.compiler.WorkflowAclPort; +import vip.mate.workflow.compiler.WorkflowCompiler; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Natural-language → workflow draft generator. + * + *

Composes a system prompt + workspace-scoped context (available + * digital employees + channels) + the user description, dispatches to + * the workspace's default chat model, parses the JSON response, and + * runs {@link WorkflowCompiler} against it without persisting. The + * compile pass is "preview-only" — auto-publish is explicitly + * forbidden in the system prompt and we don't insert any rows here. + * + *

The generator is also the shared core called by the + * {@code workflow_draft_generate} agent tool, so a chat user can ask + * an agent "把每周一汇总销售这件事做成 workflow" and the agent gets back + * the same draft shape. + * + *

Failures are surfaced rather than swallowed: if the model returns + * non-JSON or the JSON doesn't carry a {@code steps} array, the + * generator throws so the controller / tool returns a clear error + * instead of a silently-broken draft. + */ +@Slf4j +@Service +public class WorkflowDraftGenerator { + + /** System prompt — the contract the LLM must honor. Embedded as a + * text block so the file is the canonical version (no resource + * loading, no separate prompt-management infra in v0). */ + static final String SYSTEM_PROMPT = """ + 你是 MateClaw 的工作流草稿生成器。你的任务是把用户用自然语言描述的业务流程,转换成 MateClaw RFC-29 v0 workflow JSON 草稿。 + + 你只输出 JSON,不输出 Markdown,不输出解释,不输出代码块。 + + # 输出形态 + + 必须输出一个 JSON object,结构如下: + + { + "schemaVersion": "1.0", + "name": "...", + "description": "...", + "metadata": { + "generatedFrom": "natural_language", + "confidence": 0.0, + "warnings": [], + "missingFields": [] + }, + "triggerDrafts": [], + "steps": [] + } + + # v0 支持的 7 种 mode + + sequential — 一个员工执行;必须 agentId/agentName + promptTemplate。outputContentType 只能 text 或 json。 + fan_out — 至少 2 个连续 fan_out,后接 collect;每个分支必须 agentId/agentName + promptTemplate。 + collect — 不带 agentId、agentName、promptTemplate;只能跟在 fan_out group 后。 + conditional — mode.expression 必填,使用 Pebble 子集语法。 + · 比较:== != < <= > >= + · 逻辑:必须使用单词 and / or / not,禁止 && / || / ! + · 示例(单条件):{{ outputs.x.approved == true }} + · 示例(多条件):{{ outputs.finance.flag == true or outputs.ops.flag == true or outputs.customer.flag == true }} + · 示例(取反):{{ not outputs.x.skip }} + agentId/agentName + promptTemplate 必填。 + await_approval — approvalKind + approverChannels[] + approvalMessage 必填;可选 timeoutSecs;不要 agentId / agentName / promptTemplate。 + dispatch_channel — channels[] + targets{} + content 必填;不要 agentId / agentName / promptTemplate。 + write_memory — employeeId + file + mergeStrategy(append/replace_section/upsert_kv/overwrite) + content 必填;不要 agentId / agentName / promptTemplate。 + + # 不支持 + + 不要生成 loop / invoke_skill / subflow。不要生成 agent_lifecycle / content_match 触发器。 + 遇到循环、重复直到成功、调用技能、复杂嵌套,用最接近的线性步骤,并在 metadata.warnings 写明需人工确认。 + + # 触发器(triggerDrafts) + + 只允许 patternType: cron / channel_message / workflow_completion / webhook。 + triggerDrafts 默认 enabled=false,绝不自动启用。 + + # 命名 + + workflow.name 与 step.name 用英文 kebab-case (collect-sales-data / ask-finance-approval)。description 用用户母语。 + + # 占位字段 + + 找不到匹配的真实 ID/渠道/员工时使用占位: + - agentName: "TODO_*_AGENT" + - employeeId: "TODO_EMPLOYEE_ID" + - channels[*]: "TODO_SELECT_CHANNEL" + - targets["TODO_SELECT_CHANNEL"]: "TODO_TARGET_ID" + - sourceWorkflowId: "TODO_WORKFLOW_ID" + 每个 TODO 都要在 metadata.missingFields 中解释。 + 绝不能编造不存在的 agentId / channelType / 群 ID。 + + # 默认值 + + approvalKind: manager / finance / manual / legal / oncall 之一。 + approverChannels: 默认 ["web"],除非用户明确说企业 IM 渠道。 + mergeStrategy: 默认 "append"。 + schemaVersion: 始终 "1.0"。 + + # 质量 + + 只使用 v0 字段;无注释;无 trailing comma;无 Markdown;不自动启用 trigger;不自动发布。 + """; + + private final ProviderChatModelFactory chatModelFactory; + private final ModelConfigService modelConfigService; + private final RetryTemplate retryTemplate; + private final AgentMapper agentMapper; + private final ChannelMapper channelMapper; + private final ObjectMapper objectMapper; + private final WorkflowCompiler compiler; + private final WorkflowAclPort aclPort; + private final WorkflowDraftTemplateLibrary templateLibrary; + + public WorkflowDraftGenerator(ProviderChatModelFactory chatModelFactory, + ModelConfigService modelConfigService, + RetryTemplate retryTemplate, + AgentMapper agentMapper, + ChannelMapper channelMapper, + ObjectMapper objectMapper, + WorkflowCompiler compiler, + WorkflowAclPort aclPort, + WorkflowDraftTemplateLibrary templateLibrary) { + this.chatModelFactory = chatModelFactory; + this.modelConfigService = modelConfigService; + this.retryTemplate = retryTemplate; + this.agentMapper = agentMapper; + this.channelMapper = channelMapper; + this.objectMapper = objectMapper; + this.compiler = compiler; + this.aclPort = aclPort; + this.templateLibrary = templateLibrary; + } + + public GeneratedWorkflowDraft generate(String description, long workspaceId) { + if (description == null || description.isBlank()) { + throw new IllegalArgumentException("description must not be empty"); + } + + // --- 1. workspace context --------------------------------------- + String contextPrompt = buildContextPrompt(workspaceId); + + // --- 2. resolve runtime model ---------------------------------- + ModelConfigEntity model = modelConfigService.getDefaultModel(); + if (model == null) { + throw new IllegalStateException( + "No default chat model configured; cannot generate workflow draft"); + } + ChatModel chatModel = chatModelFactory.buildFor(model, retryTemplate); + ChatClient client = ChatClient.create(chatModel); + + // --- 3. call the model ----------------------------------------- + String raw; + try { + raw = client.prompt() + .system(SYSTEM_PROMPT + "\n\n" + contextPrompt) + .user(description) + .call() + .content(); + } catch (Exception e) { + throw new IllegalStateException( + "Workflow draft generator chat call failed: " + e.getMessage(), e); + } + if (raw == null || raw.isBlank()) { + throw new IllegalStateException("Workflow draft generator returned empty content"); + } + + // --- 4. parse + validate shape --------------------------------- + JsonNode root = parseStrict(raw); + if (!root.has("steps") || !root.get("steps").isArray()) { + throw new IllegalStateException( + "Generated draft has no steps[] array; raw output: " + truncate(raw)); + } + + // --- 5. extract fields ----------------------------------------- + String name = root.path("name").asText(""); + String userDescription = root.path("description").asText(""); + Double confidence = root.path("metadata").path("confidence").isNumber() + ? root.path("metadata").path("confidence").asDouble() : null; + + List warnings = readStringArray(root, "metadata", "warnings"); + List missingFields = readStringArray(root, "metadata", "missingFields"); + + // The runtime only consumes the steps part of the draft — strip + // everything else into a clean {steps:[...]} shape. + Map draftRoot = new LinkedHashMap<>(); + draftRoot.put("steps", objectMapper.convertValue(root.get("steps"), + new TypeReference>>() {})); + String draftJson; + try { + draftJson = objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(draftRoot); + } catch (Exception e) { + throw new IllegalStateException("Failed to re-serialize generated steps: " + e.getMessage(), e); + } + + // --- 6. trigger drafts ----------------------------------------- + // patternType allowlist mirrors what TriggerService accepts at + // create time. The generator prompt forbids agent_lifecycle and + // content_match; we filter defensively here too because models + // occasionally hallucinate trigger types under low confidence, + // and we don't want a future UI / tool that calls /draft/generate + // and trusts the response to silently re-introduce dropped types. + java.util.Set allowedPatternTypes = java.util.Set.of( + "cron", "channel_message", "workflow_completion", "webhook"); + List> triggerDrafts = new ArrayList<>(); + if (root.has("triggerDrafts") && root.get("triggerDrafts").isArray()) { + List> candidates = objectMapper.convertValue(root.get("triggerDrafts"), + new TypeReference>>() {}); + int dropped = 0; + for (Map td : candidates) { + String pt = td.get("patternType") instanceof String s ? s : null; + if (pt == null || !allowedPatternTypes.contains(pt)) { + dropped++; + continue; + } + // Belt-and-suspenders: never trust the LLM to honor enabled=false. + td.put("enabled", false); + triggerDrafts.add(td); + } + if (dropped > 0) { + warnings = appendWarning(warnings, + "dropped " + dropped + " unsupported triggerDraft entr" + (dropped == 1 ? "y" : "ies") + + " (allowed: " + String.join(", ", allowedPatternTypes) + ")"); + } + } + + // --- 7. compile preview --------------------------------------- + boolean compileOk; + List compileErrors; + try { + // PublishContext is (workspaceId, publisherId). + WorkflowCompiler.Result result = compiler.compile(draftJson, + new PublishContext(workspaceId, 0L), aclPort); + compileOk = result.ok(); + compileErrors = compileOk ? List.of() : result.errors(); + } catch (Exception e) { + // Compile preview failures are not fatal — the operator can + // still edit the draft. We surface them as warnings. + log.warn("[WorkflowDraftGenerator] preview compile failed: {}", e.getMessage()); + compileOk = false; + compileErrors = List.of(); + warnings = appendWarning(warnings, "preview compile threw: " + e.getMessage()); + } + + return new GeneratedWorkflowDraft( + name == null || name.isBlank() ? "untitled-workflow" : name, + userDescription, + draftJson, + triggerDrafts, + warnings, + missingFields, + confidence, + compileOk, + compileErrors); + } + + /** Compose the workspace-scoped context prompt: agent + channel + * inventory the model can pick from. Agents are filtered to enabled + * rows; channels likewise. The model is told to prefer real ids + * over TODOs but never to fabricate. */ + private String buildContextPrompt(long workspaceId) { + List agents = agentMapper.selectList(new LambdaQueryWrapper() + .eq(AgentEntity::getWorkspaceId, workspaceId) + .eq(AgentEntity::getEnabled, true)); + List channels = channelMapper.selectList(new LambdaQueryWrapper() + .eq(ChannelEntity::getWorkspaceId, workspaceId) + .eq(ChannelEntity::getEnabled, true)); + + StringBuilder sb = new StringBuilder(); + sb.append("# 当前 workspace 可用数字员工\n["); + boolean first = true; + for (AgentEntity a : agents) { + if (!first) sb.append(","); + first = false; + sb.append("{\"agentId\":").append(a.getId()) + .append(",\"name\":\"").append(escape(a.getName())) + .append("\",\"description\":\"") + .append(escape(a.getDescription() == null ? "" : a.getDescription())) + .append("\"}"); + } + sb.append("]\n\n# 当前 workspace 可用渠道\n["); + first = true; + for (ChannelEntity c : channels) { + if (!first) sb.append(","); + first = false; + sb.append("{\"channelType\":\"").append(escape(c.getChannelType())) + .append("\",\"name\":\"").append(escape(c.getName())) + .append("\"}"); + } + sb.append("]\n\n优先使用这些真实 agentId 和 channelType。不存在的 ID 必须用 TODO_* 占位,不要编造。\n"); + + // Few-shot exemplars from the template library — the LLM stays + // closer to canonical shapes when it has 2-3 concrete examples + // in the system prompt. + sb.append("\n# 模板示例(参考,不必照抄)\n"); + for (WorkflowDraftTemplate t : templateLibrary.all()) { + sb.append("## ").append(t.id()).append(" — ").append(t.label()).append("\n"); + sb.append(t.description()).append("\n"); + sb.append("draft: ").append(t.draftJson()).append("\n"); + if (t.triggerDraftsJson() != null && !"[]".equals(t.triggerDraftsJson())) { + sb.append("triggerDrafts: ").append(t.triggerDraftsJson()).append("\n"); + } + } + return sb.toString(); + } + + private JsonNode parseStrict(String raw) { + // Some models still wrap the JSON in a ```json fence even when + // the prompt says "no Markdown". Strip the fences before parsing + // so we don't reject otherwise-valid output. + String cleaned = raw.trim(); + if (cleaned.startsWith("```")) { + int firstNl = cleaned.indexOf('\n'); + if (firstNl > 0) cleaned = cleaned.substring(firstNl + 1); + int closeFence = cleaned.lastIndexOf("```"); + if (closeFence > 0) cleaned = cleaned.substring(0, closeFence); + cleaned = cleaned.trim(); + } + try { + return objectMapper.readTree(cleaned); + } catch (Exception e) { + throw new IllegalStateException( + "Workflow draft generator returned non-JSON: " + e.getMessage() + + " — raw: " + truncate(raw), e); + } + } + + private List readStringArray(JsonNode root, String... path) { + JsonNode node = root; + for (String p : path) node = node.path(p); + if (!node.isArray()) return List.of(); + List out = new ArrayList<>(node.size()); + for (JsonNode item : node) { + if (item.isTextual()) out.add(item.asText()); + } + return out; + } + + private static List appendWarning(List existing, String msg) { + List next = new ArrayList<>(existing == null ? List.of() : existing); + next.add(msg); + return next; + } + + private static String escape(String s) { + if (s == null) return ""; + return s.replace("\\", "\\\\").replace("\"", "\\\"") + .replace("\n", " ").replace("\r", " "); + } + + private static String truncate(String s) { + if (s == null) return ""; + return s.length() <= 400 ? s : s.substring(0, 400) + "…"; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplate.java b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplate.java new file mode 100644 index 00000000..af232149 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplate.java @@ -0,0 +1,43 @@ +package vip.mate.workflow.draftgen; + +import java.util.List; + +/** + * One named exemplar in the workflow template library. + * + *

Templates serve two purposes: + *

    + *
  1. As few-shot examples inside the system prompt — the LLM sees + * "here are five canonical shapes; pick the closest and adapt the + * fields" instead of inventing structure from scratch. RFC v0 + * authors should never see anything more exotic than these + * shapes.
  2. + *
  3. As "apply template" entries the UI or the + * workflow_draft_generate tool can drop in directly when the + * user's description matches a canonical pattern (saves a + * generation roundtrip and stays cheaper / faster).
  4. + *
+ * + *

{@code matchHints} is a small bag of natural-language phrases the + * tool can use to short-circuit to a template before calling the LLM — + * if the user says "周一汇总" or "weekly summary" we already know which + * shape they mean. + */ +public record WorkflowDraftTemplate( + /** Stable kebab-case id; surfaces in the API response. */ + String id, + /** Short bilingual label; the UI's "apply template" picker shows this. */ + String label, + /** One-sentence description in user-facing prose. */ + String description, + /** Natural-language phrases that should bias toward this template. */ + List matchHints, + /** Workflow draft JSON; placeholders like TODO_AGENT_ID stay + * in the body until the UI / tool fills them. */ + String draftJson, + /** Trigger drafts attached to this template, if any. Stored as + * serialised JSON arrays so the prompt doesn't have to know + * about Java types. */ + String triggerDraftsJson +) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplateLibrary.java b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplateLibrary.java new file mode 100644 index 00000000..9aab8721 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/draftgen/WorkflowDraftTemplateLibrary.java @@ -0,0 +1,210 @@ +package vip.mate.workflow.draftgen; + +import org.springframework.stereotype.Component; + +import java.util.List; + +/** + * Small in-process library of canonical workflow shapes. Used as + * few-shot exemplars in the system prompt AND as "apply template" + * entries operators / agents can drop in directly. Kept as code + * constants rather than a DB table so the templates version with the + * runtime that interprets them — a template that references modes the + * runtime doesn't support yet should never ship. + * + *

Templates are intentionally minimal: 5-7 shapes that cover what + * the v0 reviewer flagged as the actual customer use cases (weekly + * summary, approval-and-notify, customer-message routing, chained + * workflow, daily memory write). New shapes only get added when the + * customer evidence is in. + */ +@Component +public class WorkflowDraftTemplateLibrary { + + private final List templates = List.of( + weeklySummary(), + approvalAndNotify(), + customerMessageRouting(), + chainedWorkflow(), + dailyMemoryWrite(), + parallelAnalysis(), + channelAlertOnFailure() + ); + + public List all() { + return templates; + } + + /** Look up a template by id; returns null when no match. */ + public WorkflowDraftTemplate byId(String id) { + if (id == null) return null; + return templates.stream() + .filter(t -> id.equals(t.id())) + .findFirst().orElse(null); + } + + // ===== template definitions ===== + + private static WorkflowDraftTemplate weeklySummary() { + return new WorkflowDraftTemplate( + "weekly-summary", + "周报汇总 / Weekly summary", + "每周固定时间让数字员工汇总数据,再发到群里。常见于销售周报、运营日报。", + List.of("每周", "周报", "周一", "weekly", "summary", "汇总"), + """ + {"steps":[ + {"name":"collect-data","agentName":"TODO_DATA_AGENT","mode":{"type":"sequential"}, + "promptTemplate":"汇总本周的{{ inputs.topic }}并输出 JSON","outputVar":"summary","outputContentType":"json"}, + {"name":"notify-group", + "mode":{"type":"dispatch_channel","channels":["TODO_SELECT_CHANNEL"], + "targets":{"TODO_SELECT_CHANNEL":"TODO_TARGET_ID"}, + "content":"本周汇总:{{ outputs.summary }}"}} + ]}""", + """ + [{"name":"weekly-summary-cron","patternType":"cron","enabled":false, + "patternJson":{"cron":"0 0 9 ? * MON","timezone":"Asia/Shanghai"}, + "targetType":"workflow", + "payloadTemplate":"{\\"topic\\":\\"销售\\"}"}]""" + ); + } + + private static WorkflowDraftTemplate approvalAndNotify() { + return new WorkflowDraftTemplate( + "approval-and-notify", + "审批后通知 / Approval then notify", + "数字员工出方案 → 老板审批 → 通过后发到群里。常见于费用申请、采购、合同。", + List.of("审批", "确认", "老板", "approval", "approve", "确认通过"), + """ + {"steps":[ + {"name":"draft-proposal","agentName":"TODO_DRAFTER","mode":{"type":"sequential"}, + "promptTemplate":"为 {{ inputs.topic }} 起草一个方案","outputVar":"proposal","outputContentType":"text"}, + {"name":"manager-approve", + "mode":{"type":"await_approval","approvalKind":"manager", + "approverChannels":["web"], + "approvalMessage":"请审批方案:{{ outputs.proposal }}", + "timeoutSecs":86400}}, + {"name":"notify-group", + "mode":{"type":"dispatch_channel","channels":["TODO_SELECT_CHANNEL"], + "targets":{"TODO_SELECT_CHANNEL":"TODO_TARGET_ID"}, + "content":"方案已通过:{{ outputs.proposal }}"}} + ]}""", + "[]" + ); + } + + private static WorkflowDraftTemplate customerMessageRouting() { + return new WorkflowDraftTemplate( + "customer-message-routing", + "客户消息路由 / Customer message routing", + "渠道里出现关键词时,让客服员工应对,并把结果记到员工记忆。", + List.of("客户", "客服", "关键词", "customer", "support", "回复"), + """ + {"steps":[ + {"name":"answer-customer","agentName":"TODO_SUPPORT_AGENT","mode":{"type":"sequential"}, + "promptTemplate":"客户说:{{ inputs.content }}。请用礼貌的语气回复。", + "outputVar":"reply","outputContentType":"text"}, + {"name":"send-reply", + "mode":{"type":"dispatch_channel","channels":["TODO_SELECT_CHANNEL"], + "targets":{"TODO_SELECT_CHANNEL":"{{ inputs.sender }}"}, + "content":"{{ outputs.reply }}"}}, + {"name":"remember-issue", + "mode":{"type":"write_memory","employeeId":"TODO_SUPPORT_AGENT", + "file":"customer-issues.md","mergeStrategy":"append", + "content":"### {{ inputs.sender }}\\n{{ inputs.content }}\\n回复:{{ outputs.reply }}\\n"}} + ]}""", + """ + [{"name":"customer-keyword","patternType":"channel_message","enabled":false, + "patternJson":{"channelType":"TODO_SELECT_CHANNEL","contentContains":"发票"}, + "targetType":"workflow", + "payloadTemplate":"{\\"content\\":\\"{{ event.content }}\\",\\"sender\\":\\"{{ event.senderId }}\\"}"}]""" + ); + } + + private static WorkflowDraftTemplate chainedWorkflow() { + return new WorkflowDraftTemplate( + "chained-workflow", + "上游完成后接力 / Chained on upstream completion", + "上游 workflow 跑完后自动接一段处理:常见于 ETL 接出报表、运营接审计。", + List.of("接力", "上游", "完成后", "chained", "after"), + """ + {"steps":[ + {"name":"post-process","agentName":"TODO_AGENT","mode":{"type":"sequential"}, + "promptTemplate":"上游 run {{ inputs.sourceWorkflowId }} 已完成(state={{ inputs.state }}),请处理后续。", + "outputVar":"summary","outputContentType":"text"} + ]}""", + """ + [{"name":"after-upstream","patternType":"workflow_completion","enabled":false, + "patternJson":{"sourceWorkflowId":"TODO_WORKFLOW_ID","stateFilter":"succeeded"}, + "targetType":"workflow", + "payloadTemplate":"{\\"sourceWorkflowId\\":\\"{{ event.sourceWorkflowId }}\\",\\"state\\":\\"{{ event.state }}\\"}"}]""" + ); + } + + private static WorkflowDraftTemplate dailyMemoryWrite() { + return new WorkflowDraftTemplate( + "daily-memory-write", + "每日记入员工记忆 / Daily memory append", + "每天定时让员工写一段记忆,作为后续对话的上下文。", + List.of("每天", "daily", "记忆", "写入", "memory"), + """ + {"steps":[ + {"name":"summarize-day","agentName":"TODO_AGENT","mode":{"type":"sequential"}, + "promptTemplate":"用一段话总结今天的{{ inputs.topic }}。", + "outputVar":"summary","outputContentType":"text"}, + {"name":"persist-memory", + "mode":{"type":"write_memory","employeeId":"TODO_EMPLOYEE_ID", + "file":"daily-log.md","mergeStrategy":"append", + "content":"### {{ inputs.date }}\\n{{ outputs.summary }}\\n"}} + ]}""", + """ + [{"name":"daily-memory-cron","patternType":"cron","enabled":false, + "patternJson":{"cron":"0 0 22 * * ?","timezone":"Asia/Shanghai"}, + "targetType":"workflow", + "payloadTemplate":"{\\"topic\\":\\"工作\\",\\"date\\":\\"{{ event.firedAt }}\\"}"}]""" + ); + } + + private static WorkflowDraftTemplate parallelAnalysis() { + return new WorkflowDraftTemplate( + "parallel-analysis", + "并行多角度分析 / Parallel multi-angle analysis", + "三个不同员工同时从不同角度分析同一份输入,最后由 collect 汇合。", + List.of("分别", "并行", "多角度", "parallel", "fan_out"), + """ + {"steps":[ + {"name":"angle-finance","agentName":"TODO_FINANCE_AGENT","mode":{"type":"fan_out"}, + "promptTemplate":"从财务角度分析:{{ inputs.topic }}", + "outputVar":"finance","outputContentType":"text"}, + {"name":"angle-operations","agentName":"TODO_OPS_AGENT","mode":{"type":"fan_out"}, + "promptTemplate":"从运营角度分析:{{ inputs.topic }}", + "outputVar":"ops","outputContentType":"text"}, + {"name":"angle-customer","agentName":"TODO_CUSTOMER_AGENT","mode":{"type":"fan_out"}, + "promptTemplate":"从客户角度分析:{{ inputs.topic }}", + "outputVar":"customer","outputContentType":"text"}, + {"name":"merge-views","mode":{"type":"collect"}} + ]}""", + "[]" + ); + } + + private static WorkflowDraftTemplate channelAlertOnFailure() { + return new WorkflowDraftTemplate( + "channel-alert-on-failure", + "上游失败时报警 / Alert on upstream failure", + "上游 workflow 跑失败时立即推送到值班渠道,常见于关键 ETL / 自动化作业的兜底。", + List.of("失败", "报警", "alert", "failure", "失败时"), + """ + {"steps":[ + {"name":"alert-oncall", + "mode":{"type":"dispatch_channel","channels":["TODO_SELECT_CHANNEL"], + "targets":{"TODO_SELECT_CHANNEL":"TODO_ONCALL_TARGET"}, + "content":"⚠ 上游 workflow run {{ inputs.runId }} 失败:{{ inputs.errorMessage }}"}} + ]}""", + """ + [{"name":"upstream-failure","patternType":"workflow_completion","enabled":false, + "patternJson":{"sourceWorkflowId":"TODO_WORKFLOW_ID","stateFilter":"failed"}, + "targetType":"workflow", + "payloadTemplate":"{\\"runId\\":\\"{{ event.runId }}\\",\\"errorMessage\\":\\"{{ event.errorMessage }}\\"}"}]""" + ); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowEntity.java new file mode 100644 index 00000000..07300ec8 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowEntity.java @@ -0,0 +1,64 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Stable workflow identity. The current draft is stored inline (1:1 with the + * workflow row) so PK uniqueness automatically guarantees a single draft; + * published snapshots live in {@code mate_workflow_revision}. + */ +@Data +@TableName("mate_workflow") +public class WorkflowEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long workspaceId; + + private String name; + + private String description; + + private Boolean enabled; + + /** Inline draft graph_json; null when there is no active draft. */ + @TableField(value = "draft_json", updateStrategy = FieldStrategy.ALWAYS) + private String draftJson; + + @TableField(value = "draft_schema_version", updateStrategy = FieldStrategy.ALWAYS) + private String draftSchemaVersion; + + @TableField(value = "draft_updated_by", updateStrategy = FieldStrategy.ALWAYS) + private Long draftUpdatedBy; + + @TableField(value = "draft_updated_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime draftUpdatedAt; + + /** Pointer to the most recently published revision; null if never published. */ + @TableField(value = "latest_revision_id", updateStrategy = FieldStrategy.ALWAYS) + private Long latestRevisionId; + + private Long createdBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + @TableField(fill = FieldFill.INSERT_UPDATE) + private LocalDateTime updateTime; + + // The `deleted` column stays on the table for schema compatibility but + // is no longer logical-deleted — see contributing.md, the project moved + // to hard-delete project-wide. deleteById() now performs a real DELETE, + // and the unique key on (workspace_id, name, deleted) no longer collides + // when a name is recreated and re-deleted. + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowPayloadEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowPayloadEntity.java new file mode 100644 index 00000000..22233c11 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowPayloadEntity.java @@ -0,0 +1,48 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Payload body addressed by a stable URI. Small payloads (< 256KB) live + * inline in {@code contentBytes}; larger payloads point at filesystem or + * object storage via {@code storageKind} + {@code storageRef}. {@code sha256} + * is for tamper detection only — v0 does not deduplicate across runs. + */ +@Data +@TableName("mate_workflow_payload") +public class WorkflowPayloadEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private String payloadUri; + + private Long workspaceId; + + @TableField(value = "content_bytes", updateStrategy = FieldStrategy.ALWAYS) + private byte[] contentBytes; + + /** Storage flavour: inline / fs / s3 / oss. */ + private String storageKind; + + @TableField(value = "storage_ref", updateStrategy = FieldStrategy.ALWAYS) + private String storageRef; + + @TableField(value = "content_type", updateStrategy = FieldStrategy.ALWAYS) + private String contentType; + + @TableField(value = "sha256", updateStrategy = FieldStrategy.ALWAYS) + private String sha256; + + @TableField(value = "size_bytes", updateStrategy = FieldStrategy.ALWAYS) + private Long sizeBytes; + + private LocalDateTime createdAt; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRevisionEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRevisionEntity.java new file mode 100644 index 00000000..733766b9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRevisionEntity.java @@ -0,0 +1,41 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Immutable published snapshot of a workflow. The {@code revision} column is + * monotonic per workflow; rows are append-only after publish. + */ +@Data +@TableName("mate_workflow_revision") +public class WorkflowRevisionEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long workflowId; + + private Integer revision; + + @TableField(value = "graph_json", updateStrategy = FieldStrategy.ALWAYS) + private String graphJson; + + private String schemaVersion; + + @TableField(value = "published_note", updateStrategy = FieldStrategy.ALWAYS) + private String publishedNote; + + @TableField(value = "published_by", updateStrategy = FieldStrategy.ALWAYS) + private Long publishedBy; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunEntity.java new file mode 100644 index 00000000..3b0964f7 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunEntity.java @@ -0,0 +1,60 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldFill; +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Workflow run instance. Run is locked to a specific revision for stability + * even when later revisions are published. Initial input and final output are + * stored as payload URIs to avoid bloating the run row. + */ +@Data +@TableName("mate_workflow_run") +public class WorkflowRunEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long workflowId; + + private Long revisionId; + + private Long workspaceId; + + /** State machine value: pending / running / paused / succeeded / failed / cancelled / timed_out. */ + private String state; + + @TableField(value = "triggered_by", updateStrategy = FieldStrategy.ALWAYS) + private String triggeredBy; + + @TableField(value = "triggered_meta", updateStrategy = FieldStrategy.ALWAYS) + private String triggeredMeta; + + @TableField(value = "initial_input_ref", updateStrategy = FieldStrategy.ALWAYS) + private String initialInputRef; + + @TableField(value = "final_output_ref", updateStrategy = FieldStrategy.ALWAYS) + private String finalOutputRef; + + @TableField(value = "error_message", updateStrategy = FieldStrategy.ALWAYS) + private String errorMessage; + + @TableField(value = "started_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime startedAt; + + @TableField(value = "completed_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime completedAt; + + @TableField(fill = FieldFill.INSERT) + private LocalDateTime createTime; + + // Hard-delete only (project convention); column kept for schema compat. + private Integer deleted; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunPauseEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunPauseEntity.java new file mode 100644 index 00000000..0f5fe913 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunPauseEntity.java @@ -0,0 +1,51 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Durable workflow pause row. Holds the resume token and links back to the + * external approval row (or other callback source) so that resume can be + * triggered idempotently after a JVM restart. + */ +@Data +@TableName("mate_workflow_run_pause") +public class WorkflowRunPauseEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long runId; + + private Long stepId; + + /** Source of the pause: await_approval, external_callback, etc. */ + private String pauseKind; + + /** Random server-generated token used as the resume entry key. */ + private String pauseToken; + + @TableField(value = "external_approval_id", updateStrategy = FieldStrategy.ALWAYS) + private Long externalApprovalId; + + private LocalDateTime pausedAt; + + @TableField(value = "resume_deadline", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime resumeDeadline; + + @TableField(value = "resume_payload_ref", updateStrategy = FieldStrategy.ALWAYS) + private String resumePayloadRef; + + @TableField(value = "resumed_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime resumedAt; + + /** Outcome on resume: approved / rejected / timeout / cancelled. */ + @TableField(value = "resume_outcome", updateStrategy = FieldStrategy.ALWAYS) + private String resumeOutcome; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunStepEntity.java b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunStepEntity.java new file mode 100644 index 00000000..e4d0d706 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/model/WorkflowRunStepEntity.java @@ -0,0 +1,68 @@ +package vip.mate.workflow.model; + +import com.baomidou.mybatisplus.annotation.FieldStrategy; +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableField; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; + +import java.time.LocalDateTime; + +/** + * Per-step execution row. {@code stepIndex} is the zero-based index in the + * revision's steps array; {@code iterationIndex} is reserved for fan_out + * iterations (and future loop bodies). + */ +@Data +@TableName("mate_workflow_run_step") +public class WorkflowRunStepEntity { + + @TableId(type = IdType.ASSIGN_ID) + private Long id; + + private Long runId; + + private Integer stepIndex; + + @TableField(value = "iteration_index", updateStrategy = FieldStrategy.ALWAYS) + private Integer iterationIndex; + + @TableField(value = "step_name", updateStrategy = FieldStrategy.ALWAYS) + private String stepName; + + @TableField(value = "agent_id", updateStrategy = FieldStrategy.ALWAYS) + private Long agentId; + + private String state; + + @TableField(value = "input_ref", updateStrategy = FieldStrategy.ALWAYS) + private String inputRef; + + @TableField(value = "output_ref", updateStrategy = FieldStrategy.ALWAYS) + private String outputRef; + + @TableField(value = "output_summary", updateStrategy = FieldStrategy.ALWAYS) + private String outputSummary; + + @TableField(value = "output_content_type", updateStrategy = FieldStrategy.ALWAYS) + private String outputContentType; + + @TableField(value = "error_message", updateStrategy = FieldStrategy.ALWAYS) + private String errorMessage; + + @TableField(value = "duration_ms", updateStrategy = FieldStrategy.ALWAYS) + private Long durationMs; + + @TableField(value = "token_input", updateStrategy = FieldStrategy.ALWAYS) + private Integer tokenInput; + + @TableField(value = "token_output", updateStrategy = FieldStrategy.ALWAYS) + private Integer tokenOutput; + + @TableField(value = "started_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime startedAt; + + @TableField(value = "completed_at", updateStrategy = FieldStrategy.ALWAYS) + private LocalDateTime completedAt; +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowMapper.java new file mode 100644 index 00000000..2d90ddc9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowMapper.java @@ -0,0 +1,23 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; +import org.apache.ibatis.annotations.Select; +import vip.mate.workflow.model.WorkflowEntity; + +@Mapper +public interface WorkflowMapper extends BaseMapper { + + /** + * Row-locking lookup used by the publish path. Two concurrent publishes + * for the same workflow would otherwise both compute the same + * {@code max(revision)+1} and the second would crash on the + * {@code uk_workflow_revision} unique constraint, leaving + * {@code latest_revision_id} pointing at the first while the second + * caller saw a 500. Locking the workflow row in a single transaction + * serializes the two publishes cleanly. + */ + @Select("SELECT * FROM mate_workflow WHERE id = #{id} AND deleted = 0 FOR UPDATE") + WorkflowEntity selectByIdForUpdate(@Param("id") long id); +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowPayloadMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowPayloadMapper.java new file mode 100644 index 00000000..8b50cd2a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowPayloadMapper.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.workflow.model.WorkflowPayloadEntity; + +@Mapper +public interface WorkflowPayloadMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRevisionMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRevisionMapper.java new file mode 100644 index 00000000..857009f0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRevisionMapper.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.workflow.model.WorkflowRevisionEntity; + +@Mapper +public interface WorkflowRevisionMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunMapper.java new file mode 100644 index 00000000..2fd57eb1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunMapper.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.workflow.model.WorkflowRunEntity; + +@Mapper +public interface WorkflowRunMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunPauseMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunPauseMapper.java new file mode 100644 index 00000000..5a9c2094 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunPauseMapper.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.workflow.model.WorkflowRunPauseEntity; + +@Mapper +public interface WorkflowRunPauseMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunStepMapper.java b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunStepMapper.java new file mode 100644 index 00000000..f7f78e03 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/repository/WorkflowRunStepMapper.java @@ -0,0 +1,9 @@ +package vip.mate.workflow.repository; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; +import org.apache.ibatis.annotations.Mapper; +import vip.mate.workflow.model.WorkflowRunStepEntity; + +@Mapper +public interface WorkflowRunStepMapper extends BaseMapper { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentInvoker.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentInvoker.java new file mode 100644 index 00000000..68703bcc --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentInvoker.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.runtime; + +/** + * SPI for "render prompt → run agent → return text response". Kept thin so + * unit tests can stub agent execution without booting the full StateGraph + * runtime. Production binding lives in {@link DefaultAgentInvoker} and + * delegates to {@code AgentService.chat(...)}. + */ +public interface AgentInvoker { + + /** + * Invoke the resolved agent with {@code prompt} and return the agent's + * final response text. {@code conversationId} is the ephemeral conversation + * id created per workflow step — the runner generates this so each step + * has its own conversational scope. + */ + String invoke(long agentId, String prompt, String conversationId); + + /** + * Resolve a workspace-scoped agent name to its id. Returns {@code null} + * when the agent does not exist or is disabled. + */ + Long resolveAgentId(long workspaceId, String agentName); +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentStepExecutor.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentStepExecutor.java new file mode 100644 index 00000000..1fe5d373 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/AgentStepExecutor.java @@ -0,0 +1,126 @@ +package vip.mate.workflow.runtime; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.PebbleSubsetEvaluator; +import vip.mate.workflow.compiler.ir.WorkflowStep; + +import java.util.UUID; + +/** + * Shared "render prompt → invoke agent → parse output" pipeline reused by + * the sequential / fan_out / conditional adapters. Centralising this here + * keeps each adapter file focused on its mode-specific dispatch logic + * (skip-on-condition, merge semantics) instead of repeating prompt rendering + * and content-type parsing. + */ +@Component +public class AgentStepExecutor { + + private static final String TEXT = "text"; + private static final String JSON = "json"; + + private final AgentInvoker agentInvoker; + private final PebbleSubsetEvaluator pebble; + private final PayloadStore payloadStore; + private final ObjectMapper objectMapper; + + public AgentStepExecutor(AgentInvoker agentInvoker, + PebbleSubsetEvaluator pebble, + PayloadStore payloadStore, + ObjectMapper objectMapper) { + this.agentInvoker = agentInvoker; + this.pebble = pebble; + this.payloadStore = payloadStore; + this.objectMapper = objectMapper; + } + + /** + * Resolve the agent, render the prompt with the current run context, + * invoke the agent, parse the response according to {@code outputContentType}, + * and write the payload through the store. Returns a succeeded result on + * the happy path and a failed result when any step in the chain throws. + */ + public StepResult run(WorkflowStep step, WorkflowRunContext context) { + Long agentId = resolveAgentId(step, context.workspaceId()); + if (agentId == null) { + return StepResult.failed("agent not resolvable for step '" + step.name() + + "': agentName=" + step.agentName() + " agentId=" + step.agentId()); + } + + String prompt; + try { + prompt = renderPrompt(step, context); + } catch (Exception e) { + return StepResult.failed("prompt render failed for step '" + step.name() + + "': " + e.getMessage()); + } + + String response; + String conversationId = "wf-run-" + context.runId() + "-step-" + step.name() + + "-" + UUID.randomUUID(); + try { + response = agentInvoker.invoke(agentId, prompt, conversationId); + if (response == null) response = ""; + } catch (Exception e) { + return StepResult.failed("agent invocation failed for step '" + step.name() + + "': " + e.getMessage()); + } + + String contentType = step.effectiveOutputContentType(); + try { + Object parsedValue = parseResponse(response, contentType); + String payloadUri = (TEXT.equals(contentType)) + ? payloadStore.storeString(context.workspaceId(), response, "text/plain") + : payloadStore.storeString(context.workspaceId(), response, "application/json"); + String summary = summarise(response); + return StepResult.succeeded(payloadUri, contentType, parsedValue, summary); + } catch (Exception e) { + return StepResult.failed("output parse failed for step '" + step.name() + + "' (contentType=" + contentType + "): " + e.getMessage()); + } + } + + private Long resolveAgentId(WorkflowStep step, long workspaceId) { + if (step.agentId() != null) return step.agentId(); + if (step.agentName() != null && !step.agentName().isBlank()) { + return agentInvoker.resolveAgentId(workspaceId, step.agentName()); + } + return null; + } + + private String renderPrompt(WorkflowStep step, WorkflowRunContext context) { + if (step.promptTemplate() == null || step.promptTemplate().isBlank()) { + return ""; + } + var compiled = pebble.parseTemplate(step.promptTemplate()); + return pebble.evaluateAsString(compiled, context.templateContext()); + } + + private Object parseResponse(String response, String contentType) throws Exception { + if (JSON.equals(contentType)) { + // Permissive: agents often wrap JSON in ```json fences. + String cleaned = stripCodeFence(response); + return objectMapper.readValue(cleaned, Object.class); + } + return response; + } + + private static String stripCodeFence(String s) { + String trimmed = s.trim(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + int lastFence = trimmed.lastIndexOf("```"); + if (firstNewline > 0 && lastFence > firstNewline) { + return trimmed.substring(firstNewline + 1, lastFence).trim(); + } + } + return trimmed; + } + + private static String summarise(String response) { + if (response == null || response.isBlank()) return ""; + String oneLine = response.replaceAll("\\s+", " ").trim(); + return oneLine.length() <= 256 ? oneLine : oneLine.substring(0, 253) + "..."; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ApprovalResumeBridge.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ApprovalResumeBridge.java new file mode 100644 index 00000000..49ae4aff --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ApprovalResumeBridge.java @@ -0,0 +1,137 @@ +package vip.mate.workflow.runtime; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import vip.mate.approval.event.WorkflowApprovalResolvedEvent; +import vip.mate.workflow.compiler.PublishContext; +import vip.mate.workflow.compiler.WorkflowAclPort; +import vip.mate.workflow.compiler.WorkflowCompiler; +import vip.mate.workflow.model.WorkflowRevisionEntity; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.model.WorkflowRunPauseEntity; +import vip.mate.workflow.repository.WorkflowRevisionMapper; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.repository.WorkflowRunPauseMapper; + +/** + * Bridges {@link WorkflowApprovalResolvedEvent} from the approval module + * into {@link WorkflowResumer}. Without this listener an operator who + * clicks "approve" in the approval inbox would only flip the + * {@code mate_tool_approval} row terminal — the workflow run stays + * paused forever until someone separately POSTs the pause token to the + * resume endpoint. + * + *

The listener: + *

    + *
  1. Looks up the pause row by {@code external_approval_id} matching + * the resolved approval row's id. If no pause row references this + * approval (operator path already resumed, or the approval wasn't + * linked to a workflow), we silently no-op.
  2. + *
  3. Re-loads the workflow revision's graph and recompiles it under + * the run's workspace ACL — same code path the resume controller + * uses, so an ACL change after publish doesn't sneak past.
  4. + *
  5. Maps the approval decision to a {@link WorkflowResumer.ResumeOutcome}: + * {@code approved}/{@code consumed} → {@code APPROVED}; + * {@code denied}/{@code superseded} → {@code REJECTED}; + * {@code timeout} → {@code TIMEOUT}.
  6. + *
  7. Calls {@code WorkflowResumer.resume} with the pause token. The + * resumer's idempotency check handles the race where the operator + * resumed the run via the REST endpoint a fraction of a second + * before the approval row resolved — second resume returns + * ALREADY_RESOLVED and the listener swallows it.
  8. + *
+ * + *

Lives in the workflow runtime module so the approval module stays + * free of workflow / runner dependencies, mirroring the workflow ↔ + * trigger event-bridge pattern. + */ +@Slf4j +@Component +@RequiredArgsConstructor +public class ApprovalResumeBridge { + + private final WorkflowRunPauseMapper pauseMapper; + private final WorkflowRunMapper runMapper; + private final WorkflowRevisionMapper revisionMapper; + private final WorkflowCompiler compiler; + private final WorkflowAclPort aclPort; + private final WorkflowResumer resumer; + + @EventListener + public void onApprovalResolved(WorkflowApprovalResolvedEvent event) { + if (event == null || event.approvalRowId() <= 0) return; + WorkflowRunPauseEntity pause = pauseMapper.selectOne( + new LambdaQueryWrapper() + .eq(WorkflowRunPauseEntity::getExternalApprovalId, event.approvalRowId()) + .isNull(WorkflowRunPauseEntity::getResumedAt) + .last("LIMIT 1")); + if (pause == null) { + // Either there's no workflow pause linked to this approval + // (chat-driven approval), or the operator path already resumed + // it. Both are fine. + log.debug("[ApprovalResumeBridge] no open pause for approval row {} (pendingId={})", + event.approvalRowId(), event.pendingId()); + return; + } + + WorkflowResumer.ResumeOutcome outcome = mapDecision(event.decision()); + if (outcome == null) { + log.info("[ApprovalResumeBridge] decision '{}' on approval row {} is not a workflow-resume " + + "trigger; pause {} stays open", + event.decision(), event.approvalRowId(), pause.getId()); + return; + } + + // Re-load the revision graph through the same compiler the resume + // controller uses, so ACL changes after publish don't sneak past. + WorkflowRunEntity run = runMapper.selectById(pause.getRunId()); + if (run == null) { + log.warn("[ApprovalResumeBridge] pause {} references missing run {}", + pause.getId(), pause.getRunId()); + return; + } + WorkflowRevisionEntity revision = revisionMapper.selectById(run.getRevisionId()); + if (revision == null) { + log.warn("[ApprovalResumeBridge] run {} references missing revision {}", + run.getId(), run.getRevisionId()); + return; + } + // PublishContext is (workspaceId, publisherId) — mind the order. + WorkflowCompiler.Result compiled = compiler.compile(revision.getGraphJson(), + new PublishContext(run.getWorkspaceId(), 0L), aclPort); + if (!compiled.ok()) { + log.warn("[ApprovalResumeBridge] revision {} failed to recompile on approval-driven resume", + revision.getId()); + return; + } + + try { + WorkflowResumer.Outcome result = resumer.resume( + compiled.graph(), pause.getPauseToken(), outcome, /* resumePayloadBody */ null); + log.info("[ApprovalResumeBridge] resumed run {} via approval row {}: kind={}", + run.getId(), event.approvalRowId(), result.kind()); + } catch (Exception e) { + // Idempotency is the resumer's job — this catch only triggers + // on actual runtime failures during resume. Don't rethrow: + // the approval row already moved off PENDING and we don't + // want a transient resume failure to look like an + // approval-side bug to upstream observers. + log.warn("[ApprovalResumeBridge] resume failed for run {}: {}", + run.getId(), e.getMessage()); + } + } + + private static WorkflowResumer.ResumeOutcome mapDecision(String decision) { + if (decision == null) return null; + return switch (decision.toLowerCase()) { + case "approved", "consumed" -> WorkflowResumer.ResumeOutcome.APPROVED; + case "denied", "superseded" -> WorkflowResumer.ResumeOutcome.REJECTED; + case "timeout" -> WorkflowResumer.ResumeOutcome.TIMEOUT; + // pending / running / unknown — no terminal outcome to map to. + default -> null; + }; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ChannelDispatcher.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ChannelDispatcher.java new file mode 100644 index 00000000..914af03e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/ChannelDispatcher.java @@ -0,0 +1,29 @@ +package vip.mate.workflow.runtime; + +/** + * SPI for "deliver this rendered content to a target on this channel". Kept + * thin so unit tests can stub channel side effects without booting the full + * channel adapter graph; production binding lives in + * {@link DefaultChannelDispatcher} and delegates to {@code ChannelManager}. + */ +public interface ChannelDispatcher { + + /** + * Send {@code content} to {@code targetId} on the channel identified by + * {@code channelType} (e.g. {@code "feishu"}, {@code "dingtalk"}). Returns + * an {@link DispatchResult} so the step adapter can build a per-channel + * report; throwing is reserved for programmer errors. + */ + DispatchResult dispatch(long workspaceId, String channelType, String targetId, String content); + + /** + * Per-channel dispatch outcome. {@code success=false} entries are turned + * into a step failure by the calling adapter; the message field surfaces + * to the run-step row's error column. + */ + record DispatchResult(boolean success, String message) { + public static DispatchResult ok() { return new DispatchResult(true, null); } + public static DispatchResult ok(String message) { return new DispatchResult(true, message); } + public static DispatchResult fail(String message) { return new DispatchResult(false, message); } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultAgentInvoker.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultAgentInvoker.java new file mode 100644 index 00000000..8de087a9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultAgentInvoker.java @@ -0,0 +1,48 @@ +package vip.mate.workflow.runtime; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.springframework.stereotype.Component; +import vip.mate.agent.AgentService; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; + +/** + * Production binding for {@link AgentInvoker}. Looks agents up by name within + * the workspace via {@link AgentMapper} and delegates execution to + * {@link AgentService#chat(Long, String, String)}. The conversation id is + * passed through as-is — the runner is responsible for generating an ephemeral + * id per step so multi-step runs do not collide on conversation history. + */ +@Component +public class DefaultAgentInvoker implements AgentInvoker { + + private final AgentService agentService; + private final AgentMapper agentMapper; + + public DefaultAgentInvoker(AgentService agentService, AgentMapper agentMapper) { + this.agentService = agentService; + this.agentMapper = agentMapper; + } + + @Override + public String invoke(long agentId, String prompt, String conversationId) { + return agentService.chat(agentId, prompt, conversationId); + } + + @Override + public Long resolveAgentId(long workspaceId, String agentName) { + if (agentName == null || agentName.isBlank()) return null; + // Workspace-scoped only — no fallback to a global lookup. The + // earlier "fall back to workspace-agnostic" branch let an old + // revision (or any code path that bypassed publish-time ACL) + // pull a same-named agent from a different workspace at runtime, + // which is exactly what tenant isolation forbids. The + // publish-time ACL layer is also workspace-scoped, so a draft + // referencing a foreign agent is rejected before it ever runs. + AgentEntity entity = agentMapper.selectOne(new LambdaQueryWrapper() + .eq(AgentEntity::getWorkspaceId, workspaceId) + .eq(AgentEntity::getName, agentName.trim()) + .eq(AgentEntity::getEnabled, true)); + return entity == null ? null : entity.getId(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultChannelDispatcher.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultChannelDispatcher.java new file mode 100644 index 00000000..bccbbaeb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultChannelDispatcher.java @@ -0,0 +1,53 @@ +package vip.mate.workflow.runtime; + +import org.springframework.stereotype.Component; +import vip.mate.channel.ChannelAdapter; +import vip.mate.channel.ChannelManager; + +import java.util.Optional; + +/** + * Production binding for {@link ChannelDispatcher}. Looks the channel up by + * type via {@link ChannelManager#getAdapterByType} and either calls + * {@code proactiveSend} when the adapter supports it or {@code sendMessage} + * otherwise. A missing adapter or one that's not running is reported back + * as a failed dispatch — the step adapter decides whether that fails the + * step or merely records a partial result. + */ +@Component +public class DefaultChannelDispatcher implements ChannelDispatcher { + + private final ChannelManager channelManager; + + public DefaultChannelDispatcher(ChannelManager channelManager) { + this.channelManager = channelManager; + } + + @Override + public DispatchResult dispatch(long workspaceId, String channelType, String targetId, String content) { + if (channelType == null || channelType.isBlank()) { + return DispatchResult.fail("channelType is required"); + } + Optional adapterOpt = channelManager.getAdapterByType(channelType); + if (adapterOpt.isEmpty()) { + return DispatchResult.fail("no active adapter for channel type '" + channelType + "'"); + } + ChannelAdapter adapter = adapterOpt.get(); + if (!adapter.isRunning()) { + return DispatchResult.fail("channel '" + channelType + "' adapter is not running"); + } + if (targetId == null || targetId.isBlank()) { + return DispatchResult.fail("missing targetId for channel '" + channelType + "'"); + } + try { + if (adapter.supportsProactiveSend()) { + adapter.proactiveSend(targetId, content); + } else { + adapter.sendMessage(targetId, content); + } + return DispatchResult.ok(); + } catch (Exception e) { + return DispatchResult.fail("dispatch to '" + channelType + "' failed: " + e.getMessage()); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultMemoryWriter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultMemoryWriter.java new file mode 100644 index 00000000..18c7def1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/DefaultMemoryWriter.java @@ -0,0 +1,57 @@ +package vip.mate.workflow.runtime; + +import org.springframework.stereotype.Component; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +/** + * Production binding for {@link MemoryWriter}. Resolves {@code employeeId} + * (a string in the wire format) to the agent id keying + * {@code mate_workspace_file}, applies the chosen merge strategy via + * {@link MergeStrategies}, then persists the result through + * {@link WorkspaceFileService#saveFile}. + * + *

v0 treats {@code employeeId} as the numeric agent id rendered as a + * string. Looking the agent up by name was considered but pushes name + * uniqueness into the runtime — the schema validator already accepts only + * a string so the wire format does not change. When we add a "human + * employee" surface this binding will grow a separate code path. + */ +@Component +public class DefaultMemoryWriter implements MemoryWriter { + + private final WorkspaceFileService fileService; + + public DefaultMemoryWriter(WorkspaceFileService fileService) { + this.fileService = fileService; + } + + @Override + public Result write(long workspaceId, String employeeId, String file, + String mergeStrategy, String content) { + Long agentId; + try { + agentId = Long.parseLong(employeeId); + } catch (NumberFormatException e) { + return Result.fail("employeeId '" + employeeId + + "' is not a valid agent id (numeric string expected)"); + } + WorkspaceFileEntity existing = fileService.getFile(agentId, file); + String existingBody = existing == null ? "" : (existing.getContent() == null ? "" : existing.getContent()); + + String merged; + try { + merged = MergeStrategies.apply(existingBody, content, mergeStrategy); + } catch (IllegalArgumentException e) { + return Result.fail(e.getMessage()); + } + + try { + fileService.saveFile(agentId, file, merged); + } catch (Exception e) { + return Result.fail("failed to persist memory file '" + file + "': " + e.getMessage()); + } + return Result.ok(mergeStrategy + " merged " + content.length() + " chars into " + + file + " (agent " + agentId + ")"); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MemoryWriter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MemoryWriter.java new file mode 100644 index 00000000..b1266ee9 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MemoryWriter.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.runtime; + +/** + * SPI for the {@code write_memory} step. Hides the workspace-file storage + * implementation behind a small surface so tests can stub the file side + * effect without booting WorkspaceFileService. Production binding lives in + * {@link DefaultMemoryWriter}. + */ +public interface MemoryWriter { + + /** + * Apply {@code mergeStrategy} to {@code content} against the existing + * file body for {@code (workspaceId, employeeId, file)} and persist the + * result. Returns a {@link Result} carrying a short summary so the step + * row's {@code output_summary} captures what changed. + */ + Result write(long workspaceId, String employeeId, String file, + String mergeStrategy, String content); + + record Result(boolean success, String summary, String errorMessage) { + public static Result ok(String summary) { return new Result(true, summary, null); } + public static Result fail(String error) { return new Result(false, null, error); } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MergeStrategies.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MergeStrategies.java new file mode 100644 index 00000000..cc8d136b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/MergeStrategies.java @@ -0,0 +1,139 @@ +package vip.mate.workflow.runtime; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Pure helpers implementing the four v0 merge strategies the {@code write_memory} + * step supports. Stateless so the same logic backs the production + * {@link MemoryWriter} binding and any test fake. + * + *

    + *
  • {@code append} — incoming content is concatenated to the existing + * body with a separating blank line. The simplest no-magic merge.
  • + *
  • {@code replace_section} — the incoming body's first non-blank line is + * expected to be a Markdown {@code ## } heading; if a section with that + * heading already exists in the file, it is replaced (heading inclusive + * through the line before the next {@code ## } heading or EOF); + * otherwise the incoming body is appended with a blank-line separator.
  • + *
  • {@code upsert_kv} — every line of the incoming body that matches + * {@code key: value} is treated as a key/value pair. Existing matching + * keys are updated in place; new keys are appended. Non-kv lines in the + * incoming body are dropped (they would otherwise re-introduce + * freeform text on every run).
  • + *
  • {@code overwrite} — replace the file with the incoming body + * verbatim. The escape hatch when no other strategy fits.
  • + *
+ */ +public final class MergeStrategies { + + /** Heading line for {@code replace_section}. */ + private static final Pattern SECTION_HEADING = Pattern.compile( + "^##\\s+.+$", Pattern.MULTILINE); + + /** {@code key: value} line for {@code upsert_kv} parsing. */ + private static final Pattern KV_LINE = Pattern.compile( + "^([A-Za-z0-9_.\\-]+)\\s*:\\s*(.*)$"); + + private MergeStrategies() {} + + public static String apply(String existing, String incoming, String strategy) { + String existingSafe = existing == null ? "" : existing; + String incomingSafe = incoming == null ? "" : incoming; + return switch (strategy) { + case "append" -> append(existingSafe, incomingSafe); + case "replace_section" -> replaceSection(existingSafe, incomingSafe); + case "upsert_kv" -> upsertKv(existingSafe, incomingSafe); + case "overwrite" -> incomingSafe; + default -> throw new IllegalArgumentException( + "unknown merge strategy '" + strategy + + "' — must be append / replace_section / upsert_kv / overwrite"); + }; + } + + private static String append(String existing, String incoming) { + if (existing.isEmpty()) return incoming; + if (incoming.isEmpty()) return existing; + String trimmed = existing.endsWith("\n") ? existing : existing + "\n"; + return trimmed + "\n" + incoming; + } + + private static String replaceSection(String existing, String incoming) { + String heading = firstHeading(incoming); + if (heading == null) { + // No heading on the incoming side — fall back to append so the + // step never silently drops content. + return append(existing, incoming); + } + int existingStart = indexOfHeading(existing, heading); + if (existingStart < 0) { + return append(existing, incoming); + } + int existingEnd = indexOfNextHeading(existing, existingStart + heading.length()); + if (existingEnd < 0) existingEnd = existing.length(); + StringBuilder out = new StringBuilder(); + out.append(existing, 0, existingStart); + out.append(incoming); + if (!incoming.endsWith("\n")) out.append('\n'); + if (existingEnd < existing.length()) { + out.append(existing, existingEnd, existing.length()); + } + return out.toString(); + } + + private static String firstHeading(String body) { + Matcher m = SECTION_HEADING.matcher(body); + return m.find() ? m.group().stripTrailing() : null; + } + + private static int indexOfHeading(String body, String heading) { + // Match the heading at start of line (after any line break or at + // position 0) so we don't false-match an inline "## " inside a code + // block by accident. + Pattern p = Pattern.compile("(?m)^" + Pattern.quote(heading) + "\\s*$"); + Matcher m = p.matcher(body); + return m.find() ? m.start() : -1; + } + + private static int indexOfNextHeading(String body, int from) { + Matcher m = SECTION_HEADING.matcher(body); + if (m.find(from)) return m.start(); + return -1; + } + + private static String upsertKv(String existing, String incoming) { + Map updates = new LinkedHashMap<>(); + for (String line : incoming.split("\\R", -1)) { + Matcher m = KV_LINE.matcher(line.trim()); + if (m.matches()) { + updates.put(m.group(1), m.group(2)); + } + } + if (updates.isEmpty()) return existing; + + StringBuilder out = new StringBuilder(); + for (String line : existing.split("\\R", -1)) { + Matcher m = KV_LINE.matcher(line.trim()); + if (m.matches() && updates.containsKey(m.group(1))) { + out.append(m.group(1)).append(": ").append(updates.remove(m.group(1))); + } else { + out.append(line); + } + out.append('\n'); + } + // Trim trailing empty line we always added so a clean file stays clean. + if (out.length() > 0 && out.charAt(out.length() - 1) == '\n') { + out.setLength(out.length() - 1); + } + // Append any incoming keys that did not exist in the original file. + for (var e : updates.entrySet()) { + if (out.length() > 0 && out.charAt(out.length() - 1) != '\n') { + out.append('\n'); + } + out.append(e.getKey()).append(": ").append(e.getValue()); + } + return out.toString(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/PayloadStore.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/PayloadStore.java new file mode 100644 index 00000000..e657b543 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/PayloadStore.java @@ -0,0 +1,252 @@ +package vip.mate.workflow.runtime; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; +import vip.mate.workflow.model.WorkflowPayloadEntity; +import vip.mate.workflow.repository.WorkflowPayloadMapper; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.util.HexFormat; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +/** + * Write-through facade over {@code mate_workflow_payload}. Three-tier storage: + * + *
    + *
  • inline (≤ {@code inlineMaxBytes}, default 256KB) — bytes go into + * {@code content_bytes}. Cheapest and lets one DB query reconstruct the + * payload.
  • + *
  • fs (≤ {@code hardCapBytes}) — bytes go to a workspace-scoped + * file under {@code mateclaw.workflow.payload.fs.root}; the row stores + * only the relative path in {@code storage_ref}. Default for any + * deployment that hasn't enabled a configured object-storage provider.
  • + *
  • Anything above the hard cap is rejected at write time so a runaway + * fan-out can't fill the disk silently.
  • + *
+ * + *

{@code s3} / {@code oss} columns exist in the schema but the v0 ship + * only writes {@code inline} or {@code fs}; provider configuration ships in + * v1. The fs tier is what unblocks local dev / docker / private deploys + * that don't have an object store configured. + */ +@Slf4j +@Service +public class PayloadStore { + + private static final String SCHEME = "mwf://"; + private static final String STORAGE_KIND_INLINE = "inline"; + private static final String STORAGE_KIND_FS = "fs"; + + private final WorkflowPayloadMapper payloadMapper; + private final ObjectMapper objectMapper; + private final long inlineMaxBytes; + private final long hardCapBytes; + private final Path fsRoot; + private final long retentionDays; + + public PayloadStore(WorkflowPayloadMapper payloadMapper, + ObjectMapper objectMapper, + @Value("${mateclaw.workflow.payload.inline-max-bytes:262144}") long inlineMaxBytes, + @Value("${mateclaw.workflow.payload.hard-cap-bytes:52428800}") long hardCapBytes, + @Value("${mateclaw.workflow.payload.fs.root:./data/workflow-payload}") String fsRoot, + @Value("${mateclaw.workflow.payload.retention-days:30}") long retentionDays) { + this.payloadMapper = payloadMapper; + this.objectMapper = objectMapper; + this.inlineMaxBytes = inlineMaxBytes; + this.hardCapBytes = hardCapBytes; + this.fsRoot = Path.of(fsRoot).toAbsolutePath(); + this.retentionDays = retentionDays; + } + + /** Store a UTF-8 string payload and return its stable URI. */ + public String storeString(long workspaceId, String body, String contentType) { + byte[] bytes = (body == null ? "" : body).getBytes(StandardCharsets.UTF_8); + return storeBytes(workspaceId, bytes, contentType == null ? "text/plain" : contentType); + } + + /** JSON-encode {@code value} and store it. {@code contentType} is fixed to {@code application/json}. */ + public String storeJson(long workspaceId, Object value) { + try { + byte[] bytes = objectMapper.writeValueAsBytes(value); + return storeBytes(workspaceId, bytes, "application/json"); + } catch (JsonProcessingException e) { + throw new PayloadStoreException("failed to serialize payload as JSON: " + e.getMessage(), e); + } + } + + /** Store raw bytes and return the URI. Routes by size: inline → fs → reject. */ + public String storeBytes(long workspaceId, byte[] bytes, String contentType) { + Objects.requireNonNull(bytes, "bytes"); + if (bytes.length > hardCapBytes) { + throw new PayloadStoreException("payload exceeds hard cap of " + + hardCapBytes + " bytes (got " + bytes.length + ")"); + } + String uri = SCHEME + workspaceId + "/" + UUID.randomUUID(); + + WorkflowPayloadEntity row = new WorkflowPayloadEntity(); + row.setPayloadUri(uri); + row.setWorkspaceId(workspaceId); + row.setContentType(contentType); + row.setSha256(sha256Hex(bytes)); + row.setSizeBytes((long) bytes.length); + row.setCreatedAt(LocalDateTime.now()); + + if (bytes.length <= inlineMaxBytes) { + row.setContentBytes(bytes); + row.setStorageKind(STORAGE_KIND_INLINE); + } else { + // Spill to filesystem so we don't bloat the DB row. Path layout + // is {fsRoot}/{workspaceId}/{first2chars}/{uuid} so a single + // workspace can't pile millions of files into one directory. + String relative = workspaceId + "/" + uri.substring(uri.length() - 2) + + "/" + uri.substring(uri.length() - Math.min(36, uri.length())); + Path target = fsRoot.resolve(relative); + try { + Files.createDirectories(target.getParent()); + Files.write(target, bytes); + } catch (IOException e) { + throw new PayloadStoreException("failed to write fs payload " + uri + + ": " + e.getMessage(), e); + } + row.setStorageKind(STORAGE_KIND_FS); + row.setStorageRef(relative); + } + + payloadMapper.insert(row); + return uri; + } + + /** Resolve a payload URI to its raw bytes; throws when the URI is unknown. */ + public byte[] readBytes(String payloadUri) { + WorkflowPayloadEntity row = lookup(payloadUri); + if (STORAGE_KIND_FS.equals(row.getStorageKind())) { + try { + return Files.readAllBytes(fsRoot.resolve(row.getStorageRef())); + } catch (IOException e) { + throw new PayloadStoreException("failed to read fs payload " + payloadUri + + ": " + e.getMessage(), e); + } + } + return row.getContentBytes() == null ? new byte[0] : row.getContentBytes(); + } + + /** Resolve a payload URI to its UTF-8 decoded string body. */ + public String readString(String payloadUri) { + return new String(readBytes(payloadUri), StandardCharsets.UTF_8); + } + + /** Resolve a payload URI to its JSON body parsed back into the requested shape. */ + public T readJson(String payloadUri, Class type) { + try { + return objectMapper.readValue(readBytes(payloadUri), type); + } catch (Exception e) { + throw new PayloadStoreException( + "failed to deserialize payload " + payloadUri + " as " + type.getSimpleName() + + ": " + e.getMessage(), + e); + } + } + + private WorkflowPayloadEntity lookup(String payloadUri) { + WorkflowPayloadEntity row = payloadMapper.selectOne( + new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .eq(WorkflowPayloadEntity::getPayloadUri, payloadUri)); + if (row == null) { + throw new PayloadStoreException("payload not found: " + payloadUri); + } + return row; + } + + private static String sha256Hex(byte[] bytes) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + return HexFormat.of().formatHex(digest.digest(bytes)); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is part of the JCA standard set — should never happen. + throw new IllegalStateException("SHA-256 not available", e); + } + } + + /** + * Drop payload rows older than {@code retention-days}. Tombstones the + * filesystem files for fs-tier payloads in the same pass so the disk + * doesn't keep growing once the DB row is gone. Returns the number of + * rows actually deleted; primarily for tests + log lines. + * + *

v0 deletes by absolute age rather than walking the + * {@code mate_workflow_run} graph — runs that finish stay queryable + * for {@code retention-days} from the payload-write timestamp, which + * is "good enough" for an alpha. v1 can switch to run-state-driven + * GC ({@code state IN ('succeeded','failed') AND completed_at < + * threshold}) once the operator UI exposes a "preserve forever" flag + * for runs the customer wants kept. + */ + public int sweepExpired() { + if (retentionDays <= 0) return 0; + LocalDateTime cutoff = LocalDateTime.now().minusDays(retentionDays); + List stale = payloadMapper.selectList( + new com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper() + .lt(WorkflowPayloadEntity::getCreatedAt, cutoff)); + if (stale.isEmpty()) return 0; + int deleted = 0; + for (WorkflowPayloadEntity row : stale) { + // Best-effort fs cleanup before the row goes — the row IS the + // foreign key the file is reachable through; if the row goes + // first the file becomes orphaned. + if (STORAGE_KIND_FS.equals(row.getStorageKind()) && row.getStorageRef() != null) { + try { + Files.deleteIfExists(fsRoot.resolve(row.getStorageRef())); + } catch (IOException e) { + log.warn("[PayloadStore] fs delete failed for {}: {}", + row.getStorageRef(), e.getMessage()); + } + } + try { + payloadMapper.deleteById(row.getId()); + deleted++; + } catch (Exception e) { + log.warn("[PayloadStore] db delete failed for payload {}: {}", + row.getPayloadUri(), e.getMessage()); + } + } + return deleted; + } + + /** + * Periodic sweep — runs once an hour by default. Tunable via + * {@code mateclaw.workflow.payload.sweep-interval-ms}. Skips a tick + * silently when retentionDays = 0 (operator opted out of GC). + */ + @Scheduled( + fixedDelayString = "${mateclaw.workflow.payload.sweep-interval-ms:3600000}", + initialDelayString = "${mateclaw.workflow.payload.sweep-initial-delay-ms:600000}") + public void scheduledSweepExpired() { + try { + int dropped = sweepExpired(); + if (dropped > 0) { + log.info("[PayloadStore] swept {} expired payload rows (retention={} days)", + dropped, retentionDays); + } + } catch (Exception e) { + log.warn("[PayloadStore] periodic sweep failed: {}", e.getMessage()); + } + } + + /** Wrapper exception for payload-store failures. */ + public static class PayloadStoreException extends RuntimeException { + public PayloadStoreException(String message) { super(message); } + public PayloadStoreException(String message, Throwable cause) { super(message, cause); } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapter.java new file mode 100644 index 00000000..64e997f0 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapter.java @@ -0,0 +1,27 @@ +package vip.mate.workflow.runtime; + +import vip.mate.workflow.compiler.ir.WorkflowStep; + +/** + * Strategy interface for executing a single workflow step. One implementation + * per {@code StepMode.typeName()}; the runner looks up the adapter by name and + * calls {@link #execute}. Adapters MUST NOT mutate {@link WorkflowRunContext} + * directly — the runner publishes the {@link StepResult} into the context so + * fan_out groups can merge in deterministic order. + */ +public interface StepAdapter { + + /** + * The mode name this adapter handles — must match + * {@code StepMode.typeName()} (sequential / fan_out / collect / conditional / + * await_approval / dispatch_channel / write_memory). + */ + String typeName(); + + /** + * Execute one step. Implementations should never throw to signal a normal + * step failure — return {@link StepResult#failed(String)} instead. Throwing + * is reserved for programmer / framework errors that should abort the run. + */ + StepResult execute(WorkflowStep step, WorkflowRunContext context); +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapterRegistry.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapterRegistry.java new file mode 100644 index 00000000..21064a39 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepAdapterRegistry.java @@ -0,0 +1,39 @@ +package vip.mate.workflow.runtime; + +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * Registry mapping mode {@code typeName} to its {@link StepAdapter} bean. + * Spring autowires every adapter on the classpath; the runner asks the + * registry which adapter to use and the registry rejects unknown modes + * up-front so a wiring bug surfaces at the run boundary instead of inside + * the executor loop. + */ +@Component +public class StepAdapterRegistry { + + private final Map adapters; + + public StepAdapterRegistry(List adapters) { + Map mapped = adapters.stream() + .collect(Collectors.toUnmodifiableMap(StepAdapter::typeName, Function.identity())); + this.adapters = mapped; + } + + public StepAdapter get(String typeName) { + StepAdapter adapter = adapters.get(typeName); + if (adapter == null) { + throw new IllegalStateException("no step adapter registered for mode: " + typeName); + } + return adapter; + } + + public boolean has(String typeName) { + return adapters.containsKey(typeName); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepResult.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepResult.java new file mode 100644 index 00000000..b49ee83a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/StepResult.java @@ -0,0 +1,51 @@ +package vip.mate.workflow.runtime; + +/** + * Outcome reported by a {@link StepAdapter#execute}. Records: + *

    + *
  • {@link State} — succeeded / skipped / failed / paused; the runner + * translates the first three to {@code mate_workflow_run_step.state} + * and the last to a graceful run-pause exit.
  • + *
  • {@code outputPayloadUri} — payload URI for the step's output, or + * {@code null} when the step produced nothing (skipped, collect, paused).
  • + *
  • {@code outputContentType} — resolved content type, defaults to + * {@code text}; lets the runner persist {@code output_content_type} + * without rebuilding the step contract.
  • + *
  • {@code outputValue} — the in-memory value to publish into the + * run context's {@code outputs} map. {@link String} for text content, + * {@link java.util.Map} / {@link java.util.List} for json content. + * {@code null} when the step has no {@code outputVar}.
  • + *
  • {@code outputSummary} / {@code errorMessage} — short labels for the + * step row; both optional.
  • + *
  • {@code pauseToken} — set when {@code state == PAUSED}; the resume + * entry key the resumer expects callers to present.
  • + *
+ */ +public record StepResult( + State state, + String outputPayloadUri, + String outputContentType, + Object outputValue, + String outputSummary, + String errorMessage, + String pauseToken +) { + + public enum State { SUCCEEDED, SKIPPED, FAILED, PAUSED } + + public static StepResult succeeded(String payloadUri, String contentType, Object value, String summary) { + return new StepResult(State.SUCCEEDED, payloadUri, contentType, value, summary, null, null); + } + + public static StepResult skipped(String reason) { + return new StepResult(State.SKIPPED, null, null, null, reason, null, null); + } + + public static StepResult failed(String errorMessage) { + return new StepResult(State.FAILED, null, null, null, null, errorMessage, null); + } + + public static StepResult paused(String pauseToken, String summary) { + return new StepResult(State.PAUSED, null, null, null, summary, null, pauseToken); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowCompletionEvent.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowCompletionEvent.java new file mode 100644 index 00000000..c8ced26a --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowCompletionEvent.java @@ -0,0 +1,24 @@ +package vip.mate.workflow.runtime; + +/** + * Spring application event fired when a workflow run reaches a terminal + * state ({@code succeeded} / {@code failed}). The trigger module + * subscribes via {@code @EventListener} and pushes the payload through + * {@link vip.mate.trigger.ingest.TriggerEventIngestService} so downstream + * triggers (e.g. {@code workflow_completion} pattern) can chain off the + * outcome. + * + *

Going through the event bus instead of injecting the trigger + * service directly into the workflow runner breaks the + * Runner ↔ Dispatcher ↔ Ingest ↔ Runner circular dependency that Spring + * would otherwise refuse to construct. + */ +public record WorkflowCompletionEvent( + long runId, + long workflowId, + long revisionId, + long workspaceId, + String state, + String finalOutputRef, + String errorMessage +) {} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowResumer.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowResumer.java new file mode 100644 index 00000000..116d62a4 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowResumer.java @@ -0,0 +1,216 @@ +package vip.mate.workflow.runtime; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.model.WorkflowRunPauseEntity; +import vip.mate.workflow.model.WorkflowRunStepEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.repository.WorkflowRunPauseMapper; +import vip.mate.workflow.repository.WorkflowRunStepMapper; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +/** + * Settles a paused workflow run. Callers (approval callbacks, timeout sweeper, + * REST endpoints) hand in a {@code pauseToken} and an outcome; the resumer + * marks the pause and the await_approval step row, hydrates a fresh + * {@link WorkflowRunContext} from the persisted step rows, and delegates back + * to {@link WorkflowRunner#continueFromIndex} for the post-pause tail. + * + *

Idempotent: a pause that has already been resumed yields + * {@link Outcome#alreadyResolved(long)} without touching DB or memory. The + * graph is loaded by the caller (typically via a revision-id lookup) since the + * resumer has no opinion on storage. + */ +@Slf4j +@Service +public class WorkflowResumer { + + private static final String STATE_SUCCEEDED = "succeeded"; + private static final String STATE_FAILED = "failed"; + + private final WorkflowRunMapper runMapper; + private final WorkflowRunStepMapper stepMapper; + private final WorkflowRunPauseMapper pauseMapper; + private final WorkflowRunner runner; + private final PayloadStore payloadStore; + private final ObjectMapper objectMapper; + + public WorkflowResumer(WorkflowRunMapper runMapper, + WorkflowRunStepMapper stepMapper, + WorkflowRunPauseMapper pauseMapper, + WorkflowRunner runner, + PayloadStore payloadStore, + ObjectMapper objectMapper) { + this.runMapper = runMapper; + this.stepMapper = stepMapper; + this.pauseMapper = pauseMapper; + this.runner = runner; + this.payloadStore = payloadStore; + this.objectMapper = objectMapper; + } + + public Outcome resume(WorkflowGraph graph, String pauseToken, + ResumeOutcome outcome, byte[] resumePayloadBody) { + WorkflowRunPauseEntity pause = pauseMapper.selectOne(new LambdaQueryWrapper() + .eq(WorkflowRunPauseEntity::getPauseToken, pauseToken)); + if (pause == null) { + return Outcome.notFound(pauseToken); + } + if (pause.getResumedAt() != null) { + return Outcome.alreadyResolved(pause.getRunId()); + } + + WorkflowRunEntity runRow = runMapper.selectById(pause.getRunId()); + if (runRow == null) { + return Outcome.notFound(pauseToken); + } + + WorkflowRunStepEntity stepRow = stepMapper.selectById(pause.getStepId()); + if (stepRow == null) { + return Outcome.notFound(pauseToken); + } + + // Persist the pause row before doing any further work so a crash mid-resume + // leaves a clear audit trail (the pause is settled even if the post-resume + // execution never started). + String resumePayloadRef = null; + if (resumePayloadBody != null && resumePayloadBody.length > 0) { + resumePayloadRef = payloadStore.storeBytes(runRow.getWorkspaceId(), + resumePayloadBody, "application/octet-stream"); + } + pause.setResumedAt(LocalDateTime.now()); + pause.setResumeOutcome(outcome.token()); + pause.setResumePayloadRef(resumePayloadRef); + pauseMapper.updateById(pause); + + // Settle the await_approval step row first. + stepRow.setState(outcome == ResumeOutcome.APPROVED ? STATE_SUCCEEDED : STATE_FAILED); + stepRow.setOutputSummary("resumed: " + outcome.token()); + stepRow.setCompletedAt(LocalDateTime.now()); + if (outcome != ResumeOutcome.APPROVED) { + stepRow.setErrorMessage("approval " + outcome.token()); + } + stepMapper.updateById(stepRow); + + if (outcome != ResumeOutcome.APPROVED) { + // Failed approval ends the run — no further steps. + runRow.setState(STATE_FAILED); + runRow.setErrorMessage("paused step '" + stepRow.getStepName() + "' " + outcome.token()); + runRow.setCompletedAt(LocalDateTime.now()); + runMapper.updateById(runRow); + // Publish the workflow_completion event downstream — same as the + // runner's finishFailed path. Without this, runs that end on a + // rejected / timed-out approval would never fire their + // completion trigger because the resumer skips + // runner.continueFromIndex on the failure branch. + runner.publishCompletionEvent(runRow, STATE_FAILED, null, runRow.getErrorMessage()); + return Outcome.failed(runRow.getId(), runRow.getErrorMessage()); + } + + // Hydrate the run context from prior step rows so post-resume steps can + // reference {{ outputs.xxx }} from steps that completed before the pause. + WorkflowRunContext ctx = hydrateContext(runRow, graph, stepRow.getStepIndex()); + String priorOutputRef = lastSucceededOutputRef(runRow.getId(), stepRow.getStepIndex()); + + WorkflowRunResult result = runner.continueFromIndex( + graph, ctx, runRow, stepRow.getStepIndex() + 1, priorOutputRef); + return Outcome.continued(result); + } + + private WorkflowRunContext hydrateContext(WorkflowRunEntity runRow, WorkflowGraph graph, + int pausedStepIndex) { + Map inputs = (runRow.getInitialInputRef() == null) + ? Map.of() + : payloadStore.readJson(runRow.getInitialInputRef(), Map.class); + WorkflowRunContext ctx = new WorkflowRunContext( + runRow.getId(), + runRow.getWorkspaceId(), + runRow.getWorkflowId(), + runRow.getRevisionId(), + inputs); + + // Replay the rolling outputs map: walk completed succeeded step rows + // up to the pause and put their parsed payloads back into the context + // under their declared outputVar. + List rows = stepMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunStepEntity::getRunId, runRow.getId()) + .lt(WorkflowRunStepEntity::getStepIndex, pausedStepIndex) + .orderByAsc(WorkflowRunStepEntity::getStepIndex) + .orderByAsc(WorkflowRunStepEntity::getIterationIndex)); + for (WorkflowRunStepEntity row : rows) { + if (!STATE_SUCCEEDED.equals(row.getState()) || row.getOutputRef() == null) continue; + int idx = row.getStepIndex(); + if (idx < 0 || idx >= graph.steps().size()) continue; + var step = graph.steps().get(idx); + if (step.outputVar() == null || step.outputVar().isBlank()) continue; + Object value = decodeOutput(row); + if (value != null) ctx.putOutput(step.outputVar(), value); + } + return ctx; + } + + private Object decodeOutput(WorkflowRunStepEntity row) { + try { + byte[] body = payloadStore.readBytes(row.getOutputRef()); + if ("json".equals(row.getOutputContentType())) { + return objectMapper.readValue(body, Object.class); + } + return new String(body, java.nio.charset.StandardCharsets.UTF_8); + } catch (Exception e) { + log.warn("Workflow resume: failed to decode prior step output ref={}: {}", + row.getOutputRef(), e.getMessage()); + return null; + } + } + + private String lastSucceededOutputRef(long runId, int beforeStepIndex) { + WorkflowRunStepEntity row = stepMapper.selectOne(new LambdaQueryWrapper() + .eq(WorkflowRunStepEntity::getRunId, runId) + .eq(WorkflowRunStepEntity::getState, STATE_SUCCEEDED) + .lt(WorkflowRunStepEntity::getStepIndex, beforeStepIndex) + .isNotNull(WorkflowRunStepEntity::getOutputRef) + .orderByDesc(WorkflowRunStepEntity::getStepIndex) + .orderByDesc(WorkflowRunStepEntity::getIterationIndex) + .last("LIMIT 1")); + return row == null ? null : row.getOutputRef(); + } + + /** Outcome label written to {@code mate_workflow_run_pause.resume_outcome}. */ + public enum ResumeOutcome { + APPROVED("approved"), + REJECTED("rejected"), + TIMEOUT("timeout"), + CANCELLED("cancelled"); + + private final String token; + + ResumeOutcome(String token) { this.token = token; } + + public String token() { return token; } + } + + /** Result of attempting a resume — exposes the final run state when completed inline. */ + public record Outcome(Kind kind, Long runId, WorkflowRunResult finalResult, String errorMessage) { + public enum Kind { CONTINUED, FAILED, ALREADY_RESOLVED, NOT_FOUND } + + public static Outcome continued(WorkflowRunResult r) { + return new Outcome(Kind.CONTINUED, r.runId(), r, null); + } + public static Outcome failed(long runId, String err) { + return new Outcome(Kind.FAILED, runId, null, err); + } + public static Outcome alreadyResolved(long runId) { + return new Outcome(Kind.ALREADY_RESOLVED, runId, null, null); + } + public static Outcome notFound(String token) { + return new Outcome(Kind.NOT_FOUND, null, null, "pause token not found: " + token); + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunContext.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunContext.java new file mode 100644 index 00000000..0f88013b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunContext.java @@ -0,0 +1,101 @@ +package vip.mate.workflow.runtime; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Mutable run-scoped state shared across step adapters. Holds the per-run + * identity ({@code runId}, {@code workspaceId}), the resolved input bag, and + * the rolling outputs map keyed by {@code outputVar}. Adapters mutate this + * after each successful step so subsequent expressions / templates see the + * latest value via {@link #templateContext()}. + * + *

Not thread-safe by itself — the runner ensures a single writer at a time. + * For the fan_out group, adapters write to a temporary local map and the + * runner merges results back into the shared context once the group completes. + */ +public class WorkflowRunContext { + + private final long runId; + private final long workspaceId; + private final long workflowId; + private final long revisionId; + private final Map inputs; + private final Map outputs = new LinkedHashMap<>(); + + public WorkflowRunContext(long runId, long workspaceId, long workflowId, long revisionId, + Map inputs) { + this.runId = runId; + this.workspaceId = workspaceId; + this.workflowId = workflowId; + this.revisionId = revisionId; + this.inputs = inputs == null ? Map.of() : Map.copyOf(inputs); + } + + public long runId() { return runId; } + public long workspaceId() { return workspaceId; } + public long workflowId() { return workflowId; } + public long revisionId() { return revisionId; } + + public Map inputs() { return inputs; } + + /** Mutable outputs map. Use {@link #putOutput} for writes. */ + public synchronized Map outputs() { + return new LinkedHashMap<>(outputs); + } + + public synchronized void putOutput(String name, Object value) { + if (name == null || name.isBlank()) return; + outputs.put(name, value); + } + + /** + * Snapshot map shaped as {@code {"inputs": {...}, "outputs": {...}}} — + * the contract every workflow expression / template assumes. The map is a + * defensive copy so concurrent fan_out branches can render templates + * against a stable view while another branch's success completes. + */ + public synchronized Map templateContext() { + Map ctx = new LinkedHashMap<>(); + ctx.put("inputs", inputs); + ctx.put("outputs", new LinkedHashMap<>(outputs)); + return ctx; + } + + /** + * Build a child context for one fan_out branch. The child shares + * {@code inputs} with the parent (immutable already) and gets a + * deep-copied snapshot of the parent's outputs at branch-entry time + * — writes via the child's {@link #putOutput} do NOT propagate back + * to this context until the runner explicitly merges them after the + * group completes. That snapshot isolation is what stops branch B's + * Pebble template from observing branch A's mid-flight write + * (or vice-versa) when they race on the executor. + * + *

The merge step is owned by the runner — see + * {@code WorkflowRunner.executeFanOutGroup}. The branch's own + * {@code outputVar} write IS still visible inside the branch, which + * is what the schema validator promises authors: a branch can see + * its own value but never its sibling branches'. + */ + public synchronized WorkflowRunContext branchSnapshot() { + WorkflowRunContext child = new WorkflowRunContext(runId, workspaceId, + workflowId, revisionId, inputs); + // Seed the child with a snapshot of the parent's outputs so the + // branch can read everything that completed before the fan_out + // group started, but its own writes stay local. + child.outputs.putAll(this.outputs); + return child; + } + + /** + * Merge a single key/value into the outputs map. Used by the runner + * after a fan_out group completes to apply each branch's + * {@code outputVar} to the master context in deterministic + * step-index order. + */ + public synchronized void mergeOutput(String name, Object value) { + if (name == null || name.isBlank()) return; + outputs.put(name, value); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunRequest.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunRequest.java new file mode 100644 index 00000000..d74f1b38 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunRequest.java @@ -0,0 +1,22 @@ +package vip.mate.workflow.runtime; + +import java.util.Map; + +/** + * Inputs the runner needs to start a single workflow run. Identity fields + * ({@code workflowId}, {@code revisionId}, {@code workspaceId}) tie the run + * row back to the published revision the runner walks. {@code triggeredBy} + * is a free-form label written into {@code mate_workflow_run.triggered_by} + * — the runner doesn't interpret it. + */ +public record WorkflowRunRequest( + long workflowId, + long revisionId, + long workspaceId, + String triggeredBy, + Map inputs +) { + public WorkflowRunRequest { + inputs = inputs == null ? Map.of() : Map.copyOf(inputs); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunResult.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunResult.java new file mode 100644 index 00000000..c878a850 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunResult.java @@ -0,0 +1,16 @@ +package vip.mate.workflow.runtime; + +/** + * Public outcome of a workflow run. {@code state} mirrors the row state + * machine ({@code succeeded} / {@code failed}); {@code finalOutputUri} is + * the payload URI of the last non-skipped step's output, or {@code null} + * when no step produced output. {@code errorMessage} is populated when the + * run aborted; {@code null} on success. + */ +public record WorkflowRunResult( + long runId, + String state, + String finalOutputUri, + String errorMessage +) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunner.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunner.java new file mode 100644 index 00000000..9e2f8199 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/WorkflowRunner.java @@ -0,0 +1,380 @@ +package vip.mate.workflow.runtime; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.stereotype.Service; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.model.WorkflowRunStepEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.repository.WorkflowRunStepMapper; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +/** + * Linear executor for v0 workflows. Walks the graph step-by-step, batching + * adjacent {@code fan_out} steps + terminating {@code collect} into a single + * parallel group. The first failed (non-skipped) step aborts the run and + * marks the row {@code failed}. The last non-skipped step's output payload + * is recorded as {@code final_output_ref} on success. + * + *

v0 runtime decision: StateGraph is intentionally not used here. + * The seven v0 modes (sequential / fan_out / collect / conditional + + * await_approval / dispatch_channel / write_memory) are linear plus one + * bounded parallel section, which this small executor handles more + * directly than wrapping a graph DSL. {@code await_approval} pause / resume + * is implemented via {@link WorkflowResumer} reading the persisted + * {@code mate_workflow_run_pause} row, so a JVM restart still recovers the + * run. v1 will reassess whether to graduate to a graph-backed scheduler + * once {@code loop} / {@code invoke_skill} land — until then, "linear + * executor" is the explicit, supported runtime. + * + *

StateGraph remains in use elsewhere for agent-internal control flow + * (ReAct / Plan-Execute) — that's the runtime owned by + * {@link vip.mate.agent agent module}, not this workflow module. + */ +@Slf4j +@Service +public class WorkflowRunner { + + private static final String STATE_RUNNING = "running"; + private static final String STATE_SUCCEEDED = "succeeded"; + private static final String STATE_FAILED = "failed"; + private static final String STATE_SKIPPED = "skipped"; + private static final String STATE_PAUSED = "paused"; + + private static final ExecutorService FAN_OUT_EXECUTOR = + Executors.newVirtualThreadPerTaskExecutor(); + + private final WorkflowRunMapper runMapper; + private final WorkflowRunStepMapper stepMapper; + private final StepAdapterRegistry adapters; + private final PayloadStore payloadStore; + /** Optional — wired in production, may be null in narrow test contexts. + * Spring's stock publisher is always available in a full context. */ + @Autowired(required = false) + private ApplicationEventPublisher events; + + public WorkflowRunner(WorkflowRunMapper runMapper, + WorkflowRunStepMapper stepMapper, + StepAdapterRegistry adapters, + PayloadStore payloadStore) { + this.runMapper = runMapper; + this.stepMapper = stepMapper; + this.adapters = adapters; + this.payloadStore = payloadStore; + } + + public WorkflowRunResult run(WorkflowGraph graph, WorkflowRunRequest request) { + WorkflowRunEntity runRow = openRun(request); + String inputsRef = payloadStore.storeJson(request.workspaceId(), request.inputs()); + runRow.setInitialInputRef(inputsRef); + runMapper.updateById(runRow); + + WorkflowRunContext ctx = new WorkflowRunContext( + runRow.getId(), + request.workspaceId(), + request.workflowId(), + request.revisionId(), + request.inputs()); + + return executeFromIndex(graph, ctx, runRow, /*fromIndex*/ 0, /*priorOutputRef*/ null); + } + + /** + * Continue an already-open run from {@code fromIndex}. Used by the resumer + * after a pause settles. {@code priorOutputRef} is the last successful + * step's output URI from before the pause — propagated so the + * {@code final_output_ref} on success still points at meaningful data when + * the post-resume tail of the run produces no further output. + */ + public WorkflowRunResult continueFromIndex(WorkflowGraph graph, WorkflowRunContext ctx, + WorkflowRunEntity runRow, int fromIndex, + String priorOutputRef) { + // Move the run row back to running so step-completion timestamps make + // sense and the GC sweeper does not see a stale paused row. + runRow.setState(STATE_RUNNING); + runMapper.updateById(runRow); + return executeFromIndex(graph, ctx, runRow, fromIndex, priorOutputRef); + } + + private WorkflowRunResult executeFromIndex(WorkflowGraph graph, WorkflowRunContext ctx, + WorkflowRunEntity runRow, int fromIndex, + String priorOutputRef) { + String lastSucceededOutputRef = priorOutputRef; + try { + int i = fromIndex; + while (i < graph.steps().size()) { + WorkflowStep step = graph.steps().get(i); + int groupEnd = scanFanOutGroup(graph.steps(), i); + if (groupEnd > i) { + GroupOutcome out = executeFanOutGroup(graph.steps(), i, groupEnd, ctx); + if (out.failed) { + return finishFailed(runRow, out.errorMessage); + } + if (out.lastOutputRef != null) lastSucceededOutputRef = out.lastOutputRef; + i = groupEnd + 1; + } else { + StepResult result = executeStep(step, i, /*iterationIndex*/ null, ctx); + if (result.state() == StepResult.State.FAILED) { + return finishFailed(runRow, result.errorMessage()); + } + if (result.state() == StepResult.State.PAUSED) { + return finishPaused(runRow, result.pauseToken()); + } + if (result.outputPayloadUri() != null) { + lastSucceededOutputRef = result.outputPayloadUri(); + } + i++; + } + } + return finishSucceeded(runRow, lastSucceededOutputRef); + } catch (RuntimeException e) { + log.error("Workflow run {} aborted by unexpected exception", ctx.runId(), e); + return finishFailed(runRow, "runtime error: " + e.getMessage()); + } + } + + /** + * Result of executing a contiguous {@code fan_out ... collect} block: + * either every branch succeeded (or skipped) and the merged outputs are + * already in the run context, or one branch failed and the runner aborts. + */ + private record GroupOutcome(boolean failed, String errorMessage, String lastOutputRef) {} + + /** + * If {@code steps[start]} is the head of a fan_out group (≥ 2 consecutive + * fan_out followed by exactly one collect — the schema validator already + * enforced this), return the index of the terminating collect. Otherwise + * return {@code start} so the caller treats it as a single-step. + */ + private static int scanFanOutGroup(List steps, int start) { + if (!(steps.get(start).mode() instanceof StepMode.FanOut)) return start; + int j = start; + while (j < steps.size() && steps.get(j).mode() instanceof StepMode.FanOut) j++; + if (j < steps.size() && steps.get(j).mode() instanceof StepMode.Collect) { + return j; + } + return start; + } + + private GroupOutcome executeFanOutGroup(List steps, int from, int collectIdx, + WorkflowRunContext ctx) { + // Steps from..collectIdx-1 are fan_out branches; collectIdx is the join. + // + // RFC §2.4 requires every branch to render expressions / prompts + // against the SAME context snapshot taken at group entry, with + // collect doing the merge. To honour that we hand each branch its + // own isolated WorkflowRunContext via branchSnapshot() — writes + // inside a branch (via ctx.putOutput from executeStep) land in + // that local copy and stay invisible to siblings until merge + // time. Without this, a branch racing ahead would mutate the + // shared outputs map and the slower branch's Pebble template + // would observe a mid-flight value, making rendering + // schedule-dependent. + record Branch(int stepIndex, WorkflowStep step, + WorkflowRunContext branchCtx, Future future) {} + List branches = new ArrayList<>(); + for (int i = from; i < collectIdx; i++) { + int idx = i; + WorkflowStep step = steps.get(i); + WorkflowRunContext branchCtx = ctx.branchSnapshot(); + Future future = FAN_OUT_EXECUTOR.submit( + () -> executeStep(step, idx, idx - from, branchCtx)); + branches.add(new Branch(idx, step, branchCtx, future)); + } + + // Collect succeeded branch results in step-index order. The + // result list lets us merge outputs into the master context + // deterministically below — a branch's outputVar always wins + // over a smaller-index branch's outputVar with the same name, + // so the conflict policy is "later step wins" and is independent + // of completion order. + record Settled(int stepIndex, WorkflowStep step, StepResult result) {} + List settled = new ArrayList<>(branches.size()); + for (Branch branch : branches) { + try { + StepResult result = branch.future.get(resolveTimeoutSecs(branch.step), TimeUnit.SECONDS); + if (result.state() == StepResult.State.FAILED) { + return new GroupOutcome(true, + "fan_out branch '" + branch.step.name() + "' failed: " + result.errorMessage(), + null); + } + settled.add(new Settled(branch.stepIndex, branch.step, result)); + } catch (Exception e) { + return new GroupOutcome(true, + "fan_out branch '" + branch.step.name() + "' threw: " + e.getMessage(), + null); + } + } + + // Merge phase — the master context only learns about a branch's + // outputVar value here, so collect (and any subsequent step) + // sees a stable, schedule-independent view. + String lastOutputRef = null; + settled.sort((a, b) -> Integer.compare(a.stepIndex, b.stepIndex)); + for (Settled s : settled) { + if (s.result.state() != StepResult.State.SUCCEEDED) continue; + if (s.step.outputVar() != null && !s.step.outputVar().isBlank() + && s.result.outputValue() != null) { + ctx.mergeOutput(s.step.outputVar(), s.result.outputValue()); + } + if (s.result.outputPayloadUri() != null) lastOutputRef = s.result.outputPayloadUri(); + } + + // Run the collect adapter so the join is captured as its own row. + StepResult collectResult = executeStep(steps.get(collectIdx), collectIdx, null, ctx); + if (collectResult.state() == StepResult.State.FAILED) { + return new GroupOutcome(true, collectResult.errorMessage(), null); + } + return new GroupOutcome(false, null, lastOutputRef); + } + + private static long resolveTimeoutSecs(WorkflowStep step) { + if (step.timeoutSecs() == null || step.timeoutSecs() <= 0) return 600L; + return step.timeoutSecs(); + } + + private StepResult executeStep(WorkflowStep step, int stepIndex, Integer iterationIndex, + WorkflowRunContext ctx) { + StepAdapter adapter = adapters.get(step.mode().typeName()); + WorkflowRunStepEntity stepRow = openStep(ctx.runId(), stepIndex, iterationIndex, step); + + long startNanos = System.nanoTime(); + StepResult result; + try { + result = adapter.execute(step, ctx); + } catch (RuntimeException e) { + log.error("Adapter {} threw on run={} stepIndex={} step='{}'", + step.mode().typeName(), ctx.runId(), stepIndex, step.name(), e); + result = StepResult.failed("adapter threw: " + e.getMessage()); + } + long elapsedMs = Duration.ofNanos(System.nanoTime() - startNanos).toMillis(); + + // ctx.putOutput is synchronised internally so concurrent fan_out + // branches can commit their results back to the shared run context + // without external locking. + if (result.state() == StepResult.State.SUCCEEDED && step.outputVar() != null + && !step.outputVar().isBlank() && result.outputValue() != null) { + ctx.putOutput(step.outputVar(), result.outputValue()); + } + + closeStep(stepRow, result, elapsedMs); + return result; + } + + private WorkflowRunEntity openRun(WorkflowRunRequest request) { + WorkflowRunEntity row = new WorkflowRunEntity(); + row.setWorkflowId(request.workflowId()); + row.setRevisionId(request.revisionId()); + row.setWorkspaceId(request.workspaceId()); + row.setState(STATE_RUNNING); + row.setTriggeredBy(request.triggeredBy()); + row.setStartedAt(LocalDateTime.now()); + runMapper.insert(row); + return row; + } + + private WorkflowRunResult finishSucceeded(WorkflowRunEntity runRow, String finalOutputRef) { + runRow.setState(STATE_SUCCEEDED); + runRow.setFinalOutputRef(finalOutputRef); + runRow.setCompletedAt(LocalDateTime.now()); + runMapper.updateById(runRow); + publishCompletionEvent(runRow, STATE_SUCCEEDED, finalOutputRef, null); + return new WorkflowRunResult(runRow.getId(), STATE_SUCCEEDED, finalOutputRef, null); + } + + private WorkflowRunResult finishFailed(WorkflowRunEntity runRow, String errorMessage) { + runRow.setState(STATE_FAILED); + runRow.setErrorMessage(errorMessage); + runRow.setCompletedAt(LocalDateTime.now()); + runMapper.updateById(runRow); + publishCompletionEvent(runRow, STATE_FAILED, null, errorMessage); + return new WorkflowRunResult(runRow.getId(), STATE_FAILED, null, errorMessage); + } + + /** + * Fire a {@code workflow_completion} event into the trigger pipeline so + * downstream workflows (or workflows reacting to upstream success / + * failure) can chain off this run. Synchronous and best-effort: a + * fan-out failure here MUST NOT corrupt the just-completed run state. + * + *

The eventId is keyed on {@code wf-run-{runId}} so a retry of the + * same run never duplicate-fires its completion downstream — the + * mate_trigger_event UNIQUE(trigger_id, dedup_key) constraint catches + * any redundant publish at insert time. + * + *

Package-private so {@link WorkflowResumer} can publish the same + * event for resumed runs that end on a rejected / timed-out approval + * (those don't go through {@link #finishFailed} since the resumer + * writes terminal state directly). + */ + void publishCompletionEvent(WorkflowRunEntity runRow, String state, + String finalOutputRef, String errorMessage) { + if (events == null || runRow == null) return; + try { + events.publishEvent(new WorkflowCompletionEvent( + runRow.getId(), + runRow.getWorkflowId() == null ? 0L : runRow.getWorkflowId(), + runRow.getRevisionId() == null ? 0L : runRow.getRevisionId(), + runRow.getWorkspaceId() == null ? 0L : runRow.getWorkspaceId(), + state, + finalOutputRef, + errorMessage)); + } catch (Exception e) { + log.warn("Workflow run {} completion event publish failed: {}", + runRow.getId(), e.getMessage()); + } + } + + private WorkflowRunResult finishPaused(WorkflowRunEntity runRow, String pauseToken) { + runRow.setState(STATE_PAUSED); + // Pause leaves the run open — completedAt stays null until resume settles it. + runMapper.updateById(runRow); + return new WorkflowRunResult(runRow.getId(), STATE_PAUSED, null, "pauseToken=" + pauseToken); + } + + private WorkflowRunStepEntity openStep(long runId, int stepIndex, Integer iterationIndex, + WorkflowStep step) { + WorkflowRunStepEntity row = new WorkflowRunStepEntity(); + row.setRunId(runId); + row.setStepIndex(stepIndex); + row.setIterationIndex(iterationIndex); + row.setStepName(step.name()); + row.setAgentId(step.agentId()); + row.setState(STATE_RUNNING); + row.setOutputContentType(step.effectiveOutputContentType()); + row.setStartedAt(LocalDateTime.now()); + stepMapper.insert(row); + return row; + } + + private void closeStep(WorkflowRunStepEntity row, StepResult result, long durationMs) { + switch (result.state()) { + case SUCCEEDED -> row.setState(STATE_SUCCEEDED); + case SKIPPED -> row.setState(STATE_SKIPPED); + case FAILED -> row.setState(STATE_FAILED); + case PAUSED -> row.setState(STATE_PAUSED); + } + row.setOutputRef(result.outputPayloadUri()); + if (result.outputContentType() != null) { + row.setOutputContentType(result.outputContentType()); + } + row.setOutputSummary(result.outputSummary()); + row.setErrorMessage(result.errorMessage()); + row.setDurationMs(durationMs); + row.setCompletedAt(LocalDateTime.now()); + stepMapper.updateById(row); + } + +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/AwaitApprovalStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/AwaitApprovalStepAdapter.java new file mode 100644 index 00000000..082a6991 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/AwaitApprovalStepAdapter.java @@ -0,0 +1,135 @@ +package vip.mate.workflow.runtime.mode; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import vip.mate.approval.ApprovalWorkflowService; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.model.WorkflowRunPauseEntity; +import vip.mate.workflow.model.WorkflowRunStepEntity; +import vip.mate.workflow.repository.WorkflowRunPauseMapper; +import vip.mate.workflow.repository.WorkflowRunStepMapper; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +import java.time.LocalDateTime; +import java.util.UUID; + +/** + * {@code await_approval} — pauses the run pending an external approval + * decision. Inserts a {@code mate_workflow_run_pause} row keyed by a fresh + * {@code pauseToken}, then returns {@link StepResult.State#PAUSED} so the + * runner can short-circuit and mark the run row {@code paused}. + * + *

Resolution path (v0): + *

    + *
  1. Operator UI lists paused runs via {@code GET /api/v1/workflows/runs/paused}, + * which returns the run + the active pause record (including the + * {@code pauseToken}).
  2. + *
  3. Operator picks an outcome and POSTs to + * {@code /api/v1/workflows/runs/{runId}/resume} with the + * {@code pauseToken} and {@code outcome ∈ {approved, rejected, timeout, cancelled}}.
  4. + *
  5. {@code WorkflowResumer} marks the pause row resolved and advances + * the run state machine.
  6. + *
+ * + *

The pause row's {@code resume_deadline} is honoured when the step + * declares a {@code timeoutSecs}; otherwise it stays {@code null} and the + * resumer treats the pause as open-ended. + * + *

The {@code external_approval_id} column on the pause row links to the + * {@code mate_tool_approval} row created via + * {@link ApprovalWorkflowService#requestWorkflowApproval} so the workflow + * pause is visible in the same approval inbox the tool-approval flow uses. + * Resolution still goes through {@code WorkflowResumeController} + + * pauseToken — the approval row is for operator visibility today; v1 wires + * the resolve→resume callback so an inbox decision can also fire the + * resumer. + */ +@Component +public class AwaitApprovalStepAdapter implements StepAdapter { + + private final WorkflowRunPauseMapper pauseMapper; + private final WorkflowRunStepMapper stepMapper; + /** Optional — not all test contexts wire the approval module up. The + * adapter falls back to a no-op approval row when null. */ + @Autowired(required = false) + private ApprovalWorkflowService approvalService; + + public AwaitApprovalStepAdapter(WorkflowRunPauseMapper pauseMapper, + WorkflowRunStepMapper stepMapper) { + this.pauseMapper = pauseMapper; + this.stepMapper = stepMapper; + } + + @Override + public String typeName() { return "await_approval"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + if (!(step.mode() instanceof StepMode.AwaitApproval cfg)) { + return StepResult.failed("await_approval adapter received non-await mode: " + + step.mode().typeName()); + } + + // Look up the freshly opened step row so we can link the pause to it. + WorkflowRunStepEntity stepRow = stepMapper.selectOne(new LambdaQueryWrapper() + .eq(WorkflowRunStepEntity::getRunId, context.runId()) + .eq(WorkflowRunStepEntity::getStepName, step.name()) + .orderByDesc(WorkflowRunStepEntity::getId) + .last("LIMIT 1")); + if (stepRow == null) { + return StepResult.failed("await_approval could not locate its run-step row"); + } + + String pauseToken = UUID.randomUUID().toString(); + LocalDateTime now = LocalDateTime.now(); + + // Insert the pause row first so we have a stable id to reference + // even if the approval-service call below fails. + WorkflowRunPauseEntity pause = new WorkflowRunPauseEntity(); + pause.setRunId(context.runId()); + pause.setStepId(stepRow.getId()); + pause.setPauseKind("await_approval"); + pause.setPauseToken(pauseToken); + pause.setPausedAt(now); + if (cfg.timeoutSecs() != null && cfg.timeoutSecs() > 0) { + pause.setResumeDeadline(now.plusSeconds(cfg.timeoutSecs())); + } + pauseMapper.insert(pause); + + // Bridge into the approval inbox: create a mate_tool_approval row so + // the workflow pause shows up alongside tool approvals, then write + // the row id back as external_approval_id for the future + // resolve→resume callback. Failures here are non-fatal — the run + // is still resolvable via pauseToken + WorkflowResumeController. + if (approvalService != null) { + try { + Long approvalId = approvalService.requestWorkflowApproval( + context.workspaceId(), + context.runId(), + stepRow.getId(), + cfg.approvalKind(), + cfg.approvalMessage(), + cfg.approverChannels(), + cfg.timeoutSecs()); + if (approvalId != null) { + pause.setExternalApprovalId(approvalId); + pauseMapper.updateById(pause); + } + } catch (Exception e) { + // Non-fatal — log and continue. The pause row is the + // canonical record for v0; the approval row is a parallel + // visibility surface that can rebuild later if needed. + org.slf4j.LoggerFactory.getLogger(AwaitApprovalStepAdapter.class) + .warn("await_approval failed to create approval row for run {}: {}", + context.runId(), e.getMessage()); + } + } + + return StepResult.paused(pauseToken, + "awaiting " + (cfg.approvalKind() == null ? "approval" : cfg.approvalKind())); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/CollectStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/CollectStepAdapter.java new file mode 100644 index 00000000..edd2260d --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/CollectStepAdapter.java @@ -0,0 +1,26 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +/** + * {@code collect} — barrier that closes the most recent fan_out group. The + * runner awaits the parallel branches before invoking this adapter, then + * publishes their merged outputs into the run context. The adapter itself + * does no agent work; it simply records a step row so the run history shows + * where the group joined and produces no payload of its own. + */ +@Component +public class CollectStepAdapter implements StepAdapter { + + @Override + public String typeName() { return "collect"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + return StepResult.succeeded(null, null, null, "fan_out group joined"); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/ConditionalStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/ConditionalStepAdapter.java new file mode 100644 index 00000000..5da38c14 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/ConditionalStepAdapter.java @@ -0,0 +1,55 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.PebbleSubsetEvaluator; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.AgentStepExecutor; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +/** + * {@code conditional} — runs the embedded agent step only when the configured + * Pebble expression evaluates true against the current run context. A false + * verdict yields {@link StepResult.State#SKIPPED}; an evaluation error fails + * the step. Skipped steps still emit a run-step row so the history captures + * the routing decision. + */ +@Component +public class ConditionalStepAdapter implements StepAdapter { + + private final PebbleSubsetEvaluator pebble; + private final AgentStepExecutor executor; + + public ConditionalStepAdapter(PebbleSubsetEvaluator pebble, AgentStepExecutor executor) { + this.pebble = pebble; + this.executor = executor; + } + + @Override + public String typeName() { return "conditional"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + if (!(step.mode() instanceof StepMode.Conditional cond)) { + return StepResult.failed("conditional adapter received non-conditional mode: " + + step.mode().typeName()); + } + + boolean truth; + try { + var compiled = pebble.parseExpression(cond.expression()); + truth = pebble.evaluateAsBoolean(compiled, context.templateContext()); + } catch (Exception e) { + return StepResult.failed("conditional expression evaluation failed for step '" + + step.name() + "': " + e.getMessage()); + } + + if (!truth) { + return StepResult.skipped("guard expression evaluated false"); + } + + return executor.run(step, context); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/DispatchChannelStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/DispatchChannelStepAdapter.java new file mode 100644 index 00000000..0a85485b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/DispatchChannelStepAdapter.java @@ -0,0 +1,86 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.PebbleSubsetEvaluator; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.ChannelDispatcher; +import vip.mate.workflow.runtime.PayloadStore; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * {@code dispatch_channel} — render the content template, then deliver the + * rendered text to every configured channel via {@link ChannelDispatcher}. + * Targets are looked up in the step's {@code targets} map keyed by channel + * type. The step fails iff any channel fails to deliver; partial successes + * are still flagged failed because step state is binary in v0 and silent + * delivery loss would be worse than an explicit error. + * + *

The rendered content payload is also written through to + * {@code mate_workflow_payload} so the run-step row's {@code output_ref} + * points at exactly what was sent. + */ +@Component +public class DispatchChannelStepAdapter implements StepAdapter { + + private final PebbleSubsetEvaluator pebble; + private final PayloadStore payloadStore; + private final ChannelDispatcher dispatcher; + + public DispatchChannelStepAdapter(PebbleSubsetEvaluator pebble, + PayloadStore payloadStore, + ChannelDispatcher dispatcher) { + this.pebble = pebble; + this.payloadStore = payloadStore; + this.dispatcher = dispatcher; + } + + @Override + public String typeName() { return "dispatch_channel"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + if (!(step.mode() instanceof StepMode.DispatchChannel cfg)) { + return StepResult.failed("dispatch_channel adapter received non-dispatch mode: " + + step.mode().typeName()); + } + + String rendered; + try { + var compiled = pebble.parseTemplate(cfg.content()); + rendered = pebble.evaluateAsString(compiled, context.templateContext()); + } catch (Exception e) { + return StepResult.failed("dispatch_channel content render failed for step '" + + step.name() + "': " + e.getMessage()); + } + + Map targets = cfg.targets() == null ? Map.of() : cfg.targets(); + List failures = new ArrayList<>(); + List delivered = new ArrayList<>(); + for (String channel : cfg.channels()) { + String target = targets.get(channel); + ChannelDispatcher.DispatchResult result = + dispatcher.dispatch(context.workspaceId(), channel, target, rendered); + if (result.success()) { + delivered.add(channel); + } else { + failures.add(channel + ": " + result.message()); + } + } + + String payloadUri = payloadStore.storeString(context.workspaceId(), rendered, "text/plain"); + + if (!failures.isEmpty()) { + return StepResult.failed("dispatch_channel partial / total failure: " + + String.join("; ", failures)); + } + return StepResult.succeeded(payloadUri, "text", rendered, + "delivered to " + String.join(", ", delivered)); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/FanOutStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/FanOutStepAdapter.java new file mode 100644 index 00000000..74428faf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/FanOutStepAdapter.java @@ -0,0 +1,34 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.AgentStepExecutor; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +/** + * {@code fan_out} — body of a parallel group. Each fan_out step runs against + * the run context snapshot that existed when the group started; the runner + * dispatches the whole group in parallel and merges {@code outputs} only when + * the terminating {@code collect} runs. From the adapter's perspective the + * step body is identical to a sequential agent call — the parallelism is + * orchestrated upstream. + */ +@Component +public class FanOutStepAdapter implements StepAdapter { + + private final AgentStepExecutor executor; + + public FanOutStepAdapter(AgentStepExecutor executor) { + this.executor = executor; + } + + @Override + public String typeName() { return "fan_out"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + return executor.run(step, context); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/SequentialStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/SequentialStepAdapter.java new file mode 100644 index 00000000..a73227de --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/SequentialStepAdapter.java @@ -0,0 +1,31 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.AgentStepExecutor; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +/** + * {@code sequential} — runs after the previous step finishes and threads its + * output forward via {@code outputs[outputVar]}. The default mode for any + * agent-call step that does not need parallel or guarded execution. + */ +@Component +public class SequentialStepAdapter implements StepAdapter { + + private final AgentStepExecutor executor; + + public SequentialStepAdapter(AgentStepExecutor executor) { + this.executor = executor; + } + + @Override + public String typeName() { return "sequential"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + return executor.run(step, context); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/WriteMemoryStepAdapter.java b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/WriteMemoryStepAdapter.java new file mode 100644 index 00000000..30a6860c --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/runtime/mode/WriteMemoryStepAdapter.java @@ -0,0 +1,75 @@ +package vip.mate.workflow.runtime.mode; + +import org.springframework.stereotype.Component; +import vip.mate.workflow.compiler.PebbleSubsetEvaluator; +import vip.mate.workflow.compiler.ir.StepMode; +import vip.mate.workflow.compiler.ir.WorkflowStep; +import vip.mate.workflow.runtime.MemoryWriter; +import vip.mate.workflow.runtime.PayloadStore; +import vip.mate.workflow.runtime.StepAdapter; +import vip.mate.workflow.runtime.StepResult; +import vip.mate.workflow.runtime.WorkflowRunContext; + +/** + * {@code write_memory} — render the content template, then delegate to + * {@link MemoryWriter} to apply the configured merge strategy against the + * target memory file. The rendered content is also written through to + * {@code mate_workflow_payload} so the step row's {@code output_ref} points + * at the exact text that was merged in (independent of the file's final + * post-merge state, which downstream tooling may want to diff). + */ +@Component +public class WriteMemoryStepAdapter implements StepAdapter { + + private final PebbleSubsetEvaluator pebble; + private final PayloadStore payloadStore; + private final MemoryWriter memoryWriter; + + public WriteMemoryStepAdapter(PebbleSubsetEvaluator pebble, + PayloadStore payloadStore, + MemoryWriter memoryWriter) { + this.pebble = pebble; + this.payloadStore = payloadStore; + this.memoryWriter = memoryWriter; + } + + @Override + public String typeName() { return "write_memory"; } + + @Override + public StepResult execute(WorkflowStep step, WorkflowRunContext context) { + if (!(step.mode() instanceof StepMode.WriteMemory cfg)) { + return StepResult.failed("write_memory adapter received non-write_memory mode: " + + step.mode().typeName()); + } + + String rendered; + try { + var compiled = pebble.parseTemplate(cfg.content()); + rendered = pebble.evaluateAsString(compiled, context.templateContext()); + } catch (Exception e) { + return StepResult.failed("write_memory content render failed for step '" + + step.name() + "': " + e.getMessage()); + } + + // Resolve template-form employeeId now that the run context exists — + // the publish-time ACL phase deliberately skipped checking templates. + String employeeId; + try { + var compiled = pebble.parseTemplate(cfg.employeeId()); + employeeId = pebble.evaluateAsString(compiled, context.templateContext()); + } catch (Exception e) { + return StepResult.failed("write_memory employeeId template failed for step '" + + step.name() + "': " + e.getMessage()); + } + + MemoryWriter.Result result = memoryWriter.write( + context.workspaceId(), employeeId, cfg.file(), cfg.mergeStrategy(), rendered); + if (!result.success()) { + return StepResult.failed(result.errorMessage()); + } + + String payloadUri = payloadStore.storeString(context.workspaceId(), rendered, "text/markdown"); + return StepResult.succeeded(payloadUri, "text", rendered, result.summary()); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/service/DefaultWorkflowAclPort.java b/mateclaw-server/src/main/java/vip/mate/workflow/service/DefaultWorkflowAclPort.java new file mode 100644 index 00000000..e1cc97de --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/service/DefaultWorkflowAclPort.java @@ -0,0 +1,79 @@ +package vip.mate.workflow.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.springframework.stereotype.Component; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.repository.ChannelMapper; +import vip.mate.workflow.compiler.WorkflowAclPort; + +/** + * Production binding for {@link WorkflowAclPort}. Reads agents from + * {@code mate_agent}, channels from {@code mate_channel}, and treats every + * non-blank {@code employeeId} as a workspace member — until a real + * "human employee" registry exists in the system, the workflow's + * {@code employeeId} is interpreted as the agent id of the agent that owns + * the memory file. + */ +@Component +public class DefaultWorkflowAclPort implements WorkflowAclPort { + + private final AgentMapper agentMapper; + private final ChannelMapper channelMapper; + + public DefaultWorkflowAclPort(AgentMapper agentMapper, ChannelMapper channelMapper) { + this.agentMapper = agentMapper; + this.channelMapper = channelMapper; + } + + @Override + public boolean agentExists(long workspaceId, String agentName) { + if (agentName == null || agentName.isBlank()) return false; + // Workspace-scoped lookup. Without this clause a workflow in + // workspace A could reference an agent that lives in workspace B, + // which would silently bypass the per-workspace ACL the rest of + // the platform enforces. Reject cross-workspace agent references + // at publish time so the failure is visible to authors instead of + // surfacing as a runtime "agent not found". + Long count = agentMapper.selectCount(new LambdaQueryWrapper() + .eq(AgentEntity::getWorkspaceId, workspaceId) + .eq(AgentEntity::getName, agentName.trim()) + .eq(AgentEntity::getEnabled, true)); + return count != null && count > 0; + } + + @Override + public boolean agentIdExists(long workspaceId, long agentId) { + AgentEntity row = agentMapper.selectById(agentId); + return row != null + && Boolean.TRUE.equals(row.getEnabled()) + && row.getWorkspaceId() != null + && row.getWorkspaceId() == workspaceId; + } + + @Override + public boolean channelAllowed(long workspaceId, String channelName) { + if (channelName == null || channelName.isBlank()) return false; + // Same workspace constraint as above: a channel adapter enabled in + // another workspace should not satisfy this workflow's allowlist. + Long count = channelMapper.selectCount(new LambdaQueryWrapper() + .eq(ChannelEntity::getWorkspaceId, workspaceId) + .eq(ChannelEntity::getChannelType, channelName.trim()) + .eq(ChannelEntity::getEnabled, true)); + return count != null && count > 0; + } + + @Override + public boolean employeeInWorkspace(long workspaceId, String employeeId) { + if (employeeId == null || employeeId.isBlank()) return false; + try { + long parsed = Long.parseLong(employeeId); + return agentIdExists(workspaceId, parsed); + } catch (NumberFormatException e) { + // Non-numeric employeeId — let the runtime fail loudly rather + // than silently passing publish-time ACL. + return false; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/workflow/service/WorkflowService.java b/mateclaw-server/src/main/java/vip/mate/workflow/service/WorkflowService.java new file mode 100644 index 00000000..8fa5592e --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workflow/service/WorkflowService.java @@ -0,0 +1,183 @@ +package vip.mate.workflow.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import vip.mate.workflow.compiler.PublishContext; +import vip.mate.workflow.compiler.WorkflowAclPort; +import vip.mate.workflow.compiler.WorkflowCompiler; +import vip.mate.workflow.model.WorkflowEntity; +import vip.mate.workflow.model.WorkflowRevisionEntity; +import vip.mate.workflow.repository.WorkflowMapper; +import vip.mate.workflow.repository.WorkflowRevisionMapper; + +import java.time.LocalDateTime; +import java.util.List; + +/** + * Workflow CRUD + draft / publish lifecycle. Drafts live inline on the + * {@code mate_workflow} row; publishing compiles the draft and writes a + * fresh row into {@code mate_workflow_revision} with a monotonically + * increasing per-workflow revision number, then atomically points + * {@code latest_revision_id} at it. + */ +@Service +@RequiredArgsConstructor +public class WorkflowService { + + private final WorkflowMapper workflowMapper; + private final WorkflowRevisionMapper revisionMapper; + private final WorkflowCompiler compiler; + private final WorkflowAclPort aclPort; + + public List listByWorkspace(long workspaceId) { + return workflowMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowEntity::getWorkspaceId, workspaceId) + .orderByDesc(WorkflowEntity::getUpdateTime)); + } + + /** + * Workspace-scoped lookup. All read paths that take a raw {@code id} + * must use this so callers can't fetch a row from another tenant just + * by guessing a numeric id. Returns {@code null} when the row exists + * but lives in a different workspace (treated as "not found" so the + * caller doesn't get a side-channel signal that the id is real). + */ + public WorkflowEntity get(long id, long workspaceId) { + WorkflowEntity row = workflowMapper.selectById(id); + if (row == null) return null; + if (row.getWorkspaceId() == null || row.getWorkspaceId() != workspaceId) return null; + return row; + } + + /** + * Same as {@link #get(long, long)} but throws when the row is missing. + * Used by mutation paths that can fail loudly instead of returning null. + */ + private WorkflowEntity getOrThrow(long id, long workspaceId) { + WorkflowEntity row = get(id, workspaceId); + if (row == null) { + throw new IllegalArgumentException("workflow not found: " + id); + } + return row; + } + + @Transactional + public WorkflowEntity create(WorkflowEntity workflow) { + if (workflow.getEnabled() == null) workflow.setEnabled(true); + workflowMapper.insert(workflow); + return workflow; + } + + /** + * Update workflow metadata (name / description / enabled). The patch + * shape is deliberately narrow: the caller cannot replace + * {@code draftJson}, {@code latest_revision_id}, or {@code workspace_id} + * through this path. Without that narrowing, a metadata-only save from + * the UI would clobber the draft because the request body wouldn't + * carry it. + */ + @Transactional + public WorkflowEntity updateMetadata(long id, long workspaceId, String name, + String description, Boolean enabled) { + WorkflowEntity existing = getOrThrow(id, workspaceId); + if (name != null) existing.setName(name); + if (description != null) existing.setDescription(description); + if (enabled != null) existing.setEnabled(enabled); + // draftJson / latest_revision_id / workspace_id are intentionally + // left untouched here — those move only through saveDraft / publish. + workflowMapper.updateById(existing); + return existing; + } + + @Transactional + public WorkflowEntity saveDraft(long id, long workspaceId, String draftJson, Long updatedBy) { + WorkflowEntity row = getOrThrow(id, workspaceId); + row.setDraftJson(draftJson); + row.setDraftUpdatedAt(LocalDateTime.now()); + row.setDraftUpdatedBy(updatedBy); + workflowMapper.updateById(row); + return row; + } + + @Transactional + public void delete(long id, long workspaceId) { + getOrThrow(id, workspaceId); + workflowMapper.deleteById(id); + } + + /** + * Compile the workflow's current draft and persist it as a new revision + * pointed at by {@code latest_revision_id}. Throws + * {@link vip.mate.workflow.compiler.WorkflowCompileFailedException} when + * the compiler reports any errors. + */ + @Transactional + public PublishOutcome publish(long workflowId, long workspaceId, Long publisherId, String publishedNote) { + // Row-lock the workflow for the entire publish transaction so two + // concurrent publishes serialize on the same monotonic next revision + // — without this, both compute max+1 and the second one trips + // uk_workflow_revision while leaving the latest_revision_id pointer + // ambiguous. + WorkflowEntity workflow = workflowMapper.selectByIdForUpdate(workflowId); + if (workflow == null) { + throw new IllegalArgumentException("workflow not found: " + workflowId); + } + if (workflow.getWorkspaceId() == null || workflow.getWorkspaceId() != workspaceId) { + // Cross-workspace publish attempt — same surface as "not found" + // so the caller can't probe id existence by error message. + throw new IllegalArgumentException("workflow not found: " + workflowId); + } + String draft = workflow.getDraftJson(); + if (draft == null || draft.isBlank()) { + throw new IllegalStateException("cannot publish workflow " + workflowId + + " without a draft"); + } + // PublishContext is (workspaceId, publisherId) — mind the order. + // ACL validators read ctx.workspaceId() to scope agent / channel / + // employee resolution; passing the publisherId in that slot + // would silently let cross-workspace references through. + PublishContext ctx = new PublishContext(workflow.getWorkspaceId(), + publisherId == null ? 0L : publisherId); + WorkflowCompiler.Result compileResult = compiler.compile(draft, ctx, aclPort); + compileResult.requireOk(); + + int nextRevision = nextRevisionNumber(workflowId); + WorkflowRevisionEntity revision = new WorkflowRevisionEntity(); + revision.setWorkflowId(workflowId); + revision.setRevision(nextRevision); + revision.setGraphJson(draft); + revision.setSchemaVersion(compileResult.graph().schemaVersion() == null + ? "1.0" : compileResult.graph().schemaVersion()); + revision.setPublishedNote(publishedNote); + revision.setPublishedBy(publisherId); + revisionMapper.insert(revision); + + workflow.setLatestRevisionId(revision.getId()); + // RFC v0 contract: publishing clears the inline draft on the + // workflow row. The published revision is now the canonical + // graph; keeping the draft would let the UI show "draft + v3" + // when in fact the draft has just become v3, which confuses + // operators ("did my changes go in?"). Authors who want a + // continuing-edit flow can re-save a fresh draft after publish; + // it'll show up as "draft modified after publish" naturally. + workflow.setDraftJson(null); + workflow.setDraftSchemaVersion(null); + workflow.setDraftUpdatedBy(null); + workflow.setDraftUpdatedAt(null); + workflowMapper.updateById(workflow); + return new PublishOutcome(workflow, revision); + } + + private int nextRevisionNumber(long workflowId) { + WorkflowRevisionEntity max = revisionMapper.selectOne(new LambdaQueryWrapper() + .eq(WorkflowRevisionEntity::getWorkflowId, workflowId) + .orderByDesc(WorkflowRevisionEntity::getRevision) + .last("LIMIT 1")); + return max == null ? 1 : max.getRevision() + 1; + } + + /** Snapshot returned to controllers after a successful publish. */ + public record PublishOutcome(WorkflowEntity workflow, WorkflowRevisionEntity revision) {} +} diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java index 896bd77c..2883ac5f 100644 --- a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/ConversationService.java @@ -1,16 +1,27 @@ package vip.mate.workspace.conversation; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.conditions.update.LambdaUpdateWrapper; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationEventPublisher; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.transaction.support.TransactionSynchronization; +import org.springframework.transaction.support.TransactionSynchronizationManager; import vip.mate.agent.model.AgentEntity; import vip.mate.approval.ApprovalPlaceholderUtil; import vip.mate.approval.MetadataDecision; import vip.mate.agent.repository.AgentMapper; +import vip.mate.approval.model.ToolApprovalEntity; +import vip.mate.approval.repository.ToolApprovalMapper; +import vip.mate.channel.model.ChannelSessionEntity; +import vip.mate.channel.repository.ChannelSessionMapper; +import vip.mate.task.model.AsyncTaskEntity; +import vip.mate.task.repository.AsyncTaskMapper; +import vip.mate.workspace.conversation.event.ConversationDeletedEvent; import vip.mate.workspace.conversation.model.ConversationEntity; import vip.mate.workspace.conversation.model.MessageContentPart; import vip.mate.workspace.conversation.model.MessageEntity; @@ -49,6 +60,23 @@ public class ConversationService { private final MessageMapper messageMapper; private final AgentMapper agentMapper; private final ObjectMapper objectMapper; + private final ToolApprovalMapper toolApprovalMapper; + private final AsyncTaskMapper asyncTaskMapper; + private final ChannelSessionMapper channelSessionMapper; + private final ApplicationEventPublisher eventPublisher; + + /** + * Optional spill store. Injected via a setter so the existing @RequiredArgsConstructor + * stays stable and tests that build the service directly don't need to wire + * tool-result storage. When present, deleteConversation also purges any spill + * files this conversation produced so they don't outlive the row that owned them. + */ + private vip.mate.agent.graph.executor.ToolResultStorage toolResultStorage; + + @org.springframework.beans.factory.annotation.Autowired(required = false) + public void setToolResultStorage(vip.mate.agent.graph.executor.ToolResultStorage toolResultStorage) { + this.toolResultStorage = toolResultStorage; + } /** * 获取用户的会话列表(返回 VO,包含 agentName/agentIcon/status) @@ -359,6 +387,30 @@ public class ConversationService { .orderByAsc(MessageEntity::getId)); } + /** + * Returns the most recent compression boundary row for the conversation, + * or {@code null} if no boundary exists yet. Used by the agent loader to + * recover the structured summary when the boundary itself sits outside the + * recent-message window — without this, a long conversation that already + * compacted would feed the model the last N raw messages while silently + * dropping the goal / progress digest the boundary holds. + * + *

Implemented as a single indexed query rather than a full + * {@code listMessages} + filter so it stays cheap on conversations with + * thousands of messages. Selection: {@code role=system} + + * {@code metadata like '%compression_summary%'} (the metadata column always + * carries that literal — see {@link #saveCompressionSummary}). + */ + public MessageEntity findLatestCompressionBoundary(String conversationId) { + return messageMapper.selectOne(new LambdaQueryWrapper() + .eq(MessageEntity::getConversationId, conversationId) + .eq(MessageEntity::getRole, "system") + .like(MessageEntity::getMetadata, "compression_summary") + .orderByDesc(MessageEntity::getCreateTime) + .orderByDesc(MessageEntity::getId) + .last("LIMIT 1")); + } + /** * 加载最近 N 条消息(倒序取出后翻转为正序)。 * 利用复合索引 (conversation_id, create_time) 高效分页。 @@ -400,18 +452,108 @@ public class ConversationService { } /** - * 将压缩摘要持久化为 role=system 的特殊消息。 - * 下次加载历史时识别此消息,跳过它之前的已压缩消息。 + * Persist a compaction boundary as a role=system message. The body is + * the summary text; the metadata describes what happened at + * this boundary (trigger, pre/post tokens, how many messages were + * summarised, how many spill files were produced, how many tail + * messages survived). On the next load this row is the cut-off — older + * messages are skipped, the model picks up from the summary forward. + * + *

Backward-compat overload: legacy callers that only know the row + * count still work and produce a minimal metadata block. */ public void saveCompressionSummary(String conversationId, String summary, int compressedCount) { + saveCompressionSummary(conversationId, summary, compressedCount, Map.of()); + } + + /** + * Same as {@link #saveCompressionSummary(String, String, int, Map)} but + * returns the inserted row's id so callers (notably + * {@code ConversationWindowManager}) can include the {@code summaryId} + * in the {@code compact_status} SSE payload. The id is also written back + * into the row's metadata JSON by the underlying overload, so the row is + * still self-describing if a client misses the SSE event and loads + * history later. + * + *

Returns {@code null} when the insert path failed (logged at INFO); + * callers should treat that as "no boundary was persisted" and still + * broadcast a {@code done} event without {@code summaryId}. + */ + public Long saveCompressionSummaryReturningId(String conversationId, String summary, + int compressedCount, Map extraMetadata) { + return saveCompressionSummaryInternal(conversationId, summary, compressedCount, extraMetadata); + } + + /** + * Same as the 3-arg overload but accepts extra structured fields that + * are merged into the boundary's metadata JSON. Fields the frontend + * and observability pipeline care about: + *

    + *
  • {@code trigger} — what fired this boundary + * ({@code token_threshold}, {@code user_compact}, etc.)
  • + *
  • {@code preTokens} / {@code postTokens} — context size before + * and after, for the in-prompt status row
  • + *
  • {@code messagesSummarized} / {@code tailKept} — partition + * counts the user sees in the boundary card
  • + *
  • {@code toolResultsSpilled} — how many bodies the spill store + * absorbed during this boundary
  • + *
  • {@code summaryId} — stable id (the inserted message id) for + * deep-linking from the SSE event
  • + *
+ *

{@code type=compression_summary} is always present — the loader + * keys off it. {@code compressedCount} is kept for backward compat. + */ + public void saveCompressionSummary(String conversationId, String summary, int compressedCount, + Map extraMetadata) { + saveCompressionSummaryInternal(conversationId, summary, compressedCount, extraMetadata); + } + + private Long saveCompressionSummaryInternal(String conversationId, String summary, int compressedCount, + Map extraMetadata) { MessageEntity entity = new MessageEntity(); entity.setConversationId(conversationId); entity.setRole("system"); entity.setContent(summary); entity.setStatus("completed"); - entity.setMetadata("{\"type\":\"compression_summary\",\"compressedCount\":" + compressedCount + "}"); + + Map metadata = new java.util.LinkedHashMap<>(); + metadata.put("type", "compression_summary"); + metadata.put("compressedCount", compressedCount); + if (extraMetadata != null) { + extraMetadata.forEach((k, v) -> { + if (v != null) metadata.put(k, v); + }); + } + // First write a placeholder so the row lands with the structured + // fields; we backfill summaryId in a second step once MyBatis Plus + // has assigned the snowflake id. ASSIGN_ID actually populates the + // id BEFORE flushing the INSERT, but reading it back this way means + // the contract holds even if the ID generation strategy changes. + try { + entity.setMetadata(objectMapper.writeValueAsString(metadata)); + } catch (com.fasterxml.jackson.core.JsonProcessingException e) { + log.warn("[Conversation] Failed to serialise compaction metadata, falling back to minimal: {}", + e.getMessage()); + entity.setMetadata("{\"type\":\"compression_summary\",\"compressedCount\":" + compressedCount + "}"); + } messageMapper.insert(entity); - log.info("[Conversation] Saved compression summary for conv={}, compressedCount={}", conversationId, compressedCount); + + // Backfill summaryId now that the row owns an id. Best-effort: a + // failure here doesn't invalidate the boundary itself, it just + // means SSE clients won't have a deep-link target for this row. + if (entity.getId() != null) { + metadata.put("summaryId", entity.getId()); + try { + entity.setMetadata(objectMapper.writeValueAsString(metadata)); + messageMapper.updateById(entity); + } catch (Exception e) { + log.warn("[Conversation] Failed to backfill summaryId on compression boundary: {}", + e.getMessage()); + } + } + log.info("[Conversation] Saved compression boundary conv={}, compressedCount={}, metadata={}", + conversationId, compressedCount, entity.getMetadata()); + return entity.getId(); } public List listMessageViews(String conversationId) { @@ -421,15 +563,97 @@ public class ConversationService { } /** - * 删除会话(同时删除消息和附件文件) + * Delete a conversation and cascade-clean every row that referenced it. + *

+ * Tables cleaned in the same transaction: + *

    + *
  • {@code mate_message} — chat history
  • + *
  • {@code mate_tool_approval} — pending approvals would otherwise + * point to a non-existent conversation and surface as ghost items + * in the approvals list
  • + *
  • {@code mate_async_task} — long-running task records keyed on + * this conversation
  • + *
  • {@code mate_channel_session} — channel-side session row (the + * column is UNIQUE; leaving it would block reuse of the same id)
  • + *
  • {@code mate_conversation} — the conversation itself
  • + *
+ * Child conversations (delegated turns) have their + * {@code parent_conversation_id} set to NULL rather than cascade-deleted, + * so the user keeps independent access to delegated work. + *

+ * Audit / history tables ({@code mate_tool_guard_audit_log}, + * {@code mate_cron_job_run}, {@code mate_skill.source_conversation_id}, + * {@code mate_skill_usage_stat}) are intentionally left alone — those + * are append-only records that should outlive their source conversation. + *

+ * Attachment file cleanup is registered as an after-commit hook so it + * runs only when the DB cascade actually persists, and an IO failure + * cannot roll back the database deletes. */ @Transactional public void deleteConversation(String conversationId) { - conversationMapper.delete(new LambdaQueryWrapper() - .eq(ConversationEntity::getConversationId, conversationId)); - messageMapper.delete(new LambdaQueryWrapper() + int messages = messageMapper.delete(new LambdaQueryWrapper() .eq(MessageEntity::getConversationId, conversationId)); - cleanAttachmentFiles(conversationId); + int approvals = toolApprovalMapper.delete(new LambdaQueryWrapper() + .eq(ToolApprovalEntity::getConversationId, conversationId)); + int asyncTasks = asyncTaskMapper.delete(new LambdaQueryWrapper() + .eq(AsyncTaskEntity::getConversationId, conversationId)); + int channelSessions = channelSessionMapper.delete(new LambdaQueryWrapper() + .eq(ChannelSessionEntity::getConversationId, conversationId)); + int childrenUnlinked = conversationMapper.update(null, new LambdaUpdateWrapper() + .set(ConversationEntity::getParentConversationId, null) + .eq(ConversationEntity::getParentConversationId, conversationId)); + int conversations = conversationMapper.delete(new LambdaQueryWrapper() + .eq(ConversationEntity::getConversationId, conversationId)); + + log.info("[Conversation] Deleted {}: messages={}, approvals={}, asyncTasks={}," + + " channelSessions={}, childrenUnlinked={}, conversationRow={}", + conversationId, messages, approvals, asyncTasks, + channelSessions, childrenUnlinked, conversations); + + registerPostCommitCleanup(conversationId); + } + + /** + * After-commit cleanup: file IO and the {@link ConversationDeletedEvent} + * fan-out both run only if the cascade actually persists, and an IO + * failure cannot roll back the DB cascade. The event lets approval and + * async-task modules drop their in-memory state (pendingMap, active + * pollers, canceled-conv set) so workers cannot resurrect orphan rows + * after the conversation row is gone. + */ + private void registerPostCommitCleanup(String conversationId) { + if (TransactionSynchronizationManager.isSynchronizationActive()) { + TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() { + @Override + public void afterCommit() { + cleanAttachmentFiles(conversationId); + purgeToolResultSpill(conversationId); + eventPublisher.publishEvent(new ConversationDeletedEvent(conversationId)); + } + }); + } else { + cleanAttachmentFiles(conversationId); + purgeToolResultSpill(conversationId); + eventPublisher.publishEvent(new ConversationDeletedEvent(conversationId)); + } + } + + /** + * Best-effort: ask the spill store to delete every tool-result file this + * conversation produced. No-op when no spill store is wired in (legacy + * deployments or tests that don't need spill). Failures are logged but + * never propagated — leaving an extra file on disk is a small price + * compared to surfacing IO errors as a 500 on the delete endpoint. + */ + private void purgeToolResultSpill(String conversationId) { + if (toolResultStorage == null) return; + try { + toolResultStorage.purgeConversation(conversationId); + } catch (Exception e) { + log.warn("[Conversation] tool-result spill purge failed for {}: {}", + conversationId, e.getMessage()); + } } /** @@ -478,6 +702,7 @@ public class ConversationService { case "text" -> appendSegment(text, part.getText()); case "thinking", "tool_call", "parse_error" -> { /* skip — frontend reads these from contentParts directly */ } case "file" -> appendSegment(text, renderFilePart(part)); + case "image", "video", "audio", "model3d" -> appendSegment(text, renderMediaPart(part)); default -> appendSegment(text, part.getText()); } } @@ -536,6 +761,36 @@ public class ConversationService { return "[附件] " + name + "(路径: " + path + ")"; } + /** + * Render an image/video/audio/3D-model content part for the LLM prompt. + *

+ * Without this marker, media parts are invisible in the rendered text — the LLM + * sees only the user's accompanying text and has no idea an attachment was sent. + * That fails closed when the multimodal Media injection in {@code BaseAgent} is + * upstream-stripped (model heuristic claims vision but the actual provider drops + * the image), leaving the agent to ask "which image?" for an attachment the user + * already uploaded. The path lets file-reading tools ({@code read_file}, + * {@code extract_document_text}, {@code detect_file_type}) work as a fallback. + */ + private String renderMediaPart(MessageContentPart part) { + String label = switch (part.getType()) { + case "image" -> "[图片]"; + case "video" -> "[视频]"; + case "audio" -> "[音频]"; + case "model3d" -> "[3D 模型]"; + default -> "[附件]"; + }; + String name = safe(part.getFileName()); + if (name.isBlank()) { + name = "未命名"; + } + String path = safe(part.getPath()); + if (path.isBlank()) { + return label + " " + name; + } + return label + " " + name + "(路径: " + path + ")"; + } + private void appendSegment(StringBuilder builder, String text) { String safeText = safe(text); if (safeText.isBlank()) { diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/event/ConversationDeletedEvent.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/event/ConversationDeletedEvent.java new file mode 100644 index 00000000..ccfc6cbb --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/event/ConversationDeletedEvent.java @@ -0,0 +1,17 @@ +package vip.mate.workspace.conversation.event; + +/** + * Fired AFTER {@link vip.mate.workspace.conversation.ConversationService#deleteConversation} + * commits its DB cascade. + *

+ * Subscribers must use this to clean up any in-memory or scheduled state keyed + * on the deleted conversation — e.g. the approval pending map, async-task + * pollers, SSE buffers, anything that survives independently of the DB row. + *

+ * Published from a {@code TransactionSynchronization.afterCommit} hook so that + * a listener observing this event can safely assume the conversation row, + * its messages, and every cascaded associate row are gone. If the transaction + * rolls back, the event is never published. + */ +public record ConversationDeletedEvent(String conversationId) { +} diff --git a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/vo/MessageVO.java b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/vo/MessageVO.java index 37c7381e..111ce8b2 100644 --- a/mateclaw-server/src/main/java/vip/mate/workspace/conversation/vo/MessageVO.java +++ b/mateclaw-server/src/main/java/vip/mate/workspace/conversation/vo/MessageVO.java @@ -41,6 +41,12 @@ public class MessageVO { /** Completion tokens 消耗 */ private Integer completionTokens; + /** Model name actually used to produce this message (e.g. "deepseek-chat"). */ + private String runtimeModel; + + /** Provider id of the runtime model (e.g. "deepseek", "zhipu"). */ + private String runtimeProvider; + private LocalDateTime createTime; private LocalDateTime updateTime; @@ -59,6 +65,8 @@ public class MessageVO { vo.setMetadata(parseMetadataToObject(entity.getMetadata())); vo.setPromptTokens(entity.getPromptTokens()); vo.setCompletionTokens(entity.getCompletionTokens()); + vo.setRuntimeModel(entity.getRuntimeModel()); + vo.setRuntimeProvider(entity.getRuntimeProvider()); vo.setCreateTime(entity.getCreateTime()); vo.setUpdateTime(entity.getUpdateTime()); vo.setContentParts(contentParts); diff --git a/mateclaw-server/src/main/resources/application-mysql.yml b/mateclaw-server/src/main/resources/application-mysql.yml index 9bdd2af4..919d3d94 100644 --- a/mateclaw-server/src/main/resources/application-mysql.yml +++ b/mateclaw-server/src/main/resources/application-mysql.yml @@ -5,7 +5,15 @@ spring: # 自动创建的库用了 server 默认字符集也不会影响数据。要求 DB user 具备 CREATE 权限(默认 root 可)。 # 如果使用受限账号,请提前手工执行: # CREATE DATABASE mateclaw CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; - url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:mateclaw}?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true + # characterEncoding takes a Java NIO charset name (UTF-8), not a MySQL + # server charset name (utf8mb4) — passing utf8mb4 here throws + # UnsupportedEncodingException at driver init. Java's UTF-8 already + # encodes the full Unicode range including supplementary-plane chars + # and emoji, so 4-byte characters travel intact. To make the server + # treat the connection as utf8mb4, force the connection collation via + # connectionCollation=utf8mb4_unicode_ci — that is what prevents the + # `Data truncation: Incorrect string value` errors on emoji/CJK ext. + url: jdbc:mysql://${DB_HOST:localhost}:${DB_PORT:3306}/${DB_NAME:mateclaw}?createDatabaseIfNotExist=true&useUnicode=true&characterEncoding=UTF-8&connectionCollation=utf8mb4_unicode_ci&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true driver-class-name: com.mysql.cj.jdbc.Driver username: ${DB_USERNAME:root} password: ${DB_PASSWORD:mateclaw123} diff --git a/mateclaw-server/src/main/resources/application.yml b/mateclaw-server/src/main/resources/application.yml index 369a4695..77b1c9eb 100644 --- a/mateclaw-server/src/main/resources/application.yml +++ b/mateclaw-server/src/main/resources/application.yml @@ -67,10 +67,19 @@ spring: enabled: ${H2_CONSOLE_ENABLED:false} path: /h2-console - # Spring AI Alibaba (DashScope) - Spring AI Alibaba 1.1.x 配置路径 + # Spring AI Alibaba (DashScope) — Spring AI Alibaba 1.1.x configuration path. + # + # The api-key here is only consumed by Spring AI Alibaba's auto-configured beans + # as a *fallback*. The real source of truth for every provider/key/model is the + # admin UI ("Settings → Models", persisted in mate_model_provider / + # mate_model_config); AgentDashScopeChatModelBuilder resolves the key per-request + # from the provider row first, only falling back to this property when the row + # is incomplete. The placeholder default keeps DashScopeChatAutoConfiguration + # happy at startup when no env var is set (Docker / fresh install) — leave it + # alone unless you know what you're doing. ai: dashscope: - api-key: ${DASHSCOPE_API_KEY:your-dashscope-api-key-here} + api-key: ${DASHSCOPE_API_KEY:configure-in-admin-ui} chat: options: model: qwen-max @@ -171,6 +180,14 @@ mateclaw: enabled: true failure-threshold: 3 cooldown-ms: 300000 + # Multi-agent delegation (DelegateAgentTool). + delegation: + # Wall-clock budget for one delegateParallel batch (shared across all + # children — they run concurrently on virtual threads, so this is total + # latency, not per-child). 300 s headroom is needed because thinking + # models (Kimi / GLM / MiniMax) routinely take 90–290 s per LLM turn + # when the child must produce multi-section structured output. + parallel-timeout-seconds: 300 # MateClaw Agent 配置 mate: @@ -190,15 +207,17 @@ mate: per-category: shell: 120 web: 30 - # RFC-008 Phase 3: tool-result three-layer budget (per-result spill + per-turn aggregate budget). - # Layer 1 (per-tool cap) lives inside individual tools; Layer 2 spills oversized - # single results to disk; Layer 3 enforces an aggregate cap on the combined - # response size of one tool turn. The full output is preserved on disk and - # the in-context preview points the agent at the spill file (read_file tool). + # Tool-result budget (per-result spill + per-turn aggregate budget). + # The executor tries to spill the RAW result first so the full output is + # preserved on disk; the in-context preview points the agent at the spill + # file via read_file. When spill is disabled, the tool is on the exclusion + # list, the body is at or below the threshold, or the disk write fails, + # the executor falls back to inline hard-truncation to the same character + # cap. Per-turn aggregate caps the combined size across one tool turn. tool-result: enabled: true - per-result-threshold-chars: 16000 # was 4000 — prevents WebSearch spill-to-disk - per-turn-budget-chars: 32000 # was 16000 — headroom for multi-tool turns + per-result-threshold-chars: 8000 # aligned with executor hard cap; > this size → spill, ≤ → inline verbatim + per-turn-budget-chars: 32000 # headroom for multi-tool turns preview-head-chars: 800 excluded-tool-inline-chars: 2500 storage-base-dir: "" @@ -210,6 +229,14 @@ mate: excluded-tools: - read_file - read_workspace_memory_file + # Spill files are deleted after this many days. Default 0 disables the + # scheduled sweep entirely so a summary/preview that points at a spill + # path stays valid for the whole life of the conversation. Files are + # still purged when the conversation is deleted explicitly via + # ConversationService.deleteConversation. Raise to a positive value if + # disk pressure outweighs recoverability for your deployment. + retention-days: 0 + cleanup-cron: "0 0 3 * * ?" conversation: window: # 测试时临时调低:2000 token ≈ 2000 中文字,3 轮对话即可触发压缩 diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index 320e533f..3564ff6f 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -5,26 +5,26 @@ MERGE INTO mate_user (id, username, password, nickname, role, enabled, create_ti KEY (id) VALUES (1, 'admin', '$2a$10$7JB720yubVSZvUI0rEqK/.VqGOZTH.ulu33dHOiBE8ByOhJIrdAu2', 'MateClaw Admin', 'admin', TRUE, NOW(), NOW(), 0); --- Default Agent: General Assistant (ReAct mode) +-- Default digital employee: General Assistant (ReAct mode) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000001, 'MateClaw Assistant', 'Default AI assistant with ReAct mode and tool calling', 'react', - 'You are MateClaw, an intelligent AI assistant. You can help users answer questions, analyze data, and execute tasks. Please respond professionally and in a friendly manner.', +VALUES (1000000001, 'General Assistant', 'All-purpose helper for day-to-day questions, data analysis, and tool calling', 'react', + 'You are MateClaw''s General Assistant. You can help users answer questions, analyze data, and call tools to get things done. Please respond professionally and in a friendly manner.', NULL, 100, TRUE, 'pi:robot-face-happy', 'default,assistant', NOW(), NOW(), 0); --- Default Agent: Task Planner (Plan-Execute mode) +-- Default digital employee: Task Planner (Plan-Execute mode) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000002, 'Task Planner', 'Task planning assistant for complex multi-step tasks', 'plan_execute', - 'You are a professional task planning and execution assistant. You excel at breaking complex goals into executable steps and completing them systematically.', +VALUES (1000000002, 'Task Planner', 'Breaks complex goals into executable steps and drives them forward to completion', 'plan_execute', + 'You are a professional Task Planner. You excel at breaking complex goals into executable steps and completing them systematically.', NULL, 100, TRUE, 'pi:clipboard-note', 'planning,task', NOW(), NOW(), 0); --- StateGraph ReAct Agent (StateGraph architecture) +-- Default digital employee: Reasoning Analyst (explicit reasoning loops + tool calling) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000003, 'StateGraph ReAct', 'StateGraph-based ReAct Agent with explicit reasoning loops and tool calling', 'react', - 'You are an intelligent assistant based on the StateGraph architecture. You can use tools to help users solve problems. Please respond professionally and in a friendly manner.', - NULL, 100, TRUE, 'pi:cpu', 'react,stategraph,tools', NOW(), NOW(), 0); +VALUES (1000000003, 'Reasoning Analyst', 'Thinks step by step with visible reasoning, ideal for problems that need thorough deliberation', 'react', + 'You are a Reasoning Analyst, an assistant that excels at deep reasoning. When facing a problem, first think through it step by step with a clear reasoning trace, then call tools or give the answer. Please respond professionally and in a friendly manner.', + NULL, 100, TRUE, 'pi:cpu', 'react,reasoning,tools', NOW(), NOW(), 0); -- ==================== Local Model Providers (displayed first) ==================== @@ -50,6 +50,13 @@ MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, a KEY (provider_id) VALUES ('dashscope', 'DashScope', 'sk-', 'DashScopeChatModel', '', '', '{}', FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, NOW(), NOW()); +-- DashScope OpenAI-compatible endpoint: shares the same sk- key as the +-- dashscope provider but routes to compatible-mode/v1. Dot-versioned qwen +-- families (qwen3.5-*, qwen3.6-*) are only callable here. +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ('dashscope-compat', 'DashScope (OpenAI-compatible)', 'sk-', 'OpenAIChatModel', '', 'https://dashscope.aliyuncs.com/compatible-mode/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); + MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) KEY (provider_id) VALUES ('modelscope', 'ModelScope', 'ms', 'OpenAIChatModel', '', 'https://api-inference.modelscope.cn/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); @@ -185,11 +192,18 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000103, 'DeepSeek-V3.2', 'dashscope', 'deepseek-v3.2', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), -- Note: dotted Qwen3 versions (qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-*) only ship on the -- OpenAI-compatible endpoint. Calling them through DashScope native (text-generation/generation) --- returns 400 InvalidParameter — use the bailian-team OpenAI-compat provider instead. +-- returns 400 InvalidParameter. They are registered under the dashscope-compat provider, which shares +-- the same sk- key but routes to compatible-mode/v1. (1000000173, 'Qwen Long', 'dashscope', 'qwen-long', 'Long-context model with extended context support', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000174, 'Qwen Plus (latest)', 'dashscope', 'qwen-plus-latest', 'Latest stable snapshot of Qwen Plus — auto-updates as Bailian rolls new releases', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000175, 'Qwen Max (latest)', 'dashscope', 'qwen-max-latest', 'Latest stable snapshot of Qwen Max — strongest reasoning capability', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000176, 'Qwen Turbo (latest)', 'dashscope', 'qwen-turbo-latest', 'Latest stable snapshot of Qwen Turbo — low latency, high frequency', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +-- DashScope OpenAI-compat exclusive models (dot-versioned families) — share the same sk- key. +-- Only the -plus variants are seeded; -max / -vl-max are visible in the model market but return +-- 404 for general accounts. Users on a whitelist can add them via Settings → Models manually. +(1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', 'Qwen3.6 Plus flagship — balanced reasoning and speed (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', 'Qwen3.5 Plus (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', 'Qwen3 vision-language Plus — accepts image / video input (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000104, 'Qwen3.5-122B-A10B', 'modelscope', 'Qwen/Qwen3.5-122B-A10B', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000105, 'GLM-5', 'modelscope', 'ZhipuAI/GLM-5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000106, 'Qwen3.5 Plus', 'aliyun-codingplan', 'qwen3.5-plus', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), @@ -482,6 +496,21 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000019, 'DocxRenderTool', 'DOCX Render', 'Render Markdown directly into a .docx and return a one-time download link. In-process Apache POI implementation, no Node.js subprocess; supports headings, bold, lists, tables. Preferred tool for creating new documents.', 'builtin', 'docxRenderTool', '📝', TRUE, TRUE, NOW(), NOW(), 0); +-- Built-in tool: XLSX Render (in-process Apache POI; markdown tables -> multi-sheet workbook) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX Render', 'Render Markdown directly into a .xlsx workbook and return a one-time download link. In-process Apache POI; each # heading becomes a sheet, pipe tables become rows, numeric cells auto-detected.', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0); + +-- Built-in tool: PPTX Render (in-process Apache POI; Marp-style markdown -> .pptx deck) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000021, 'PptxRenderTool', 'PPTX Render', 'Render Marp-style Markdown directly into a .pptx deck and return a one-time download link. In-process Apache POI; --- separates slides, # / ## titles, - bullets, .', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0); + +-- Built-in tool: PDF Render (dual backend: LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000022, 'PdfRenderTool', 'PDF Render', 'Render Markdown into a final-form .pdf and return a one-time download link. Two backends (LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback); supports YAML frontmatter for cover / page header / page footer.', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0); + -- Example MCP Server: Filesystem (see MateClaw docs mcpServers.filesystem) MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/data-mysql-en.sql b/mateclaw-server/src/main/resources/db/data-mysql-en.sql index aeef963f..e5183ead 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-en.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-en.sql @@ -5,25 +5,25 @@ INSERT INTO mate_user (id, username, password, nickname, role, enabled, create_t VALUES (1, 'admin', '$2a$10$7JB720yubVSZvUI0rEqK/.VqGOZTH.ulu33dHOiBE8ByOhJIrdAu2', 'MateClaw Admin', 'admin', TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE username=VALUES(username), password=VALUES(password), nickname=VALUES(nickname), role=VALUES(role), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); --- Default Agent: General Assistant (ReAct mode) +-- Default digital employee: General Assistant (ReAct mode) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000001, 'MateClaw Assistant', 'Default AI assistant with ReAct mode and tool calling', 'react', - 'You are MateClaw, an intelligent AI assistant. You can help users answer questions, analyze data, and execute tasks. Please respond professionally and in a friendly manner.', +VALUES (1000000001, 'General Assistant', 'All-purpose helper for day-to-day questions, data analysis, and tool calling', 'react', + 'You are MateClaw''s General Assistant. You can help users answer questions, analyze data, and call tools to get things done. Please respond professionally and in a friendly manner.', NULL, 100, TRUE, 'pi:robot-face-happy', 'default,assistant', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); --- Default Agent: Task Planner (Plan-Execute mode) +-- Default digital employee: Task Planner (Plan-Execute mode) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000002, 'Task Planner', 'Task planning assistant for complex multi-step tasks', 'plan_execute', - 'You are a professional task planning and execution assistant. You excel at breaking complex goals into executable steps and completing them systematically.', +VALUES (1000000002, 'Task Planner', 'Breaks complex goals into executable steps and drives them forward to completion', 'plan_execute', + 'You are a professional Task Planner. You excel at breaking complex goals into executable steps and completing them systematically.', NULL, 100, TRUE, 'pi:clipboard-note', 'planning,task', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); --- StateGraph ReAct Agent (StateGraph architecture) +-- Default digital employee: Reasoning Analyst (explicit reasoning loops + tool calling) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000003, 'StateGraph ReAct', 'StateGraph-based ReAct Agent with explicit reasoning loops and tool calling', 'react', - 'You are an intelligent assistant based on the StateGraph architecture. You can use tools to help users solve problems. Please respond professionally and in a friendly manner.', - NULL, 100, TRUE, 'pi:cpu', 'react,stategraph,tools', NOW(), NOW(), 0) +VALUES (1000000003, 'Reasoning Analyst', 'Thinks step by step with visible reasoning, ideal for problems that need thorough deliberation', 'react', + 'You are a Reasoning Analyst, an assistant that excels at deep reasoning. When facing a problem, first think through it step by step with a clear reasoning trace, then call tools or give the answer. Please respond professionally and in a friendly manner.', + NULL, 100, TRUE, 'pi:cpu', 'react,reasoning,tools', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); -- ==================== Local Model Providers (displayed first) ==================== @@ -50,6 +50,14 @@ INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, VALUES ('dashscope', 'DashScope', 'sk-', 'DashScopeChatModel', '', '', '{}', FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); +-- DashScope OpenAI-compatible endpoint: shares the same sk- key as the +-- dashscope provider but routes to compatible-mode/v1. Dot-versioned qwen +-- families (qwen3.5-*, qwen3.6-*) are only callable here; the native endpoint +-- returns 400 InvalidParameter for them. +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ('dashscope-compat', 'DashScope (OpenAI-compatible)', 'sk-', 'OpenAIChatModel', '', 'https://dashscope.aliyuncs.com/compatible-mode/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) +ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); + INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) VALUES ('modelscope', 'ModelScope', 'ms', 'OpenAIChatModel', '', 'https://api-inference.modelscope.cn/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); @@ -200,11 +208,18 @@ VALUES (1000000103, 'DeepSeek-V3.2', 'dashscope', 'deepseek-v3.2', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), -- Note: dotted Qwen3 versions (qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-*) only ship on the -- OpenAI-compatible endpoint. Calling them through DashScope native (text-generation/generation) --- returns 400 InvalidParameter — use the bailian-team OpenAI-compat provider instead. +-- returns 400 InvalidParameter. They are registered under the dashscope-compat provider, which shares +-- the same sk- key but routes to compatible-mode/v1. (1000000173, 'Qwen Long', 'dashscope', 'qwen-long', 'Long-context model with extended context support', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000174, 'Qwen Plus (latest)', 'dashscope', 'qwen-plus-latest', 'Latest stable snapshot of Qwen Plus — auto-updates as Bailian rolls new releases', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000175, 'Qwen Max (latest)', 'dashscope', 'qwen-max-latest', 'Latest stable snapshot of Qwen Max — strongest reasoning capability', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000176, 'Qwen Turbo (latest)', 'dashscope', 'qwen-turbo-latest', 'Latest stable snapshot of Qwen Turbo — low latency, high frequency', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +-- DashScope OpenAI-compat exclusive models (dot-versioned families) — share the same sk- key. +-- Only the -plus variants are seeded; -max / -vl-max are visible in the model market but return +-- 404 for general accounts. Users on a whitelist can add them via Settings → Models manually. +(1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', 'Qwen3.6 Plus flagship — balanced reasoning and speed (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', 'Qwen3.5 Plus (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', 'Qwen3 vision-language Plus — accepts image / video input (compat-mode only)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000104, 'Qwen3.5-122B-A10B', 'modelscope', 'Qwen/Qwen3.5-122B-A10B', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000105, 'GLM-5', 'modelscope', 'ZhipuAI/GLM-5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000106, 'Qwen3.5 Plus', 'aliyun-codingplan', 'qwen3.5-plus', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), @@ -533,6 +548,21 @@ INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name VALUES (1000000019, 'DocxRenderTool', 'DOCX Render', 'Render Markdown directly into a .docx and return a one-time download link. In-process Apache POI implementation, no Node.js subprocess; supports headings, bold, lists, tables. Preferred tool for creating new documents.', 'builtin', 'docxRenderTool', '📝', TRUE, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); +-- Built-in tool: XLSX Render (in-process Apache POI; markdown tables -> multi-sheet workbook) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX Render', 'Render Markdown directly into a .xlsx workbook and return a one-time download link. In-process Apache POI; each # heading becomes a sheet, pipe tables become rows, numeric cells auto-detected.', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + +-- Built-in tool: PPTX Render (in-process Apache POI; Marp-style markdown -> .pptx deck) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000021, 'PptxRenderTool', 'PPTX Render', 'Render Marp-style Markdown directly into a .pptx deck and return a one-time download link. In-process Apache POI; --- separates slides, # / ## titles, - bullets, .', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + +-- Built-in tool: PDF Render (dual backend: LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000022, 'PdfRenderTool', 'PDF Render', 'Render Markdown into a final-form .pdf and return a one-time download link. Two backends (LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback); supports YAML frontmatter for cover / page header / page footer.', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + -- Example MCP Server: Filesystem (see MateClaw docs mcpServers.filesystem) INSERT INTO mate_mcp_server (id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, enabled, connect_timeout_seconds, read_timeout_seconds, last_status, last_error, diff --git a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql index bc326250..fe3787a8 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql @@ -5,25 +5,25 @@ INSERT INTO mate_user (id, username, password, nickname, role, enabled, create_t VALUES (1, 'admin', '$2a$10$7JB720yubVSZvUI0rEqK/.VqGOZTH.ulu33dHOiBE8ByOhJIrdAu2', 'MateClaw Admin', 'admin', TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE username=VALUES(username), password=VALUES(password), nickname=VALUES(nickname), role=VALUES(role), enabled=VALUES(enabled), update_time=VALUES(update_time), deleted=VALUES(deleted); --- 默认 Agent:通用助手(ReAct 模式) +-- 默认数字员工:通用助手(ReAct 模式) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000001, 'MateClaw Assistant', '默认 AI 助手,基于 ReAct 模式,支持工具调用', 'react', - '你是 MateClaw,一个智能 AI 助手。你可以帮助用户回答问题、分析数据、执行任务。请用中文回复,保持专业、友好的态度。', +VALUES (1000000001, '通用助手', '日常问答、数据分析、工具调用都能搞定的全能助手', 'react', + '你是 MateClaw 的通用助手。你可以帮助用户回答问题、分析数据、调用工具完成任务。请用中文回复,保持专业、友好的态度。', NULL, 100, TRUE, 'pi:robot-face-happy', 'default,assistant', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); --- 默认 Agent:任务规划助手(Plan-Execute 模式) +-- 默认数字员工:任务规划师(Plan-Execute 模式) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000002, 'Task Planner', '任务规划助手,适合复杂多步骤任务', 'plan_execute', - '你是一个专业的任务规划和执行助手。你擅长将复杂目标分解为可执行的步骤,并逐步完成。请用中文回复。', +VALUES (1000000002, '任务规划师', '把复杂目标拆成可执行步骤,逐步推进直到完成', 'plan_execute', + '你是一位专业的任务规划师。你擅长将复杂目标分解为可执行的步骤,并逐步完成。请用中文回复。', NULL, 100, TRUE, 'pi:clipboard-note', 'planning,task', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); --- StateGraph ReAct Agent(支持 StateGraph 架构) +-- 默认数字员工:推理分析师(显式推理循环 + 工具调用) INSERT INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) -VALUES (1000000003, 'StateGraph ReAct', '基于 StateGraph 的 ReAct Agent,支持显式推理循环和工具调用', 'react', - '你是基于 StateGraph 架构的智能助手。你可以使用工具来帮助用户解决问题。请用中文回复,保持专业、友好的态度。', - NULL, 100, TRUE, 'pi:cpu', 'react,stategraph,tools', NOW(), NOW(), 0) +VALUES (1000000003, '推理分析师', '分步思考、推理过程清晰可见,适合需要"想清楚再回答"的问题', 'react', + '你是一位推理分析师,善于深度推理。面对问题时,请先分步思考、清晰呈现推理过程,再调用工具或给出答案。请用中文回复,保持专业、友好的态度。', + NULL, 100, TRUE, 'pi:cpu', 'react,reasoning,tools', NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), description=VALUES(description), agent_type=VALUES(agent_type), system_prompt=VALUES(system_prompt), model_name=VALUES(model_name), max_iterations=VALUES(max_iterations), enabled=VALUES(enabled), icon=VALUES(icon), tags=VALUES(tags), update_time=VALUES(update_time), deleted=VALUES(deleted); -- ==================== 本地模型 Provider(优先展示) ==================== @@ -50,6 +50,13 @@ INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, VALUES ('dashscope', 'DashScope', 'sk-', 'DashScopeChatModel', '', '', '{}', FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); +-- DashScope OpenAI 兼容端点:与 dashscope provider 共用同一把 sk- key,但走 +-- compatible-mode/v1 路径。带点号版本号的 qwen 系列(qwen3.5-*, qwen3.6-*)只在 +-- 这里能调通——native 端点会返回 400 InvalidParameter。 +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ('dashscope-compat', 'DashScope (兼容模式)', 'sk-', 'OpenAIChatModel', '', 'https://dashscope.aliyuncs.com/compatible-mode/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) +ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); + INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) VALUES ('modelscope', 'ModelScope', 'ms', 'OpenAIChatModel', '', 'https://api-inference.modelscope.cn/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()) ON DUPLICATE KEY UPDATE name=VALUES(name), api_key_prefix=VALUES(api_key_prefix), chat_model=VALUES(chat_model), api_key=VALUES(api_key), base_url=VALUES(base_url), generate_kwargs=VALUES(generate_kwargs), is_custom=VALUES(is_custom), is_local=VALUES(is_local), support_model_discovery=VALUES(support_model_discovery), support_connection_check=VALUES(support_connection_check), freeze_url=VALUES(freeze_url), require_api_key=VALUES(require_api_key), update_time=VALUES(update_time); @@ -198,12 +205,18 @@ VALUES (1000000101, 'Qwen3 Max', 'dashscope', 'qwen3-max', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000102, 'Qwen3 235B A22B Thinking', 'dashscope', 'qwen3-235b-a22b-thinking-2507', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000103, 'DeepSeek-V3.2', 'dashscope', 'deepseek-v3.2', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), --- 注意: qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-* 等带点号的版本只在 OpenAI 兼容端点上线, --- DashScope native(text-generation/generation)调用会返回 400 InvalidParameter,请使用 bailian-team 等 OpenAI-compat provider。 +-- 注意: qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-* 等带点号的版本只在 OpenAI 兼容端点上线。 +-- DashScope native(text-generation/generation)调用会返回 400 InvalidParameter。 +-- 这些模型挂在 dashscope-compat provider 下,复用同一把 sk- key 但走 compatible-mode/v1 端点。 (1000000173, 'Qwen Long', 'dashscope', 'qwen-long', '长文本模型,支持超长上下文', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000174, 'Qwen Plus (latest)', 'dashscope', 'qwen-plus-latest', '通义千问 Plus 最新稳定快照,自动跟随官方更新', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000175, 'Qwen Max (latest)', 'dashscope', 'qwen-max-latest', '通义千问 Max 最新稳定快照,最强推理能力', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000176, 'Qwen Turbo (latest)', 'dashscope', 'qwen-turbo-latest', '通义千问 Turbo 最新稳定快照,低延迟、高并发', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +-- DashScope 兼容模式专属模型(点号版本号系列)—— 与 dashscope provider 共用同一把 sk- key。 +-- 仅收录在通用账号上确实可调通的 -plus 版本;-max / -vl-max 在 model market 可见但 API 返回 404。 +(1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', '通义千问 3.6 Plus 旗舰,平衡推理与速度(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', '通义千问 3.5 Plus(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', '通义千问 3 视觉理解 Plus,支持图像、视频输入(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000104, 'Qwen3.5-122B-A10B', 'modelscope', 'Qwen/Qwen3.5-122B-A10B', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000105, 'GLM-5', 'modelscope', 'ZhipuAI/GLM-5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000106, 'Qwen3.5 Plus', 'aliyun-codingplan', 'qwen3.5-plus', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), @@ -531,6 +544,21 @@ INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name VALUES (1000000019, 'DocxRenderTool', 'DOCX 渲染', '将 Markdown 直接渲染为 .docx 并返回一次性下载链接。进程内 Apache POI 实现,无需 Node.js 子进程;支持标题、加粗、列表、表格。新建文档场景的首选工具。', 'builtin', 'docxRenderTool', '📝', TRUE, TRUE, NOW(), NOW(), 0) ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); +-- 内置工具:XLSX 渲染(进程内 Apache POI,从 Markdown 表格生成多 sheet 工作簿) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX 渲染', '将 Markdown 直接渲染为 .xlsx 工作簿并返回一次性下载链接。进程内 Apache POI 实现;每个 # 一级标题生成一个 sheet,竖线表格成为行内容,数字单元格自动识别。', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + +-- 内置工具:PPTX 渲染(进程内 Apache POI,Marp 风格 Markdown 生成 .pptx) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000021, 'PptxRenderTool', 'PPTX 渲染', '将 Marp 风格的 Markdown 直接渲染为 .pptx 演示文稿并返回一次性下载链接。进程内 Apache POI 实现;--- 分页、# / ## 作幻灯片标题、- 作要点、 作演讲者备注。', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + +-- 内置工具:PDF 渲染(双 backend:LibreOffice 子进程优先,进程内 OpenPDF + Flying Saucer 兜底) +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000022, 'PdfRenderTool', 'PDF 渲染', '将 Markdown 渲染为最终交付形态的 .pdf 并返回一次性下载链接。双 backend 自动切换(优先 LibreOffice,不可用时回落到进程内 OpenPDF + Flying Saucer);通过 YAML frontmatter 控制封面、页眉、页脚。', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), tool_type=VALUES(tool_type), bean_name=VALUES(bean_name), icon=VALUES(icon), enabled=VALUES(enabled), builtin=VALUES(builtin), update_time=VALUES(update_time), deleted=VALUES(deleted); + -- 示例 MCP Server:Filesystem(参考 MateClaw 文档中的 mcpServers.filesystem) INSERT INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index 9b1b9641..f0060263 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -5,26 +5,26 @@ MERGE INTO mate_user (id, username, password, nickname, role, enabled, create_ti KEY (id) VALUES (1, 'admin', '$2a$10$7JB720yubVSZvUI0rEqK/.VqGOZTH.ulu33dHOiBE8ByOhJIrdAu2', 'MateClaw Admin', 'admin', TRUE, NOW(), NOW(), 0); --- 默认 Agent:通用助手(ReAct 模式) +-- 默认数字员工:通用助手(ReAct 模式) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000001, 'MateClaw Assistant', '默认 AI 助手,基于 ReAct 模式,支持工具调用', 'react', - '你是 MateClaw,一个智能 AI 助手。你可以帮助用户回答问题、分析数据、执行任务。请用中文回复,保持专业、友好的态度。', +VALUES (1000000001, '通用助手', '日常问答、数据分析、工具调用都能搞定的全能助手', 'react', + '你是 MateClaw 的通用助手。你可以帮助用户回答问题、分析数据、调用工具完成任务。请用中文回复,保持专业、友好的态度。', NULL, 100, TRUE, 'pi:robot-face-happy', 'default,assistant', NOW(), NOW(), 0); --- 默认 Agent:任务规划助手(Plan-Execute 模式) +-- 默认数字员工:任务规划师(Plan-Execute 模式) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000002, 'Task Planner', '任务规划助手,适合复杂多步骤任务', 'plan_execute', - '你是一个专业的任务规划和执行助手。你擅长将复杂目标分解为可执行的步骤,并逐步完成。请用中文回复。', +VALUES (1000000002, '任务规划师', '把复杂目标拆成可执行步骤,逐步推进直到完成', 'plan_execute', + '你是一位专业的任务规划师。你擅长将复杂目标分解为可执行的步骤,并逐步完成。请用中文回复。', NULL, 100, TRUE, 'pi:clipboard-note', 'planning,task', NOW(), NOW(), 0); --- StateGraph ReAct Agent(支持 StateGraph 架构) +-- 默认数字员工:推理分析师(显式推理循环 + 工具调用) MERGE INTO mate_agent (id, name, description, agent_type, system_prompt, model_name, max_iterations, enabled, icon, tags, create_time, update_time, deleted) KEY (id) -VALUES (1000000003, 'StateGraph ReAct', '基于 StateGraph 的 ReAct Agent,支持显式推理循环和工具调用', 'react', - '你是基于 StateGraph 架构的智能助手。你可以使用工具来帮助用户解决问题。请用中文回复,保持专业、友好的态度。', - NULL, 100, TRUE, 'pi:cpu', 'react,stategraph,tools', NOW(), NOW(), 0); +VALUES (1000000003, '推理分析师', '分步思考、推理过程清晰可见,适合需要"想清楚再回答"的问题', 'react', + '你是一位推理分析师,善于深度推理。面对问题时,请先分步思考、清晰呈现推理过程,再调用工具或给出答案。请用中文回复,保持专业、友好的态度。', + NULL, 100, TRUE, 'pi:cpu', 'react,reasoning,tools', NOW(), NOW(), 0); -- ==================== 本地模型 Provider(优先展示) ==================== @@ -50,6 +50,13 @@ MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, a KEY (provider_id) VALUES ('dashscope', 'DashScope', 'sk-', 'DashScopeChatModel', '', '', '{}', FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, NOW(), NOW()); +-- DashScope OpenAI 兼容端点:与 dashscope provider 共用同一把 sk- key,但走 +-- compatible-mode/v1 路径。带点号版本号的 qwen 系列(qwen3.5-*, qwen3.6-*)只在 +-- 这里能调通——native 端点会返回 400 InvalidParameter。 +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ('dashscope-compat', 'DashScope (兼容模式)', 'sk-', 'OpenAIChatModel', '', 'https://dashscope.aliyuncs.com/compatible-mode/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); + MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) KEY (provider_id) VALUES ('modelscope', 'ModelScope', 'ms', 'OpenAIChatModel', '', 'https://api-inference.modelscope.cn/v1', '{}', FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, NOW(), NOW()); @@ -187,12 +194,18 @@ MERGE INTO mate_model_config (id, name, provider, model_name, description, tempe (1000000101, 'Qwen3 Max', 'dashscope', 'qwen3-max', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000102, 'Qwen3 235B A22B Thinking', 'dashscope', 'qwen3-235b-a22b-thinking-2507', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000103, 'DeepSeek-V3.2', 'dashscope', 'deepseek-v3.2', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), --- 注意: qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-* 等带点号的版本只在 OpenAI 兼容端点上线, --- DashScope native(text-generation/generation)调用会返回 400 InvalidParameter,请使用 bailian-team 等 OpenAI-compat provider。 +-- 注意: qwen3-plus / qwen3.5-plus / qwen3.5-max / qwen3.6-* 等带点号的版本只在 OpenAI 兼容端点上线。 +-- DashScope native(text-generation/generation)调用会返回 400 InvalidParameter。 +-- 这些模型挂在 dashscope-compat provider 下,复用同一把 sk- key 但走 compatible-mode/v1 端点。 (1000000173, 'Qwen Long', 'dashscope', 'qwen-long', '长文本模型,支持超长上下文', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000174, 'Qwen Plus (latest)', 'dashscope', 'qwen-plus-latest', '通义千问 Plus 最新稳定快照,自动跟随官方更新', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000175, 'Qwen Max (latest)', 'dashscope', 'qwen-max-latest', '通义千问 Max 最新稳定快照,最强推理能力', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000176, 'Qwen Turbo (latest)', 'dashscope', 'qwen-turbo-latest', '通义千问 Turbo 最新稳定快照,低延迟、高并发', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +-- DashScope 兼容模式专属模型(点号版本号系列)—— 与 dashscope provider 共用同一把 sk- key。 +-- 仅收录在通用账号上确实可调通的 -plus 版本;-max / -vl-max 在 model market 可见但 API 返回 404。 +(1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', '通义千问 3.6 Plus 旗舰,平衡推理与速度(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', '通义千问 3.5 Plus(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), +(1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', '通义千问 3 视觉理解 Plus,支持图像、视频输入(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000104, 'Qwen3.5-122B-A10B', 'modelscope', 'Qwen/Qwen3.5-122B-A10B', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000105, 'GLM-5', 'modelscope', 'ZhipuAI/GLM-5', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), (1000000106, 'Qwen3.5 Plus', 'aliyun-codingplan', 'qwen3.5-plus', '', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), @@ -485,6 +498,21 @@ MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, KEY (id) VALUES (1000000019, 'DocxRenderTool', 'DOCX 渲染', '将 Markdown 直接渲染为 .docx 并返回一次性下载链接。进程内 Apache POI 实现,无需 Node.js 子进程;支持标题、加粗、列表、表格。新建文档场景的首选工具。', 'builtin', 'docxRenderTool', '📝', TRUE, TRUE, NOW(), NOW(), 0); +-- 内置工具:XLSX 渲染(进程内 Apache POI,从 Markdown 表格生成多 sheet 工作簿) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX 渲染', '将 Markdown 直接渲染为 .xlsx 工作簿并返回一次性下载链接。进程内 Apache POI 实现;每个 # 一级标题生成一个 sheet,竖线表格成为行内容,数字单元格自动识别。', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0); + +-- 内置工具:PPTX 渲染(进程内 Apache POI,Marp 风格 Markdown 生成 .pptx) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000021, 'PptxRenderTool', 'PPTX 渲染', '将 Marp 风格的 Markdown 直接渲染为 .pptx 演示文稿并返回一次性下载链接。进程内 Apache POI 实现;--- 分页、# / ## 作幻灯片标题、- 作要点、 作演讲者备注。', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0); + +-- 内置工具:PDF 渲染(双 backend:LibreOffice 子进程优先,进程内 OpenPDF + Flying Saucer 兜底) +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000022, 'PdfRenderTool', 'PDF 渲染', '将 Markdown 渲染为最终交付形态的 .pdf 并返回一次性下载链接。双 backend 自动切换(优先 LibreOffice,不可用时回落到进程内 OpenPDF + Flying Saucer);通过 YAML frontmatter 控制封面、页眉、页脚。', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0); + -- 示例 MCP Server:Filesystem(参考 MateClaw 文档中的 mcpServers.filesystem) MERGE INTO mate_mcp_server ( id, name, description, transport, url, headers_json, command, args_json, env_json, cwd, diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql b/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql new file mode 100644 index 00000000..aa7acd8b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V100__multimodal_default_models.sql @@ -0,0 +1,16 @@ +-- V100: System-level defaults for vision and video sidecar routing. +-- When the agent's primary model lacks the modality required by an attachment, +-- the runtime delegates a single caption call to the model recorded here. +-- Empty value = not configured; the UI then asks the user to pick one. +-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique). +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000002001, 'default.vision_model', '', + 'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality', + NOW(), NOW()); + +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000002002, 'default.video_model', '', + 'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality', + NOW(), NOW()); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V101__cleanup_blank_tool_guard_rule_id.sql b/mateclaw-server/src/main/resources/db/migration/h2/V101__cleanup_blank_tool_guard_rule_id.sql new file mode 100644 index 00000000..49954404 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V101__cleanup_blank_tool_guard_rule_id.sql @@ -0,0 +1,26 @@ +-- Earlier releases let the rule-create API persist a custom guard rule +-- with a blank rule_id because the service skipped the not-blank check. +-- The resulting row was undeletable from the UI: the delete endpoint is +-- /guard/rules/{ruleId}, and a blank path variable produces a 404 instead +-- of resolving to the row. This migration does two things: +-- +-- 1. Purge any orphan rows already persisted on existing installations +-- so users who hit the bug on v1.2.0 can recover without direct DB +-- surgery. NULL, empty, and whitespace-only rule_id are all swept; +-- built-in rules are excluded defensively because they are seeded +-- with stable IDs and should never appear here. +-- +-- 2. Add a CHECK constraint so the database itself rejects blank +-- rule_id going forward. The service-layer guard already prevents +-- this from the UI, but the DB constraint defends against any +-- future code path that bypasses the service (batch import, direct +-- SQL, future endpoints) and makes the invariant explicit at the +-- schema level. + +DELETE FROM mate_tool_guard_rule +WHERE (rule_id IS NULL OR LENGTH(TRIM(rule_id)) = 0) + AND (builtin IS NULL OR builtin = FALSE); + +ALTER TABLE mate_tool_guard_rule + ADD CONSTRAINT ck_tool_guard_rule_id_nonblank + CHECK (rule_id IS NOT NULL AND LENGTH(TRIM(rule_id)) > 0); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V102__agent_unique_name_per_workspace.sql b/mateclaw-server/src/main/resources/db/migration/h2/V102__agent_unique_name_per_workspace.sql new file mode 100644 index 00000000..7f9e3e0a --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V102__agent_unique_name_per_workspace.sql @@ -0,0 +1,35 @@ +-- Enforce unique Agent name within a workspace. +-- +-- Before V102 the application allowed two Agents with the same name in the +-- same workspace, which made name-based routing (e.g. @-mention an Agent in +-- an IM channel) ambiguous and let an attacker shadow an existing Agent. +-- +-- Step 1 — rename pre-existing duplicates so the new index can be created +-- without an offline migration. The oldest row per (workspace_id, name) +-- keeps the original name; later rows are renamed to a fully synthetic +-- migration tag. +-- +-- The rename target intentionally drops the original name and substitutes +-- `__mate_dup_v102____`. Any deterministic transformation of +-- the original name has a non-zero collision risk against a hand-typed +-- pre-existing row that happens to match the pattern (e.g. someone named +-- their agent `foo__v102_dup__2`). A random UUID component drives the +-- collision probability to ~1/2^122, low enough to call "provably unique" +-- for a one-shot admin migration. The original name is recoverable via +-- the audit log; the row id stays embedded in the new name for traceability. +UPDATE mate_agent +SET name = CONCAT('__mate_dup_v102__', id, '__', RANDOM_UUID()) +WHERE id IN ( + SELECT a.id FROM mate_agent a + WHERE EXISTS ( + SELECT 1 FROM mate_agent b + WHERE b.workspace_id = a.workspace_id + AND b.name = a.name + AND b.id < a.id + ) +); + +-- Step 2 — DB-level guarantee. Service layer also pre-checks for friendly +-- 409 messages; this index is the racy-write safety net. +CREATE UNIQUE INDEX IF NOT EXISTS uk_agent_workspace_name + ON mate_agent(workspace_id, name); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V103__drop_fact_entity_ref.sql b/mateclaw-server/src/main/resources/db/migration/h2/V103__drop_fact_entity_ref.sql new file mode 100644 index 00000000..50cd447b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V103__drop_fact_entity_ref.sql @@ -0,0 +1,15 @@ +-- Drop the dead mate_fact_entity_ref table. +-- +-- Introduced in V29 to back a multi-hop "find facts related to entity X" +-- query, but no writer was ever shipped — FactProjectionBuilder only +-- populated mate_fact, never mate_fact_entity_ref. The downstream +-- FactQueryService.related() and the fact_related agent tool therefore +-- always returned empty results, and the table was missing an agent_id +-- column that would have been needed for tenancy isolation if a writer +-- ever did land. Removing the empty table + the dead Java code (deleted +-- in the same change set) keeps the fact projection honest about what +-- it actually offers. +-- +-- If multi-hop fact graph queries become desirable later, add agent_id +-- from day one and ship the writer in the same change. +DROP TABLE IF EXISTS mate_fact_entity_ref; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql b/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql new file mode 100644 index 00000000..2b399048 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V104__wiki_chunk_embedding_text_version.sql @@ -0,0 +1,6 @@ +-- V104: track which input format a chunk's stored embedding was generated against. +-- The embedding input builder concatenates raw title / header breadcrumb / page +-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION +-- forces a re-embed pass without changing the model. NULL is treated as the +-- legacy content-only format and re-embedded lazily on the next pass. +ALTER TABLE mate_wiki_chunk ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql b/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql new file mode 100644 index 00000000..8cbb83e2 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V105__wiki_transformation.sql @@ -0,0 +1,90 @@ +-- Reusable user-defined prompt templates ("transformations") that run over +-- a raw material's extracted text and persist the LLM output as an artifact +-- on the knowledge base. Templates can be flagged apply_default so the +-- ingestion pipeline runs them automatically once a raw material reaches +-- the completed state. Manual / agent-tool runs are also supported. +-- +-- mate_wiki_transformation — the template (prompt + metadata) +-- mate_wiki_transformation_run — one row per execution attempt + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + -- NULL = workspace-wide template available to every KB in the workspace. + -- Non-NULL = pinned to a single KB. + kb_id BIGINT NULL, + + workspace_id BIGINT NOT NULL DEFAULT 1, + + -- Short stable identifier (e.g. "risk-extract"). Used by agent tools to + -- target a transformation without exposing numeric IDs. + name VARCHAR(64) NOT NULL, + + -- Human-readable label shown in the UI. + title VARCHAR(255) NOT NULL, + + description VARCHAR(1024), + + -- Prompt body. Placeholders supported by the executor: + -- {input_text} — extracted text of the source raw material + -- {title} — title of the source raw material + prompt_template CLOB NOT NULL, + + -- When true, the executor fires this transformation automatically for + -- every raw material that reaches completed in the matching KB. + apply_default BOOLEAN NOT NULL DEFAULT FALSE, + + -- Optional explicit model override. NULL = fall back to the KB-bound + -- chat model (same routing chain WikiCompileService uses). + model_id BIGINT NULL, + + enabled BOOLEAN NOT NULL DEFAULT TRUE, + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wtr_kb ON mate_wiki_transformation (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtr_ws ON mate_wiki_transformation (workspace_id, deleted); +CREATE UNIQUE INDEX IF NOT EXISTS uk_wtr_kb_name ON mate_wiki_transformation (kb_id, name, deleted); + + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation_run ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + transformation_id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + -- Either raw_id or page_id is set; input_kind says which. + input_kind VARCHAR(16) NOT NULL, + raw_id BIGINT NULL, + page_id BIGINT NULL, + + -- pending | running | completed | failed + status VARCHAR(16) NOT NULL DEFAULT 'pending', + + -- LLM output. Treat as Markdown unless the prompt asked for JSON. + output CLOB, + + error VARCHAR(2048), + + -- Model that actually produced the output (after routing). + model_id BIGINT NULL, + + -- apply_default | manual | agent_tool + triggered_by VARCHAR(32) NOT NULL DEFAULT 'manual', + + started_at TIMESTAMP NULL, + completed_at TIMESTAMP NULL, + duration_ms BIGINT NULL, + + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_wtrn_tr ON mate_wiki_transformation_run (transformation_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtrn_kb ON mate_wiki_transformation_run (kb_id, deleted); +CREATE INDEX IF NOT EXISTS idx_wtrn_raw ON mate_wiki_transformation_run (raw_id, deleted); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql b/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql new file mode 100644 index 00000000..3fd5f75f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V106__wiki_transformation_output_target.sql @@ -0,0 +1,19 @@ +-- Two-part follow-up to V105 so a transformation's output can flow back +-- into the KB as a first-class artifact: +-- +-- 1. mate_wiki_transformation.output_target — declarative target for the +-- template's output. `none` = legacy behaviour (output stays in the run +-- history only). `page` = after a successful run, persist the output as +-- a synthesis wiki page derived from the source raw material. Runs an +-- upsert against a deterministic slug so re-running is idempotent. +-- +-- 2. mate_wiki_transformation_run.output_page_id — when a run was saved as +-- a page (either via apply_default=page or the manual save-as-page +-- endpoint), this points at mate_wiki_page.id so the UI can render a +-- "saved as: " link without a join through sourceRawIds. + +ALTER TABLE mate_wiki_transformation + ADD COLUMN IF NOT EXISTS output_target VARCHAR(16) NOT NULL DEFAULT 'none'; + +ALTER TABLE mate_wiki_transformation_run + ADD COLUMN IF NOT EXISTS output_page_id BIGINT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql b/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql new file mode 100644 index 00000000..a6a4c7bb --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V107__wiki_page_embedding.sql @@ -0,0 +1,10 @@ +-- Page-level embedding so synthesis pages produced by transformations can be +-- surfaced by semantic search even when their generated content doesn't +-- appear in the source raw's chunks. The retriever combines chunk-level +-- cosine (via sourceRawIds) with these page-level vectors taking the max, +-- so a synthesis page that the LLM authored with vocabulary not present in +-- the original PDF can still match a user's natural-language query. + +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding BLOB DEFAULT NULL; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_model VARCHAR(64) DEFAULT NULL; +ALTER TABLE mate_wiki_page ADD COLUMN IF NOT EXISTS embedding_text_version VARCHAR(32) DEFAULT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V108__wiki_transformation_starter_pack.sql b/mateclaw-server/src/main/resources/db/migration/h2/V108__wiki_transformation_starter_pack.sql new file mode 100644 index 00000000..b87a450f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V108__wiki_transformation_starter_pack.sql @@ -0,0 +1,250 @@ +-- Starter pack: 7 workspace-wide transformation templates aligned with the +-- enterprise scenarios surface (contract review / sales intel / approvals). +-- kb_id = NULL means the template is offered to every KB in workspace 1. +-- Fixed ids in the seed range so future migrations can reference them. +-- Flyway runs this once; user edits to these rows are not clobbered on +-- a future repair pass because Flyway only repairs the schema_history +-- table, not the seeded rows. + +INSERT INTO mate_wiki_transformation + (id, kb_id, workspace_id, name, title, description, prompt_template, + apply_default, model_id, enabled, output_target, create_time, update_time, deleted) +VALUES +(1000004001, NULL, 1, + 'contract-risk-extract', + '合同风险点提取', + '逐条审查合同条款,标注风险等级、原文位置、AI 建议改写。配合企业场景 → 合同审查使用。', +'你是一名企业法务审查员。从下面的合同文本中完整提取所有需要关注的风险条款,按以下结构输出 Markdown: + +## 风险条款清单 + +对每条值得审查的条款,输出三级标题: + +### <条款简称> +- **风险等级**:高 / 中 / 低 +- **条款类型**:赔偿 / 责任限制 / 付款 / 保密 / 竞业 / 终止 / 管辖 / 数据保护 / 其他 +- **原文位置**:第 X 条 / 第 Y 页(材料未标号时写「未标注」) +- **原文摘录**:用「」引用关键句 +- **风险描述**:≤ 50 字说明风险所在 +- **建议改写**:给出可直接采用的修订版本 + +## 总体评估 + +一段话总结这份合同的整体风险水位与签字建议(≤ 200 字)。 + +要求: +- 不要虚构原文没有的条款 +- 数字与条款编号保留原样 +- 中文输出,不要任何客套或元描述 + +合同标题:{title} + +合同正文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004002, NULL, 1, + 'meeting-action-items', + '会议纪要 → 行动项', + '从会议纪要中穷尽抽取决议 + 行动项(owner / 截止日 / 验收标准),适合周会、决策会议。', +'你是会议纪要分析助理。从下面的纪要中穷尽抽取所有行动项与决议,按以下结构输出 Markdown: + +## 决议清单 +按时间或重要性顺序列出每条明确决议;每条 ≤ 一句话。 + +## 行动项清单 + +| 序号 | 行动 | 负责人 | 截止日 | 验收标准 | +|---|---|---|---|---| + +要求: +- 「行动」用动词开头(如「提交」「完成」「对齐」) +- 负责人若未明确写「未指派」 +- 截止日若未明确写「未定」 +- 验收标准一句话写出「做完是什么样」 +- 不要把「讨论了 X」当作行动项 + +## 风险与依赖 +一句话列出会议中提到的潜在阻塞或跨团队依赖(≤ 5 条)。 + +要求:中文,无客套,无元描述。 + +会议主题:{title} + +纪要正文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004003, NULL, 1, + 'customer-profile', + '客户邮件 / 访谈画像', + '把客户邮件、会议纪要、CRM 记录合成一份结构化客户画像,配合企业场景 → 客户情报使用。', +'你是销售情报员。从下面的客户邮件 / CRM 记录 / 访谈中提取一份客户画像,按以下结构输出 Markdown: + +## 客户档案 +- **名称**: +- **行业 / 规模**: +- **当前阶段**:潜在 / 沟通中 / 谈判中 / 已成交(若无明确信号写「未知」) +- **决策链关键人**:列出姓名 + 角色 + 倾向 + +## 痛点与机会 +- 3-5 条关键痛点,每条带原文引用 +- 2-3 条潜在切入点 + +## 异议预判 +列出客户可能的反对意见 + 对应应对话术。 + +## 下一步建议 +- 3 条具体动作,按优先级排序,每条带「为什么现在做」 + +要求:不要发明文本没说的事;不确定时写「未提及」。 + +客户:{title} + +原文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004004, NULL, 1, + 'competitor-update', + '竞品动态摘要', + '把新闻 / 产品 release / 招聘信号 / 客户提及合成一份竞品动态简报。', +'你是市场情报员。从下面的材料中提取与竞争对手相关的动态,按以下结构输出 Markdown: + +## 涉及对手 +列出材料中提到的所有竞品公司或产品。 + +## 关键动态 + +按时间倒序,每条输出: + +### <对手 / 产品> · <动态简称> +- **类型**:新产品 / 招聘 / 融资 / 客户胜出 / 价格调整 / 团队变动 / 其他 +- **原文摘录**:「」引用 +- **来源**:网页 / 邮件 / 新闻渠道 +- **对我们的影响**:威胁 / 机会 / 中性,一句话说明 + +## 战术建议 +3 条针对性的应对动作,按优先级排序。 + +## 监控建议 +列出值得长期追踪的关键词或信号。 + +要求:中文,不发明内容,不确定时跳过。 + +材料:{title} + +原文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004005, NULL, 1, + 'resume-structured-extract', + '简历结构化', + '把简历提取为标准化档案:教育、工作、技能、亮点。适合批量初筛。', +'你是 HR 助理。把下面的简历提取为结构化档案: + +## 候选人信息 +- **姓名**: +- **当前职位**: +- **总工作年限**: +- **专业领域**: + +## 教育经历 + +| 学校 | 学位 / 专业 | 时间 | +|---|---|---| + +## 工作经历 + +按时间倒序,每段输出: + +### <公司> · <职位> · <时间> +- **职责摘要**:≤ 30 字 +- **关键产出**:≤ 3 条 bullet(量化优先) + +## 技能矩阵 + +| 技能 | 熟练度 | +|---|---| + +## 候选人亮点 +一段话归纳最值得关注的 3 件事(≤ 150 字)。 + +要求:不要发明文本没有的经历;不确定写「未提及」;中文输出。注意:不要把性别 / 年龄 / 户籍写进画像。 + +简历:{title} + +原文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004006, NULL, 1, + 'incident-postmortem', + '事故 5-Why 复盘', + '从事故报告 / 时间线生成 5-Why 链 + 整改清单 + 相似事故关键词。适合 SRE / 运维团队。', +'你是 SRE 事故复盘助理。从下面的事故报告 / 时间线中输出 5-Why 分析: + +## 事故概要 +- **现象**:1 句话 +- **影响范围**:用户数 / 系统 / 持续时间 +- **触发时间**: + +## 5 Whys 链 + +1. **现象**:… + **Why?** … +2. **Why?** … +3. **Why?** … +4. **Why?** … +5. **根因 (Why?)** … + +## 整改清单 + +| 序号 | 行动 | 负责团队 | 优先级 | 截止 | +|---|---|---|---|---| + +## 相似事故关联 +列出可能相关的历史事故关键词(用于后续 wiki 检索)。 + +## 复盘要点 +3 条最值得团队记住的教训。 + +要求:中文,技术准确,不发明数据。 + +事故:{title} + +报告: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0), + +(1000004007, NULL, 1, + 'paper-imrad', + '论文 IMRaD 摘要', + '把论文 / 技术报告浓缩为 IMRaD 结构化摘要 + 关键术语表,适合研究型团队。', +'你是学术摘要助理。把下面的论文 / 技术报告浓缩为 IMRaD 结构化摘要: + +## Introduction +解决什么问题,为什么重要(≤ 100 字) + +## Methods +使用什么方法 / 数据 / 模型(≤ 150 字) + +## Results +最重要的 3-5 个量化或定性结果(每条 ≤ 30 字) + +## Discussion +- **主要洞察**:1-2 句 +- **局限性**:1-2 条 +- **可复现性**:高 / 中 / 低,附 1 句理由 + +## 关键术语 +列出 5-8 个核心术语,每个加一句话定义。 + +要求:保留 LaTeX 公式(如有),不发明结果,中文写作。 + +论文:{title} + +原文: +{input_text}', + FALSE, NULL, TRUE, 'page', NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql b/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql new file mode 100644 index 00000000..e1fe5520 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V109__wiki_transformation_output_format.sql @@ -0,0 +1,8 @@ +-- Output format declared on the template so the executor can validate the +-- LLM's response shape. 'markdown' (default) keeps the legacy behaviour +-- where output is treated as Markdown and saved as page content; 'json' +-- asks the LLM for a single JSON object and the executor parses + validates +-- before persisting. Future formats (table, yaml) can extend this column. + +ALTER TABLE mate_wiki_transformation + ADD COLUMN IF NOT EXISTS output_format VARCHAR(16) NOT NULL DEFAULT 'markdown'; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V110__wiki_transformation_run_tokens.sql b/mateclaw-server/src/main/resources/db/migration/h2/V110__wiki_transformation_run_tokens.sql new file mode 100644 index 00000000..c00ee60c --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V110__wiki_transformation_run_tokens.sql @@ -0,0 +1,8 @@ +-- Record per-run token usage so operators can see which templates burn the +-- most tokens and which models produce the most expensive output. Spring AI +-- surfaces the values via ChatResponseMetadata.getUsage(); the executor +-- snapshots them into the run row after the LLM call. + +ALTER TABLE mate_wiki_transformation_run ADD COLUMN IF NOT EXISTS input_tokens BIGINT NULL; +ALTER TABLE mate_wiki_transformation_run ADD COLUMN IF NOT EXISTS output_tokens BIGINT NULL; +ALTER TABLE mate_wiki_transformation_run ADD COLUMN IF NOT EXISTS total_tokens BIGINT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V111__wiki_transformation_output_schema.sql b/mateclaw-server/src/main/resources/db/migration/h2/V111__wiki_transformation_output_schema.sql new file mode 100644 index 00000000..cd9cce1d --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V111__wiki_transformation_output_schema.sql @@ -0,0 +1,7 @@ +-- Optional JSON Schema describing the shape the LLM should produce when +-- output_format='json'. The executor injects the schema into the prompt +-- so the model has explicit field/type expectations, and validates the +-- parsed JSON against a lightweight required-fields check after parsing. +-- Stored as TEXT — the schema can be arbitrary JSON Schema text. + +ALTER TABLE mate_wiki_transformation ADD COLUMN IF NOT EXISTS output_schema CLOB DEFAULT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V112__skill_file.sql b/mateclaw-server/src/main/resources/db/migration/h2/V112__skill_file.sql new file mode 100644 index 00000000..bc83eb7f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V112__skill_file.sql @@ -0,0 +1,24 @@ +-- V112: persist skill bundle files (scripts/ + references/) in the database. +-- +-- Until now scripts/references only lived on the local filesystem of whichever +-- node handled the upload. Multi-instance deployments sharing one MySQL would +-- have the skill row visible everywhere but the script files only on one node, +-- so any other node attempting to run a skill script either failed or ran a +-- stale local copy. Treating the database as the canonical bundle store and +-- the filesystem as a materialized cache resolves that gap and matches the +-- existing pattern for SKILL.md (canonical in mate_skill.skill_content, +-- mirrored to disk by the workspace manager). + +CREATE TABLE IF NOT EXISTS mate_skill_file ( + id BIGINT NOT NULL PRIMARY KEY, + skill_id BIGINT NOT NULL, + file_path VARCHAR(512) NOT NULL, + content CLOB, + content_size INT NOT NULL DEFAULT 0, + sha256 CHAR(64), + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS uk_skill_file_path ON mate_skill_file (skill_id, file_path); +CREATE INDEX IF NOT EXISTS idx_skill_file_skill ON mate_skill_file (skill_id); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V91__widen_message_and_skill_content.sql b/mateclaw-server/src/main/resources/db/migration/h2/V91__widen_message_and_skill_content.sql new file mode 100644 index 00000000..42e6fe6e --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V91__widen_message_and_skill_content.sql @@ -0,0 +1,7 @@ +-- V91: Mirror MySQL widening of mate_message.content / content_parts and +-- mate_skill.skill_content. H2's TEXT is already CLOB (effectively unbounded) +-- so the change is a no-op semantically; it keeps both dialects in sync. + +ALTER TABLE mate_message ALTER COLUMN content CLOB; +ALTER TABLE mate_message ALTER COLUMN content_parts CLOB; +ALTER TABLE mate_skill ALTER COLUMN skill_content CLOB; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V92__mcp_server_tools_cache.sql b/mateclaw-server/src/main/resources/db/migration/h2/V92__mcp_server_tools_cache.sql new file mode 100644 index 00000000..36bdc75f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V92__mcp_server_tools_cache.sql @@ -0,0 +1,10 @@ +-- V92: Persist each MCP server's discovered tool list as a per-row JSON +-- snapshot so the agent edit picker can render the tools even when the +-- upstream server is briefly disconnected, and so the per-tool atomic +-- binding flow has a stable place to resolve raw tool names from the +-- prefixed callback name. +-- +-- Idempotent on re-runs (Flyway's repair-on-startup applies). + +ALTER TABLE mate_mcp_server ADD COLUMN IF NOT EXISTS tools_cache_json CLOB; +ALTER TABLE mate_mcp_server ADD COLUMN IF NOT EXISTS tools_cache_updated_at TIMESTAMP; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V93__xiaomi_mimo_provider.sql b/mateclaw-server/src/main/resources/db/migration/h2/V93__xiaomi_mimo_provider.sql new file mode 100644 index 00000000..ea019707 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V93__xiaomi_mimo_provider.sql @@ -0,0 +1,36 @@ +-- V93: register Xiaomi MiMo as an OpenAI-compatible provider with a +-- pre-seeded model catalog covering the MiMo-V2.5 and MiMo-V2 families. +-- +-- Endpoint: https://api.xiaomimimo.com/v1 (OpenAI-compatible chat +-- completions schema). API keys issued by the Xiaomi MiMo platform are +-- accepted directly as bearer tokens; no special prefix is enforced. +-- Model discovery and connection check both follow the standard +-- OpenAI /v1/models contract, so they are enabled by default. + +-- -- Provider -------------------------------------------------------------- +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ( + 'xiaomi-mimo', + 'Xiaomi MiMo', + '', + 'OpenAIChatModel', + '', + 'https://api.xiaomimimo.com/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +); + +-- -- Model catalog --------------------------------------------------------- +-- Five entries covering the V2.5 and V2 families. Temperature defaults to +-- 0.7 to match peer OpenAI-compatible providers; max_tokens 4096 follows +-- the same conservative default used by other built-in entries. +MERGE INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +KEY (id) +VALUES + (1000001200, 'MiMo V2.5 Pro', 'xiaomi-mimo', 'mimo-v2.5-pro', 'Xiaomi MiMo V2.5 Pro — latest flagship reasoning + coding model', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001201, 'MiMo V2.5', 'xiaomi-mimo', 'mimo-v2.5', 'Xiaomi MiMo V2.5 — balanced model in the V2.5 family', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001202, 'MiMo V2 Pro', 'xiaomi-mimo', 'mimo-v2-pro', 'Xiaomi MiMo V2 Pro — 1M token context window flagship', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001203, 'MiMo V2 Omni', 'xiaomi-mimo', 'mimo-v2-omni', 'Xiaomi MiMo V2 Omni — multimodal variant supporting text, vision, audio', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001204, 'MiMo V2 Flash', 'xiaomi-mimo', 'mimo-v2-flash', 'Xiaomi MiMo V2 Flash — fast, low-latency variant with 262K context', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V94__register_office_render_tools.sql b/mateclaw-server/src/main/resources/db/migration/h2/V94__register_office_render_tools.sql new file mode 100644 index 00000000..ede5b3d6 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V94__register_office_render_tools.sql @@ -0,0 +1,16 @@ +-- V94: Register XlsxRenderTool / PptxRenderTool / PdfRenderTool as built-in tools. +-- These mirror DocxRenderTool (V31) so agents can bind them through the tool picker +-- and so the AvailableToolService surfaces them in the UI. +-- Idempotent: MERGE INTO updates existing rows when id matches. + +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX Render', 'Render Markdown directly into a .xlsx workbook and return a one-time download link. In-process Apache POI; each # heading becomes a sheet, pipe tables become rows, numeric cells auto-detected.', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0); + +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000021, 'PptxRenderTool', 'PPTX Render', 'Render Marp-style Markdown directly into a .pptx deck and return a one-time download link. In-process Apache POI; --- separates slides, # / ## titles, - bullets, .', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0); + +MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +KEY (id) +VALUES (1000000022, 'PdfRenderTool', 'PDF Render', 'Render Markdown into a final-form .pdf and return a one-time download link. Two backends (LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback); supports YAML frontmatter for cover / page header / page footer.', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V95__wiki_raw_material_cancel.sql b/mateclaw-server/src/main/resources/db/migration/h2/V95__wiki_raw_material_cancel.sql new file mode 100644 index 00000000..7de8846a --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V95__wiki_raw_material_cancel.sql @@ -0,0 +1,6 @@ +-- V95: cancellation flag for in-progress wiki raw material processing. +-- Lets the user request a stop on a long-running PDF analysis (e.g. when +-- the embedding model has run out of credits) without having to delete +-- the raw material. The processing pipeline checks the flag at its +-- existing abort checkpoints and bails out with a 'cancelled' status. +ALTER TABLE mate_wiki_raw_material ADD COLUMN IF NOT EXISTS cancel_requested BOOLEAN NOT NULL DEFAULT FALSE; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V96__workflow_foundations.sql b/mateclaw-server/src/main/resources/db/migration/h2/V96__workflow_foundations.sql new file mode 100644 index 00000000..2990d072 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V96__workflow_foundations.sql @@ -0,0 +1,173 @@ +-- V96: Foundational schema for the workflow runtime. +-- Eight tables establish workflow identity (workflow + immutable revisions), +-- run state (run + per-step rows + durable pause rows for await_approval), +-- payload URI storage with inline / filesystem fallback, and trigger +-- definitions paired with a dedup-window table for envelope-based event +-- governance. H2 dialect uses CLOB for MEDIUMTEXT and BLOB for LONGBLOB; +-- secondary indexes are emitted as separate CREATE INDEX statements per +-- project convention. + +-- 1. Stable workflow identity + draft (1:1 with workflow row). +CREATE TABLE IF NOT EXISTS mate_workflow ( + id BIGINT NOT NULL PRIMARY KEY, + workspace_id BIGINT NOT NULL, + name VARCHAR(128) NOT NULL, + description VARCHAR(1024), + enabled BOOLEAN NOT NULL DEFAULT TRUE, + draft_json CLOB, + draft_schema_version VARCHAR(8), + draft_updated_by BIGINT, + draft_updated_at TIMESTAMP, + latest_revision_id BIGINT, + created_by BIGINT, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_workflow_workspace_name + ON mate_workflow (workspace_id, name, deleted); + +-- 2. Immutable published revisions; integer revision is monotonic per workflow. +CREATE TABLE IF NOT EXISTS mate_workflow_revision ( + id BIGINT NOT NULL PRIMARY KEY, + workflow_id BIGINT NOT NULL, + revision INT NOT NULL, + graph_json CLOB NOT NULL, + schema_version VARCHAR(8) NOT NULL, + published_note VARCHAR(512), + published_by BIGINT, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_workflow_revision + ON mate_workflow_revision (workflow_id, revision); + +-- 3. Workflow run instance; payload bodies live behind URIs in mate_workflow_payload. +CREATE TABLE IF NOT EXISTS mate_workflow_run ( + id BIGINT NOT NULL PRIMARY KEY, + workflow_id BIGINT NOT NULL, + revision_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL, + state VARCHAR(16) NOT NULL, + triggered_by VARCHAR(32), + triggered_meta CLOB, + initial_input_ref VARCHAR(256), + final_output_ref VARCHAR(256), + error_message VARCHAR(2048), + started_at TIMESTAMP, + completed_at TIMESTAMP, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_workflow_run_started + ON mate_workflow_run (workflow_id, started_at); + +-- 4. Per-step run row; iteration_index reserved for fan_out (and future loop). +CREATE TABLE IF NOT EXISTS mate_workflow_run_step ( + id BIGINT NOT NULL PRIMARY KEY, + run_id BIGINT NOT NULL, + step_index INT NOT NULL, + iteration_index INT, + step_name VARCHAR(128), + agent_id BIGINT, + state VARCHAR(16), + input_ref VARCHAR(256), + output_ref VARCHAR(256), + output_summary VARCHAR(512), + output_content_type VARCHAR(64), + error_message VARCHAR(2048), + duration_ms BIGINT, + token_input INT, + token_output INT, + started_at TIMESTAMP, + completed_at TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_workflow_run_step + ON mate_workflow_run_step (run_id, step_index, iteration_index); + +-- 5. Durable pause rows so await_approval can resume across restarts. +-- pause_token is the resume entry key; external_approval_id ties back to +-- ApprovalWorkflowService rows so the approval callback can find the pause. +CREATE TABLE IF NOT EXISTS mate_workflow_run_pause ( + id BIGINT NOT NULL PRIMARY KEY, + run_id BIGINT NOT NULL, + step_id BIGINT NOT NULL, + pause_kind VARCHAR(32) NOT NULL, + pause_token VARCHAR(128) NOT NULL, + external_approval_id BIGINT, + paused_at TIMESTAMP NOT NULL, + resume_deadline TIMESTAMP, + resume_payload_ref VARCHAR(256), + resumed_at TIMESTAMP, + resume_outcome VARCHAR(32) +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_workflow_pause_run_step + ON mate_workflow_run_pause (run_id, step_id); +CREATE UNIQUE INDEX IF NOT EXISTS uk_workflow_pause_token + ON mate_workflow_run_pause (pause_token); +CREATE INDEX IF NOT EXISTS idx_workflow_pause_external_approval + ON mate_workflow_run_pause (external_approval_id); +CREATE INDEX IF NOT EXISTS idx_workflow_pause_open_deadline + ON mate_workflow_run_pause (resumed_at, resume_deadline); + +-- 6. Payload URI storage. Inline blob for < 256KB; storage_kind=fs/s3/oss +-- carries the external object key in storage_ref. sha256 is for tamper +-- detection only — v0 does not deduplicate across runs. +CREATE TABLE IF NOT EXISTS mate_workflow_payload ( + id BIGINT NOT NULL PRIMARY KEY, + payload_uri VARCHAR(256) NOT NULL, + workspace_id BIGINT NOT NULL, + content_bytes BLOB, + storage_kind VARCHAR(16) NOT NULL, + storage_ref VARCHAR(512), + content_type VARCHAR(64), + sha256 CHAR(64), + size_bytes BIGINT, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_workflow_payload_uri + ON mate_workflow_payload (payload_uri); +CREATE INDEX IF NOT EXISTS idx_workflow_payload_workspace_created + ON mate_workflow_payload (workspace_id, created_at); + +-- 7. Trigger definitions. pattern_version is a lamport counter that fire +-- callbacks compare against on every fire to detect that another instance +-- has updated the cron expression and self-cancel the local schedule. +CREATE TABLE IF NOT EXISTS mate_trigger ( + id BIGINT NOT NULL PRIMARY KEY, + workspace_id BIGINT NOT NULL, + name VARCHAR(128), + pattern_type VARCHAR(32) NOT NULL, + pattern_json CLOB NOT NULL, + target_type VARCHAR(16) NOT NULL, + target_id BIGINT NOT NULL, + payload_template CLOB, + rate_limit_per_min INT NOT NULL DEFAULT 60, + dedup_window_secs INT NOT NULL DEFAULT 60, + bot_self_filter BOOLEAN NOT NULL DEFAULT TRUE, + enabled BOOLEAN NOT NULL DEFAULT TRUE, + fire_count BIGINT NOT NULL DEFAULT 0, + max_fires BIGINT NOT NULL DEFAULT 0, + last_fired_at TIMESTAMP, + pattern_version BIGINT NOT NULL DEFAULT 1, + create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + deleted INT NOT NULL DEFAULT 0 +); +CREATE INDEX IF NOT EXISTS idx_trigger_workspace_enabled + ON mate_trigger (workspace_id, enabled, deleted); +CREATE INDEX IF NOT EXISTS idx_trigger_target + ON mate_trigger (target_type, target_id); + +-- 8. Event dedup window. dedup_key is envelope.eventId, falling back to +-- sourceHash when the upstream channel did not provide a stable id. +CREATE TABLE IF NOT EXISTS mate_trigger_event ( + id BIGINT NOT NULL PRIMARY KEY, + trigger_id BIGINT NOT NULL, + dedup_key VARCHAR(128) NOT NULL, + received_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMP NOT NULL +); +CREATE UNIQUE INDEX IF NOT EXISTS uk_trigger_dedup + ON mate_trigger_event (trigger_id, dedup_key); +CREATE INDEX IF NOT EXISTS idx_trigger_event_expires + ON mate_trigger_event (expires_at); diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V97__workflow_purge_tombstones.sql b/mateclaw-server/src/main/resources/db/migration/h2/V97__workflow_purge_tombstones.sql new file mode 100644 index 00000000..1202c754 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V97__workflow_purge_tombstones.sql @@ -0,0 +1,16 @@ +-- The workflow / trigger entities originally shipped with @TableLogic, which +-- caused deleteById() to soft-update `deleted=1`. The project convention is +-- hard-delete everywhere (see contributing.md), and the soft-delete path +-- collided with the (workspace_id, name, deleted) unique key whenever a name +-- was recreated and re-deleted: the second update tried to write a tombstone +-- that already existed. +-- +-- The entity annotations are removed in this same change set so deleteById() +-- now performs a real DELETE. This migration purges any tombstones that the +-- old soft-delete path may have written, because the annotation-driven query +-- filter is no longer applied — a stale `deleted=1` row would otherwise show +-- up in list endpoints. + +DELETE FROM mate_workflow WHERE deleted <> 0; +DELETE FROM mate_workflow_run WHERE deleted <> 0; +DELETE FROM mate_trigger WHERE deleted <> 0; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V98__trigger_last_error.sql b/mateclaw-server/src/main/resources/db/migration/h2/V98__trigger_last_error.sql new file mode 100644 index 00000000..3ea77051 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V98__trigger_last_error.sql @@ -0,0 +1,7 @@ +-- Persist the most recent dispatch outcome message on the trigger row +-- itself so the UI can show *why* a trigger has stopped firing without +-- joining trigger_event for forensics. The dispatcher writes a non-null +-- message on SKIPPED / FAILED outcomes and clears it on FIRED. + +ALTER TABLE mate_trigger ADD COLUMN IF NOT EXISTS last_error VARCHAR(2048); +ALTER TABLE mate_trigger ADD COLUMN IF NOT EXISTS last_dispatched_at TIMESTAMP; diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V99__dashscope_compat_provider.sql b/mateclaw-server/src/main/resources/db/migration/h2/V99__dashscope_compat_provider.sql new file mode 100644 index 00000000..6e846054 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V99__dashscope_compat_provider.sql @@ -0,0 +1,48 @@ +-- V99: register a DashScope OpenAI-compatible provider entry alongside the +-- existing native dashscope provider, plus the dot-versioned Qwen families +-- (qwen3.5-*, qwen3.6-*) that only ship on compatible-mode/v1. +-- +-- Why a separate provider: +-- The dashscope provider runs on DashScopeChatModel (native protocol). Calling +-- a dot-versioned model id through the native text-generation/generation +-- endpoint returns 400 InvalidParameter — those models are only exposed via +-- the OpenAI-compatible endpoint. Rather than dynamically rewriting the +-- protocol per model, we register a sibling provider that uses +-- OpenAIChatModel against compatible-mode/v1 with the same sk- API key. +-- +-- Existing seed file db/data-zh.sql already carries the same rows for fresh +-- installs; this migration is the upgrade path for already-deployed databases +-- (DatabaseBootstrapRunner skips the seed when mate_user is non-empty). + +-- -- Provider -------------------------------------------------------------- +MERGE INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +KEY (provider_id) +VALUES ( + 'dashscope-compat', + 'DashScope (兼容模式)', + 'sk-', + 'OpenAIChatModel', + '', + 'https://dashscope.aliyuncs.com/compatible-mode/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +); + +-- -- Model catalog --------------------------------------------------------- +-- Dot-versioned Qwen families exposed through compatible-mode. IDs use the +-- 1000000601-1000000606 block reserved for this provider so future additions +-- under dashscope-compat can grow contiguously. +-- +-- NB: only the {-plus, -vl-plus} variants are public on compatible-mode at the +-- time this migration was written. The {-max, -vl-max} variants exist in the +-- model marketplace but return 404 (`The model 'qwen3.6-max' does not exist +-- or you do not have access to it.`) for all general accounts. We seed only +-- the verified-callable ones; users with whitelist access can add the others +-- through Settings → Models manually. +MERGE INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +KEY (id) +VALUES + (1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', '通义千问 3.6 Plus 旗舰,平衡推理与速度(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', '通义千问 3.5 Plus(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', '通义千问 3 视觉理解 Plus,支持图像、视频输入(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql new file mode 100644 index 00000000..5aa02d34 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V100__multimodal_default_models.sql @@ -0,0 +1,16 @@ +-- V100: System-level defaults for vision and video sidecar routing. +-- When the agent's primary model lacks the modality required by an attachment, +-- the runtime delegates a single caption call to the model recorded here. +-- Empty value = not configured; the UI then asks the user to pick one. +-- Setting value stores mate_model_config.id as a string (provider+model_name pairs are not unique). +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000002001, 'default.vision_model', '', + 'Default vision-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VISION modality', + NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key = setting_key; + +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000002002, 'default.video_model', '', + 'Default video-capable model id (mate_model_config.id) used by sidecar router when primary model lacks VIDEO modality', + NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key = setting_key; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V101__cleanup_blank_tool_guard_rule_id.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V101__cleanup_blank_tool_guard_rule_id.sql new file mode 100644 index 00000000..49960ed6 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V101__cleanup_blank_tool_guard_rule_id.sql @@ -0,0 +1,14 @@ +-- See the matching H2 file for context. This migration purges any +-- orphan rows that earlier releases persisted with a blank rule_id and +-- then installs a CHECK constraint so the schema itself rejects blank +-- rule_id, defending against any future code path that bypasses the +-- service-layer guard. CHECK constraints are enforced from MySQL 8.0.16 +-- onward; this project targets MySQL 8.0+ so the constraint is live. + +DELETE FROM mate_tool_guard_rule +WHERE (rule_id IS NULL OR LENGTH(TRIM(rule_id)) = 0) + AND (builtin IS NULL OR builtin = FALSE); + +ALTER TABLE mate_tool_guard_rule + ADD CONSTRAINT ck_tool_guard_rule_id_nonblank + CHECK (rule_id IS NOT NULL AND LENGTH(TRIM(rule_id)) > 0); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V102__agent_unique_name_per_workspace.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V102__agent_unique_name_per_workspace.sql new file mode 100644 index 00000000..21867d60 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V102__agent_unique_name_per_workspace.sql @@ -0,0 +1,37 @@ +-- Enforce unique Agent name within a workspace. See H2 variant for context. +-- +-- Step 1 — rename pre-existing duplicates. MySQL forbids referencing the +-- target table in a subquery for UPDATE, so we use a self-join with a +-- derived "min id per group" table to pick which row keeps the original +-- name (the oldest by id) and rename the rest. +-- +-- The rename target drops the original name and substitutes +-- `__mate_dup_v102____`. Any deterministic transformation of +-- the original name has a non-zero collision risk against a hand-typed +-- pre-existing row that happens to match the pattern (e.g. someone named +-- their agent `foo__v102_dup__2`). A random UUID component drives the +-- collision probability to ~1/2^122 — provably unique for a one-shot +-- migration. Mirrors the H2 variant via MySQL's UUID() function. +UPDATE mate_agent t +JOIN ( + SELECT workspace_id, name, MIN(id) AS keep_id + FROM mate_agent + GROUP BY workspace_id, name + HAVING COUNT(*) > 1 +) k + ON t.workspace_id = k.workspace_id + AND t.name = k.name + AND t.id <> k.keep_id +SET t.name = CONCAT('__mate_dup_v102__', t.id, '__', UUID()); + +-- Step 2 — add the unique index, idempotent via INFORMATION_SCHEMA guard +-- (matches the V69 cron-job pattern; works on MySQL < 8.0.29 which has no +-- CREATE INDEX IF NOT EXISTS). +SET @idx_exists := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_agent' + AND INDEX_NAME = 'uk_agent_workspace_name'); +SET @stmt := IF(@idx_exists = 0, + 'CREATE UNIQUE INDEX uk_agent_workspace_name ON mate_agent(workspace_id, name)', + 'SELECT 1'); +PREPARE s FROM @stmt; EXECUTE s; DEALLOCATE PREPARE s; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V103__drop_fact_entity_ref.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V103__drop_fact_entity_ref.sql new file mode 100644 index 00000000..9f0ce948 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V103__drop_fact_entity_ref.sql @@ -0,0 +1,2 @@ +-- Drop the dead mate_fact_entity_ref table. See H2 variant for context. +DROP TABLE IF EXISTS mate_fact_entity_ref; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql new file mode 100644 index 00000000..6c50d76b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V104__wiki_chunk_embedding_text_version.sql @@ -0,0 +1,9 @@ +-- V104: track which input format a chunk's stored embedding was generated against. +-- The embedding input builder concatenates raw title / header breadcrumb / page +-- number alongside chunk content; bumping the builder's CURRENT_INPUT_VERSION +-- forces a re-embed pass without changing the model. NULL is treated as the +-- legacy content-only format and re-embedded lazily on the next pass. +-- MySQL lacks `ADD COLUMN IF NOT EXISTS`; use INFORMATION_SCHEMA guard instead. +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_chunk' AND COLUMN_NAME = 'embedding_text_version'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_chunk ADD COLUMN embedding_text_version VARCHAR(32) NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql new file mode 100644 index 00000000..55c28709 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V105__wiki_transformation.sql @@ -0,0 +1,68 @@ +-- Reusable user-defined prompt templates ("transformations") that run over +-- a raw material's extracted text and persist the LLM output as an artifact +-- on the knowledge base. Templates can be flagged apply_default so the +-- ingestion pipeline runs them automatically once a raw material reaches +-- the completed state. Manual / agent-tool runs are also supported. + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + kb_id BIGINT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + name VARCHAR(64) NOT NULL, + title VARCHAR(255) NOT NULL, + description VARCHAR(1024), + + prompt_template MEDIUMTEXT NOT NULL, + + apply_default TINYINT(1) NOT NULL DEFAULT 0, + model_id BIGINT NULL, + enabled TINYINT(1) NOT NULL DEFAULT 1, + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + KEY idx_wtr_kb (kb_id, deleted), + KEY idx_wtr_ws (workspace_id, deleted) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; + +-- Unique name per KB (NULL kb_id rows compete in a shared "global" bucket). +-- MySQL treats NULL as distinct in unique indexes, so workspace-wide names +-- can technically collide; the service layer enforces uniqueness for the +-- NULL-kb_id case in software. +CREATE UNIQUE INDEX uk_wtr_kb_name ON mate_wiki_transformation (kb_id, name, deleted); + + +CREATE TABLE IF NOT EXISTS mate_wiki_transformation_run ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + + transformation_id BIGINT NOT NULL, + kb_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL DEFAULT 1, + + input_kind VARCHAR(16) NOT NULL, + raw_id BIGINT NULL, + page_id BIGINT NULL, + + status VARCHAR(16) NOT NULL DEFAULT 'pending', + + output MEDIUMTEXT, + error VARCHAR(2048), + model_id BIGINT NULL, + + triggered_by VARCHAR(32) NOT NULL DEFAULT 'manual', + + started_at DATETIME(3) NULL, + completed_at DATETIME(3) NULL, + duration_ms BIGINT NULL, + + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted TINYINT NOT NULL DEFAULT 0, + + KEY idx_wtrn_tr (transformation_id, deleted), + KEY idx_wtrn_kb (kb_id, deleted), + KEY idx_wtrn_raw (raw_id, deleted) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql new file mode 100644 index 00000000..44b0435b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V106__wiki_transformation_output_target.sql @@ -0,0 +1,22 @@ +-- Two-part follow-up to V105 so a transformation's output can flow back +-- into the KB as a first-class artifact. See the h2 sibling migration for +-- the prose explanation. MySQL lacks `ADD COLUMN IF NOT EXISTS`, so each +-- column is guarded by an INFORMATION_SCHEMA check + prepared statement. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation' + AND COLUMN_NAME = 'output_target'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation ADD COLUMN output_target VARCHAR(16) NOT NULL DEFAULT ''none''', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation_run' + AND COLUMN_NAME = 'output_page_id'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation_run ADD COLUMN output_page_id BIGINT NULL', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql new file mode 100644 index 00000000..a1be50d0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V107__wiki_page_embedding.sql @@ -0,0 +1,24 @@ +-- Page-level embedding columns. See the h2 sibling for the prose +-- explanation. MySQL lacks `ADD COLUMN IF NOT EXISTS`, so each column +-- guarded by an INFORMATION_SCHEMA check + prepared statement. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding BLOB DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding_model'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_model VARCHAR(64) DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_page' + AND COLUMN_NAME = 'embedding_text_version'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_page ADD COLUMN embedding_text_version VARCHAR(32) DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V108__wiki_transformation_starter_pack.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V108__wiki_transformation_starter_pack.sql new file mode 100644 index 00000000..19b890a4 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V108__wiki_transformation_starter_pack.sql @@ -0,0 +1,246 @@ +-- Starter pack: 7 workspace-wide transformation templates. See h2 sibling +-- for the prose explanation. INSERT IGNORE so a re-run (e.g. via repair) +-- never clobbers user edits. + +INSERT IGNORE INTO mate_wiki_transformation + (id, kb_id, workspace_id, name, title, description, prompt_template, + apply_default, model_id, enabled, output_target, create_time, update_time, deleted) +VALUES +(1000004001, NULL, 1, + 'contract-risk-extract', + '合同风险点提取', + '逐条审查合同条款,标注风险等级、原文位置、AI 建议改写。配合企业场景 → 合同审查使用。', +'你是一名企业法务审查员。从下面的合同文本中完整提取所有需要关注的风险条款,按以下结构输出 Markdown: + +## 风险条款清单 + +对每条值得审查的条款,输出三级标题: + +### <条款简称> +- **风险等级**:高 / 中 / 低 +- **条款类型**:赔偿 / 责任限制 / 付款 / 保密 / 竞业 / 终止 / 管辖 / 数据保护 / 其他 +- **原文位置**:第 X 条 / 第 Y 页(材料未标号时写「未标注」) +- **原文摘录**:用「」引用关键句 +- **风险描述**:≤ 50 字说明风险所在 +- **建议改写**:给出可直接采用的修订版本 + +## 总体评估 + +一段话总结这份合同的整体风险水位与签字建议(≤ 200 字)。 + +要求: +- 不要虚构原文没有的条款 +- 数字与条款编号保留原样 +- 中文输出,不要任何客套或元描述 + +合同标题:{title} + +合同正文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004002, NULL, 1, + 'meeting-action-items', + '会议纪要 → 行动项', + '从会议纪要中穷尽抽取决议 + 行动项(owner / 截止日 / 验收标准),适合周会、决策会议。', +'你是会议纪要分析助理。从下面的纪要中穷尽抽取所有行动项与决议,按以下结构输出 Markdown: + +## 决议清单 +按时间或重要性顺序列出每条明确决议;每条 ≤ 一句话。 + +## 行动项清单 + +| 序号 | 行动 | 负责人 | 截止日 | 验收标准 | +|---|---|---|---|---| + +要求: +- 「行动」用动词开头(如「提交」「完成」「对齐」) +- 负责人若未明确写「未指派」 +- 截止日若未明确写「未定」 +- 验收标准一句话写出「做完是什么样」 +- 不要把「讨论了 X」当作行动项 + +## 风险与依赖 +一句话列出会议中提到的潜在阻塞或跨团队依赖(≤ 5 条)。 + +要求:中文,无客套,无元描述。 + +会议主题:{title} + +纪要正文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004003, NULL, 1, + 'customer-profile', + '客户邮件 / 访谈画像', + '把客户邮件、会议纪要、CRM 记录合成一份结构化客户画像,配合企业场景 → 客户情报使用。', +'你是销售情报员。从下面的客户邮件 / CRM 记录 / 访谈中提取一份客户画像,按以下结构输出 Markdown: + +## 客户档案 +- **名称**: +- **行业 / 规模**: +- **当前阶段**:潜在 / 沟通中 / 谈判中 / 已成交(若无明确信号写「未知」) +- **决策链关键人**:列出姓名 + 角色 + 倾向 + +## 痛点与机会 +- 3-5 条关键痛点,每条带原文引用 +- 2-3 条潜在切入点 + +## 异议预判 +列出客户可能的反对意见 + 对应应对话术。 + +## 下一步建议 +- 3 条具体动作,按优先级排序,每条带「为什么现在做」 + +要求:不要发明文本没说的事;不确定时写「未提及」。 + +客户:{title} + +原文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004004, NULL, 1, + 'competitor-update', + '竞品动态摘要', + '把新闻 / 产品 release / 招聘信号 / 客户提及合成一份竞品动态简报。', +'你是市场情报员。从下面的材料中提取与竞争对手相关的动态,按以下结构输出 Markdown: + +## 涉及对手 +列出材料中提到的所有竞品公司或产品。 + +## 关键动态 + +按时间倒序,每条输出: + +### <对手 / 产品> · <动态简称> +- **类型**:新产品 / 招聘 / 融资 / 客户胜出 / 价格调整 / 团队变动 / 其他 +- **原文摘录**:「」引用 +- **来源**:网页 / 邮件 / 新闻渠道 +- **对我们的影响**:威胁 / 机会 / 中性,一句话说明 + +## 战术建议 +3 条针对性的应对动作,按优先级排序。 + +## 监控建议 +列出值得长期追踪的关键词或信号。 + +要求:中文,不发明内容,不确定时跳过。 + +材料:{title} + +原文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004005, NULL, 1, + 'resume-structured-extract', + '简历结构化', + '把简历提取为标准化档案:教育、工作、技能、亮点。适合批量初筛。', +'你是 HR 助理。把下面的简历提取为结构化档案: + +## 候选人信息 +- **姓名**: +- **当前职位**: +- **总工作年限**: +- **专业领域**: + +## 教育经历 + +| 学校 | 学位 / 专业 | 时间 | +|---|---|---| + +## 工作经历 + +按时间倒序,每段输出: + +### <公司> · <职位> · <时间> +- **职责摘要**:≤ 30 字 +- **关键产出**:≤ 3 条 bullet(量化优先) + +## 技能矩阵 + +| 技能 | 熟练度 | +|---|---| + +## 候选人亮点 +一段话归纳最值得关注的 3 件事(≤ 150 字)。 + +要求:不要发明文本没有的经历;不确定写「未提及」;中文输出。注意:不要把性别 / 年龄 / 户籍写进画像。 + +简历:{title} + +原文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004006, NULL, 1, + 'incident-postmortem', + '事故 5-Why 复盘', + '从事故报告 / 时间线生成 5-Why 链 + 整改清单 + 相似事故关键词。适合 SRE / 运维团队。', +'你是 SRE 事故复盘助理。从下面的事故报告 / 时间线中输出 5-Why 分析: + +## 事故概要 +- **现象**:1 句话 +- **影响范围**:用户数 / 系统 / 持续时间 +- **触发时间**: + +## 5 Whys 链 + +1. **现象**:… + **Why?** … +2. **Why?** … +3. **Why?** … +4. **Why?** … +5. **根因 (Why?)** … + +## 整改清单 + +| 序号 | 行动 | 负责团队 | 优先级 | 截止 | +|---|---|---|---|---| + +## 相似事故关联 +列出可能相关的历史事故关键词(用于后续 wiki 检索)。 + +## 复盘要点 +3 条最值得团队记住的教训。 + +要求:中文,技术准确,不发明数据。 + +事故:{title} + +报告: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0), + +(1000004007, NULL, 1, + 'paper-imrad', + '论文 IMRaD 摘要', + '把论文 / 技术报告浓缩为 IMRaD 结构化摘要 + 关键术语表,适合研究型团队。', +'你是学术摘要助理。把下面的论文 / 技术报告浓缩为 IMRaD 结构化摘要: + +## Introduction +解决什么问题,为什么重要(≤ 100 字) + +## Methods +使用什么方法 / 数据 / 模型(≤ 150 字) + +## Results +最重要的 3-5 个量化或定性结果(每条 ≤ 30 字) + +## Discussion +- **主要洞察**:1-2 句 +- **局限性**:1-2 条 +- **可复现性**:高 / 中 / 低,附 1 句理由 + +## 关键术语 +列出 5-8 个核心术语,每个加一句话定义。 + +要求:保留 LaTeX 公式(如有),不发明结果,中文写作。 + +论文:{title} + +原文: +{input_text}', + 0, NULL, 1, 'page', NOW(3), NOW(3), 0); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql new file mode 100644 index 00000000..331b969b --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V109__wiki_transformation_output_format.sql @@ -0,0 +1,11 @@ +-- Output format declared on the template. See h2 sibling for the prose +-- explanation. MySQL needs the INFORMATION_SCHEMA guard pattern. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_wiki_transformation' + AND COLUMN_NAME = 'output_format'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_wiki_transformation ADD COLUMN output_format VARCHAR(16) NOT NULL DEFAULT ''markdown''', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V110__wiki_transformation_run_tokens.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V110__wiki_transformation_run_tokens.sql new file mode 100644 index 00000000..97288351 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V110__wiki_transformation_run_tokens.sql @@ -0,0 +1,19 @@ +-- Record per-run token usage. See h2 sibling for prose explanation. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_transformation_run' + AND COLUMN_NAME = 'input_tokens'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_transformation_run ADD COLUMN input_tokens BIGINT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_transformation_run' + AND COLUMN_NAME = 'output_tokens'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_transformation_run ADD COLUMN output_tokens BIGINT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_transformation_run' + AND COLUMN_NAME = 'total_tokens'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_transformation_run ADD COLUMN total_tokens BIGINT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V111__wiki_transformation_output_schema.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V111__wiki_transformation_output_schema.sql new file mode 100644 index 00000000..2bec544c --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V111__wiki_transformation_output_schema.sql @@ -0,0 +1,7 @@ +-- Optional JSON Schema column. See h2 sibling for the prose explanation. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_transformation' + AND COLUMN_NAME = 'output_schema'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_transformation ADD COLUMN output_schema MEDIUMTEXT DEFAULT NULL', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V112__skill_file.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V112__skill_file.sql new file mode 100644 index 00000000..aea34aa7 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V112__skill_file.sql @@ -0,0 +1,26 @@ +-- V112: persist skill bundle files (scripts/ + references/) in the database. +-- +-- Until now scripts/references only lived on the local filesystem of whichever +-- node handled the upload. Multi-instance deployments sharing one MySQL would +-- have the skill row visible everywhere but the script files only on one node, +-- so any other node attempting to run a skill script either failed or ran a +-- stale local copy. Treating the database as the canonical bundle store and +-- the filesystem as a materialized cache resolves that gap and matches the +-- existing pattern for SKILL.md (canonical in mate_skill.skill_content, +-- mirrored to disk by the workspace manager). +-- +-- MEDIUMTEXT (16MB) comfortably covers the per-file 1MB cap enforced by +-- ZipSkillFetcher and the 50MB total bundle cap. + +CREATE TABLE IF NOT EXISTS mate_skill_file ( + id BIGINT NOT NULL PRIMARY KEY, + skill_id BIGINT NOT NULL, + file_path VARCHAR(512) NOT NULL, + content MEDIUMTEXT, + content_size INT NOT NULL DEFAULT 0, + sha256 CHAR(64), + create_time DATETIME NOT NULL, + update_time DATETIME NOT NULL, + UNIQUE KEY uk_skill_file_path (skill_id, file_path), + KEY idx_skill_file_skill (skill_id) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V91__widen_message_and_skill_content.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V91__widen_message_and_skill_content.sql new file mode 100644 index 00000000..5fd212ed --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V91__widen_message_and_skill_content.sql @@ -0,0 +1,38 @@ +-- V91: Widen mate_message.content / content_parts and mate_skill.skill_content +-- from TEXT (64KB) to MEDIUMTEXT (16MB). +-- +-- TEXT caps at 65,535 bytes. A multi-turn ReAct session accumulates tool calls +-- and observations into content_parts JSON well past that cap, and a long +-- Chinese final answer (~22k chars × 3 bytes UTF-8) overflows `content`. +-- The truncation rejects the assistant message INSERT after the SSE stream +-- has already finished, so users see the reply live but it disappears on +-- page reload (only the user message survives in the DB). +-- +-- Idempotent: only modifies the column when its current type is still TEXT. + +SET @c := (SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_message' + AND COLUMN_NAME = 'content'); +SET @s := IF(@c = 'text', + 'ALTER TABLE mate_message MODIFY COLUMN content MEDIUMTEXT', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_message' + AND COLUMN_NAME = 'content_parts'); +SET @s := IF(@c = 'text', + 'ALTER TABLE mate_message MODIFY COLUMN content_parts MEDIUMTEXT', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_skill' + AND COLUMN_NAME = 'skill_content'); +SET @s := IF(@c = 'text', + 'ALTER TABLE mate_skill MODIFY COLUMN skill_content MEDIUMTEXT', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V92__mcp_server_tools_cache.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V92__mcp_server_tools_cache.sql new file mode 100644 index 00000000..e151ddfb --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V92__mcp_server_tools_cache.sql @@ -0,0 +1,29 @@ +-- V92: Persist each MCP server's discovered tool list as a per-row JSON +-- snapshot so the agent edit picker can render the tools even when the +-- upstream server is briefly disconnected, and so the per-tool atomic +-- binding flow has a stable place to resolve raw tool names from the +-- prefixed callback name. +-- +-- MySQL doesn't support `ADD COLUMN IF NOT EXISTS` natively (5.7 and most +-- 8.0 deployments), so guard each ALTER with an INFORMATION_SCHEMA lookup +-- + PREPARE/EXECUTE so re-runs become no-ops instead of failing the +-- migration. Flyway's repair-on-startup compensates for any partial +-- failure. + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_mcp_server' + AND COLUMN_NAME = 'tools_cache_json'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_mcp_server ADD COLUMN tools_cache_json MEDIUMTEXT', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; + +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_mcp_server' + AND COLUMN_NAME = 'tools_cache_updated_at'); +SET @s := IF(@c = 0, + 'ALTER TABLE mate_mcp_server ADD COLUMN tools_cache_updated_at TIMESTAMP NULL', + 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V93__xiaomi_mimo_provider.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V93__xiaomi_mimo_provider.sql new file mode 100644 index 00000000..06005447 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V93__xiaomi_mimo_provider.sql @@ -0,0 +1,43 @@ +-- V93: register Xiaomi MiMo as an OpenAI-compatible provider with a +-- pre-seeded model catalog. See the H2 copy for full background. + +-- -- Provider -------------------------------------------------------------- +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ( + 'xiaomi-mimo', + 'Xiaomi MiMo', + '', + 'OpenAIChatModel', + '', + 'https://api.xiaomimimo.com/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + api_key_prefix = VALUES(api_key_prefix), + chat_model = VALUES(chat_model), + base_url = VALUES(base_url), + generate_kwargs = VALUES(generate_kwargs), + support_model_discovery = VALUES(support_model_discovery), + support_connection_check = VALUES(support_connection_check), + freeze_url = VALUES(freeze_url), + require_api_key = VALUES(require_api_key), + update_time = VALUES(update_time); + +-- -- Model catalog --------------------------------------------------------- +INSERT INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +VALUES + (1000001200, 'MiMo V2.5 Pro', 'xiaomi-mimo', 'mimo-v2.5-pro', 'Xiaomi MiMo V2.5 Pro — latest flagship reasoning + coding model', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001201, 'MiMo V2.5', 'xiaomi-mimo', 'mimo-v2.5', 'Xiaomi MiMo V2.5 — balanced model in the V2.5 family', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001202, 'MiMo V2 Pro', 'xiaomi-mimo', 'mimo-v2-pro', 'Xiaomi MiMo V2 Pro — 1M token context window flagship', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001203, 'MiMo V2 Omni', 'xiaomi-mimo', 'mimo-v2-omni', 'Xiaomi MiMo V2 Omni — multimodal variant supporting text, vision, audio', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000001204, 'MiMo V2 Flash', 'xiaomi-mimo', 'mimo-v2-flash', 'Xiaomi MiMo V2 Flash — fast, low-latency variant with 262K context', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + model_name = VALUES(model_name), + description = VALUES(description), + builtin = VALUES(builtin), + enabled = VALUES(enabled), + update_time = VALUES(update_time); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V94__register_office_render_tools.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V94__register_office_render_tools.sql new file mode 100644 index 00000000..80740109 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V94__register_office_render_tools.sql @@ -0,0 +1,16 @@ +-- V94: Register XlsxRenderTool / PptxRenderTool / PdfRenderTool as built-in tools. +-- These mirror DocxRenderTool (V31) so agents can bind them through the tool picker +-- and so the AvailableToolService surfaces them in the UI. +-- Idempotent: ON DUPLICATE KEY UPDATE keeps rows in sync if they already exist. + +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000020, 'XlsxRenderTool', 'XLSX Render', 'Render Markdown directly into a .xlsx workbook and return a one-time download link. In-process Apache POI; each # heading becomes a sheet, pipe tables become rows, numeric cells auto-detected.', 'builtin', 'xlsxRenderTool', '📊', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), bean_name=VALUES(bean_name), icon=VALUES(icon), update_time=VALUES(update_time); + +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000021, 'PptxRenderTool', 'PPTX Render', 'Render Marp-style Markdown directly into a .pptx deck and return a one-time download link. In-process Apache POI; --- separates slides, # / ## titles, - bullets, .', 'builtin', 'pptxRenderTool', '🎞️', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), bean_name=VALUES(bean_name), icon=VALUES(icon), update_time=VALUES(update_time); + +INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) +VALUES (1000000022, 'PdfRenderTool', 'PDF Render', 'Render Markdown into a final-form .pdf and return a one-time download link. Two backends (LibreOffice subprocess preferred, OpenPDF + Flying Saucer fallback); supports YAML frontmatter for cover / page header / page footer.', 'builtin', 'pdfRenderTool', '📄', TRUE, TRUE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE name=VALUES(name), display_name=VALUES(display_name), description=VALUES(description), bean_name=VALUES(bean_name), icon=VALUES(icon), update_time=VALUES(update_time); diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V95__wiki_raw_material_cancel.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V95__wiki_raw_material_cancel.sql new file mode 100644 index 00000000..3a5e5130 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V95__wiki_raw_material_cancel.sql @@ -0,0 +1,9 @@ +-- V95: cancellation flag for in-progress wiki raw material processing. +-- Lets the user request a stop on a long-running PDF analysis (e.g. when +-- the embedding model has run out of credits) without having to delete +-- the raw material. The processing pipeline checks the flag at its +-- existing abort checkpoints and bails out with a 'cancelled' status. +-- MySQL lacks `ADD COLUMN IF NOT EXISTS`; use INFORMATION_SCHEMA guard instead. +SET @c := (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'mate_wiki_raw_material' AND COLUMN_NAME = 'cancel_requested'); +SET @s := IF(@c = 0, 'ALTER TABLE mate_wiki_raw_material ADD COLUMN cancel_requested BOOLEAN NOT NULL DEFAULT FALSE', 'SELECT 1'); +PREPARE stmt FROM @s; EXECUTE stmt; DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V96__workflow_foundations.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V96__workflow_foundations.sql new file mode 100644 index 00000000..edc23508 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V96__workflow_foundations.sql @@ -0,0 +1,162 @@ +-- V96: Foundational schema for the workflow runtime. +-- Eight tables establish workflow identity (workflow + immutable revisions), +-- run state (run + per-step rows + durable pause rows for await_approval), +-- payload URI storage with inline / filesystem fallback, and trigger +-- definitions paired with a dedup-window table for envelope-based event +-- governance. CREATE TABLE IF NOT EXISTS is itself idempotent on MySQL. + +-- 1. Stable workflow identity + draft (1:1 with workflow row). +CREATE TABLE IF NOT EXISTS mate_workflow ( + id BIGINT NOT NULL PRIMARY KEY, + workspace_id BIGINT NOT NULL, + name VARCHAR(128) NOT NULL, + description VARCHAR(1024), + enabled TINYINT NOT NULL DEFAULT 1, + draft_json MEDIUMTEXT, + draft_schema_version VARCHAR(8), + draft_updated_by BIGINT, + draft_updated_at DATETIME(3), + latest_revision_id BIGINT, + created_by BIGINT, + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted INT NOT NULL DEFAULT 0, + UNIQUE KEY uk_workflow_workspace_name (workspace_id, name, deleted) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Workflow definition with stable identity and inline draft.'; + +-- 2. Immutable published revisions; integer revision is monotonic per workflow. +CREATE TABLE IF NOT EXISTS mate_workflow_revision ( + id BIGINT NOT NULL PRIMARY KEY, + workflow_id BIGINT NOT NULL, + revision INT NOT NULL, + graph_json MEDIUMTEXT NOT NULL, + schema_version VARCHAR(8) NOT NULL, + published_note VARCHAR(512), + published_by BIGINT, + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + UNIQUE KEY uk_workflow_revision (workflow_id, revision) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Immutable published workflow revisions.'; + +-- 3. Workflow run instance; payload bodies live behind URIs in mate_workflow_payload. +CREATE TABLE IF NOT EXISTS mate_workflow_run ( + id BIGINT NOT NULL PRIMARY KEY, + workflow_id BIGINT NOT NULL, + revision_id BIGINT NOT NULL, + workspace_id BIGINT NOT NULL, + state VARCHAR(16) NOT NULL, + triggered_by VARCHAR(32), + triggered_meta MEDIUMTEXT, + initial_input_ref VARCHAR(256), + final_output_ref VARCHAR(256), + error_message VARCHAR(2048), + started_at DATETIME(3), + completed_at DATETIME(3), + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + deleted INT NOT NULL DEFAULT 0, + KEY idx_workflow_run_started (workflow_id, started_at) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Workflow run instances locked to a specific revision.'; + +-- 4. Per-step run row; iteration_index reserved for fan_out (and future loop). +CREATE TABLE IF NOT EXISTS mate_workflow_run_step ( + id BIGINT NOT NULL PRIMARY KEY, + run_id BIGINT NOT NULL, + step_index INT NOT NULL, + iteration_index INT, + step_name VARCHAR(128), + agent_id BIGINT, + state VARCHAR(16), + input_ref VARCHAR(256), + output_ref VARCHAR(256), + output_summary VARCHAR(512), + output_content_type VARCHAR(64), + error_message VARCHAR(2048), + duration_ms BIGINT, + token_input INT, + token_output INT, + started_at DATETIME(3), + completed_at DATETIME(3), + KEY idx_workflow_run_step (run_id, step_index, iteration_index) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Per-step run rows with input/output references and timings.'; + +-- 5. Durable pause rows so await_approval can resume across restarts. +CREATE TABLE IF NOT EXISTS mate_workflow_run_pause ( + id BIGINT NOT NULL PRIMARY KEY, + run_id BIGINT NOT NULL, + step_id BIGINT NOT NULL, + pause_kind VARCHAR(32) NOT NULL, + pause_token VARCHAR(128) NOT NULL, + external_approval_id BIGINT, + paused_at DATETIME(3) NOT NULL, + resume_deadline DATETIME(3), + resume_payload_ref VARCHAR(256), + resumed_at DATETIME(3), + resume_outcome VARCHAR(32), + UNIQUE KEY uk_workflow_pause_run_step (run_id, step_id), + UNIQUE KEY uk_workflow_pause_token (pause_token), + KEY idx_workflow_pause_external_approval (external_approval_id), + KEY idx_workflow_pause_open_deadline (resumed_at, resume_deadline) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Durable workflow pause rows for await_approval resume.'; + +-- 6. Payload URI storage. Inline blob for < 256KB; storage_kind=fs/s3/oss +-- carries the external object key in storage_ref. +CREATE TABLE IF NOT EXISTS mate_workflow_payload ( + id BIGINT NOT NULL PRIMARY KEY, + payload_uri VARCHAR(256) NOT NULL, + workspace_id BIGINT NOT NULL, + content_bytes LONGBLOB, + storage_kind VARCHAR(16) NOT NULL, + storage_ref VARCHAR(512), + content_type VARCHAR(64), + sha256 CHAR(64), + size_bytes BIGINT, + created_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + UNIQUE KEY uk_workflow_payload_uri (payload_uri), + KEY idx_workflow_payload_workspace_created (workspace_id, created_at) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Payload bodies addressed by stable URIs.'; + +-- 7. Trigger definitions. pattern_version is a lamport counter that fire +-- callbacks compare against on every fire to detect that another instance +-- has updated the cron expression and self-cancel the local schedule. +CREATE TABLE IF NOT EXISTS mate_trigger ( + id BIGINT NOT NULL PRIMARY KEY, + workspace_id BIGINT NOT NULL, + name VARCHAR(128), + pattern_type VARCHAR(32) NOT NULL, + pattern_json MEDIUMTEXT NOT NULL, + target_type VARCHAR(16) NOT NULL, + target_id BIGINT NOT NULL, + payload_template MEDIUMTEXT, + rate_limit_per_min INT NOT NULL DEFAULT 60, + dedup_window_secs INT NOT NULL DEFAULT 60, + bot_self_filter TINYINT NOT NULL DEFAULT 1, + enabled TINYINT NOT NULL DEFAULT 1, + fire_count BIGINT NOT NULL DEFAULT 0, + max_fires BIGINT NOT NULL DEFAULT 0, + last_fired_at DATETIME(3), + pattern_version BIGINT NOT NULL DEFAULT 1, + create_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + update_time DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) ON UPDATE CURRENT_TIMESTAMP(3), + deleted INT NOT NULL DEFAULT 0, + KEY idx_trigger_workspace_enabled (workspace_id, enabled, deleted), + KEY idx_trigger_target (target_type, target_id) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Workflow / agent trigger definitions with pattern versioning.'; + +-- 8. Event dedup window. dedup_key is envelope.eventId, falling back to +-- sourceHash when the upstream channel did not provide a stable id. +CREATE TABLE IF NOT EXISTS mate_trigger_event ( + id BIGINT NOT NULL PRIMARY KEY, + trigger_id BIGINT NOT NULL, + dedup_key VARCHAR(128) NOT NULL, + received_at DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3), + expires_at DATETIME(3) NOT NULL, + UNIQUE KEY uk_trigger_dedup (trigger_id, dedup_key), + KEY idx_trigger_event_expires (expires_at) +) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci + COMMENT = 'Per-trigger event dedup window with TTL-style expiry.'; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V97__workflow_purge_tombstones.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V97__workflow_purge_tombstones.sql new file mode 100644 index 00000000..8b3a9125 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V97__workflow_purge_tombstones.sql @@ -0,0 +1,9 @@ +-- See the matching H2 file for context. The workflow / trigger entities +-- moved off @TableLogic to align with the project's hard-delete convention; +-- this migration drops any tombstones the old soft-delete path persisted so +-- list endpoints don't expose them after the annotation-driven filter is +-- removed. + +DELETE FROM mate_workflow WHERE deleted <> 0; +DELETE FROM mate_workflow_run WHERE deleted <> 0; +DELETE FROM mate_trigger WHERE deleted <> 0; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V98__trigger_last_error.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V98__trigger_last_error.sql new file mode 100644 index 00000000..b136783f --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V98__trigger_last_error.sql @@ -0,0 +1,29 @@ +-- See the H2 file for context. MySQL 8.0 doesn't support +-- `ADD COLUMN IF NOT EXISTS`, so the existence check goes through +-- INFORMATION_SCHEMA + a prepared statement. + +SET @col_exists := ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_trigger' + AND COLUMN_NAME = 'last_error' +); +SET @ddl := IF(@col_exists = 0, + 'ALTER TABLE mate_trigger ADD COLUMN last_error VARCHAR(2048)', + 'SELECT 1'); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; + +SET @col_exists := ( + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'mate_trigger' + AND COLUMN_NAME = 'last_dispatched_at' +); +SET @ddl := IF(@col_exists = 0, + 'ALTER TABLE mate_trigger ADD COLUMN last_dispatched_at TIMESTAMP NULL', + 'SELECT 1'); +PREPARE stmt FROM @ddl; +EXECUTE stmt; +DEALLOCATE PREPARE stmt; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V99__dashscope_compat_provider.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V99__dashscope_compat_provider.sql new file mode 100644 index 00000000..9de29fe0 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V99__dashscope_compat_provider.sql @@ -0,0 +1,51 @@ +-- V99: register a DashScope OpenAI-compatible provider entry alongside the +-- existing native dashscope provider, plus the dot-versioned Qwen families +-- (qwen3.5-*, qwen3.6-*) that only ship on compatible-mode/v1. +-- +-- See the H2 copy for full background. The MySQL copy uses INSERT ... ON +-- DUPLICATE KEY UPDATE; the api_key column is intentionally omitted from the +-- update list so existing deployments that have already configured a key keep +-- it (this only matters if a future migration re-applies a similar block; +-- Flyway runs each version once today). + +-- -- Provider -------------------------------------------------------------- +INSERT INTO mate_model_provider (provider_id, name, api_key_prefix, chat_model, api_key, base_url, generate_kwargs, is_custom, is_local, support_model_discovery, support_connection_check, freeze_url, require_api_key, create_time, update_time) +VALUES ( + 'dashscope-compat', + 'DashScope (兼容模式)', + 'sk-', + 'OpenAIChatModel', + '', + 'https://dashscope.aliyuncs.com/compatible-mode/v1', + '{}', + FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, + NOW(), NOW() +) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + api_key_prefix = VALUES(api_key_prefix), + chat_model = VALUES(chat_model), + base_url = VALUES(base_url), + generate_kwargs = VALUES(generate_kwargs), + support_model_discovery = VALUES(support_model_discovery), + support_connection_check = VALUES(support_connection_check), + freeze_url = VALUES(freeze_url), + require_api_key = VALUES(require_api_key), + update_time = VALUES(update_time); + +-- -- Model catalog --------------------------------------------------------- +-- Only seed the variants that are publicly callable on compatible-mode. The +-- -max / -vl-max variants exist in the marketplace but return 404 for general +-- accounts; users with whitelist access can add them via Settings → Models. +INSERT INTO mate_model_config (id, name, provider, model_name, description, temperature, max_tokens, top_p, builtin, enabled, is_default, create_time, update_time, deleted) +VALUES + (1000000601, 'Qwen3.6 Plus', 'dashscope-compat', 'qwen3.6-plus', '通义千问 3.6 Plus 旗舰,平衡推理与速度(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000603, 'Qwen3.5 Plus', 'dashscope-compat', 'qwen3.5-plus', '通义千问 3.5 Plus(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0), + (1000000605, 'Qwen3 VL Plus', 'dashscope-compat', 'qwen3-vl-plus', '通义千问 3 视觉理解 Plus,支持图像、视频输入(兼容模式专属)', 0.7, 4096, 0.8, TRUE, TRUE, FALSE, NOW(), NOW(), 0) +ON DUPLICATE KEY UPDATE + name = VALUES(name), + model_name = VALUES(model_name), + description = VALUES(description), + builtin = VALUES(builtin), + enabled = VALUES(enabled), + update_time = VALUES(update_time); diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index 2d012a77..fe18b3b4 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -146,6 +146,8 @@ err.auth.user_not_found=\u7528\u6237\u4e0d\u5b58\u5728 err.auth.wrong_password=\u539f\u5bc6\u7801\u9519\u8bef err.agent.not_found=Agent\u4e0d\u5b58\u5728 err.agent.disabled=Agent \u5df2\u7981\u7528 +err.agent.name_required=Agent \u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a +err.agent.duplicate_name=\u5f53\u524d\u5de5\u4f5c\u533a\u5df2\u5b58\u5728\u540c\u540d\u5458\u5de5\uff0c\u8bf7\u6362\u4e2a\u540d\u5b57\u518d\u8bd5 err.workspace.not_found=\u5de5\u4f5c\u533a\u4e0d\u5b58\u5728 err.workspace.slug_exists=\u5de5\u4f5c\u533a\u6807\u8bc6\u5df2\u5b58\u5728 err.workspace.cannot_modify_default=\u4e0d\u80fd\u4fee\u6539\u9ed8\u8ba4\u5de5\u4f5c\u533a\u7684\u6807\u8bc6 @@ -164,6 +166,7 @@ err.skill.not_found=\u6280\u80fd\u4e0d\u5b58\u5728 err.skill.name_required=\u6280\u80fd\u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a err.skill.name_exists=\u6280\u80fd\u540d\u79f0\u5df2\u5b58\u5728 err.skill.builtin_readonly=\u5185\u7f6e\u6280\u80fd\u4e0d\u53ef\u5220\u9664 +err.skill.cross_workspace_binding=\u4e0d\u80fd\u5c06\u5176\u5b83\u5de5\u4f5c\u533a\u7684\u6280\u80fd\u7ed1\u5b9a\u5230\u5f53\u524d Agent err.mcp.not_found=MCP server \u4e0d\u5b58\u5728 err.mcp.builtin_readonly=\u5185\u7f6e MCP server \u4e0d\u53ef\u5220\u9664 err.mcp.name_required=MCP server \u540d\u79f0\u4e0d\u80fd\u4e3a\u7a7a diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index fa383fe5..acd419f6 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -152,6 +152,8 @@ err.auth.wrong_password=Incorrect current password # agent err.agent.not_found=Agent not found err.agent.disabled=Agent is disabled +err.agent.name_required=Agent name is required +err.agent.duplicate_name=An employee with this name already exists in this workspace — try a different name # workspace err.workspace.not_found=Workspace not found err.workspace.slug_exists=Workspace slug already exists @@ -174,6 +176,7 @@ err.skill.not_found=Skill not found err.skill.name_required=Skill name cannot be empty err.skill.name_exists=Skill name already exists err.skill.builtin_readonly=Built-in skill cannot be deleted +err.skill.cross_workspace_binding=Cannot bind a skill from a different workspace to this Agent # mcp err.mcp.not_found=MCP server not found err.mcp.builtin_readonly=Built-in MCP server cannot be deleted diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-system.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-system.txt new file mode 100644 index 00000000..605cb321 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-system.txt @@ -0,0 +1,16 @@ +You are a senior synthesis editor merging several AI-generated extracts. +Each extract was produced by running the same template against a different +source material; your job is to produce one unified KB-level document. + +Rules: +- Merge entries that describe the same concept / theorem / clause / person / + signal. Keep the entry once but list every source that contributed it. +- Preserve the per-source output structure (headings, tables, bullets) but + compress all sources into one cohesive document, not a concatenation. +- Add a "Sources" section at the very top listing every source you merged, + with a one-line note on each. +- Within each merged entry, when a fact came from more than one source, + cite the source titles in parentheses. +- Never invent content. If sources disagree, surface the disagreement + rather than smoothing it over. +- Output only Markdown. No preamble. No closing remarks. diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-user.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-user.txt new file mode 100644 index 00000000..84683e8d --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-aggregate-user.txt @@ -0,0 +1,9 @@ +## Template + +**{template_title}** — {template_description} + +The per-source extracts below were all produced by running this template. + +## Per-source outputs + +{outputs} diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt new file mode 100644 index 00000000..34f1be05 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system-json.txt @@ -0,0 +1,16 @@ +You are a content transformation worker producing structured JSON. The user +supplies (a) a transformation instruction and (b) a source text. Follow +the instruction precisely and return exactly one valid JSON document. + +Rules: +- Return ONLY a JSON document — no prose, no commentary, no markdown code + fences. The first character of your reply must be `{` or `[`. +- Do not add framing like "Here is the JSON:" — emit the JSON object alone. +- Use only JSON-valid escapes; double-quote all strings. +- Do not invent facts beyond the supplied source text. If the source is + empty, return `{"error": "empty source"}`. +- Preserve the language of the source text in string values unless the + instruction explicitly says otherwise. +- If the instruction describes a schema (fields, arrays, types), follow + it exactly; missing values get an empty string or `null` per JSON + convention rather than being omitted entirely. diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt new file mode 100644 index 00000000..f80e9c5f --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-system.txt @@ -0,0 +1,10 @@ +You are a content transformation worker. The user supplies (a) a transformation +instruction and (b) a source text. Follow the instruction precisely and return +only the transformed content. + +Rules: +- Do not add framing such as "Here is the result:" — emit only the transformation output. +- If the instruction asks for JSON, return exactly one valid JSON document and nothing else. +- Otherwise return Markdown. +- Do not invent facts beyond the supplied source text. If the source is empty, return a one-line note saying so. +- Preserve the language of the source text unless the instruction explicitly says otherwise. diff --git a/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt b/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt new file mode 100644 index 00000000..c04d7895 --- /dev/null +++ b/mateclaw-server/src/main/resources/prompts/wiki/transformation-user.txt @@ -0,0 +1,7 @@ +## Instruction + +{instruction} + +## Source — {source_title} + +{source_text} diff --git a/mateclaw-server/src/main/resources/skills/apple-notes/SKILL.md b/mateclaw-server/src/main/resources/skills/apple-notes/SKILL.md index 46215d17..8f17441c 100644 --- a/mateclaw-server/src/main/resources/skills/apple-notes/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/apple-notes/SKILL.md @@ -2,6 +2,7 @@ name: apple-notes description: 'Manage Apple Notes via memo CLI: create, search, edit.' version: 1.0.0 +optional: true platforms: - macos requires: diff --git a/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md b/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md index f41af49e..ea209160 100644 --- a/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/architecture-diagram/SKILL.md @@ -41,7 +41,8 @@ Based on [Cocoon AI's architecture-diagram-generator](https://github.com/Cocoon- 1. User describes their system architecture (components, connections, technologies) 2. Generate the HTML file following the design system below 3. Save with `write_file` to a `.html` file (e.g. `~/architecture-diagram.html`) -4. User opens in any browser — works offline, no dependencies +4. **If the user wants to view/share the diagram in chat (web console, WeCom / 企业微信, DingTalk, Feishu, Telegram, ...): call `render_html_image(filePath="", filename="")`** and return the markdown link it produces. IM channels can only deliver rasterised images natively, so a PNG is required for the diagram to appear inline rather than as a dead link or a file attachment. +5. Otherwise, the user opens the `.html` directly in a browser — works offline, no dependencies. ### Output Location @@ -50,9 +51,19 @@ Save diagrams to a user-specified path, or default to the current working direct ./[project-name]-architecture.html ``` -### Preview +### Delivering through chat / IM channels -After saving, suggest the user open it: +When the current channel is anything other than a local browser session, follow up `write_file` with: + +``` +render_html_image(filePath="./architecture-diagram.html", filename="architecture") +``` + +This returns a `/api/v1/files/generated/` URL with `image/png` MIME. The channel layer detects the image MIME and uploads the PNG as a native image message (so it renders inline in WeCom / DingTalk / Feishu / Telegram / Web). Without this step, an `.html` artifact reaches IM channels as either a dead markdown link or, at best, a non-previewable file attachment. + +### Local preview + +After saving, the user can open the `.html` directly: ```bash # macOS open ./my-architecture.html diff --git a/mateclaw-server/src/main/resources/skills/blogwatcher/SKILL.md b/mateclaw-server/src/main/resources/skills/blogwatcher/SKILL.md index 9d4e6cbb..7563fb23 100644 --- a/mateclaw-server/src/main/resources/skills/blogwatcher/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/blogwatcher/SKILL.md @@ -2,6 +2,7 @@ name: blogwatcher description: Monitor blogs and RSS/Atom feeds via blogwatcher-cli tool. version: 2.0.0 +optional: true requires: - key: blogwatcher-cli type: binary diff --git a/mateclaw-server/src/main/resources/skills/ckjia-shopping/SKILL.md b/mateclaw-server/src/main/resources/skills/ckjia-shopping/SKILL.md index 19072144..7cdc57d5 100644 --- a/mateclaw-server/src/main/resources/skills/ckjia-shopping/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/ckjia-shopping/SKILL.md @@ -3,6 +3,7 @@ name: ckjia-shopping nameZh: 参考价 - 比价购物 nameEn: CKJIA Shopping version: "1.0.1" +optional: true icon: /skill-assets/ckjia-shopping/assets/ckjia_app_icon.png description: "跨平台比价与购物推荐 / Cross-platform price comparison. 淘宝 / 京东 / 天猫 / 拼多多商品聚合搜索 + 拍图识物。需要先启用 ckjia-shopping MCP server 并配置 CKJIA_MCP_KEY 才能用。" category: data diff --git a/mateclaw-server/src/main/resources/skills/dingtalk_channel_connect/SKILL.md b/mateclaw-server/src/main/resources/skills/dingtalk_channel_connect/SKILL.md index 8a125843..94af30fd 100644 --- a/mateclaw-server/src/main/resources/skills/dingtalk_channel_connect/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/dingtalk_channel_connect/SKILL.md @@ -1,6 +1,7 @@ --- name: dingtalk_channel_connect version: "1.3.0" +optional: true description: "使用可见浏览器自动完成 MateClaw 钉钉渠道接入。遇到登录页必须暂停等待用户手动登录后继续。" dependencies: tools: diff --git a/mateclaw-server/src/main/resources/skills/himalaya/SKILL.md b/mateclaw-server/src/main/resources/skills/himalaya/SKILL.md index 6f7cb5df..8eca6fca 100644 --- a/mateclaw-server/src/main/resources/skills/himalaya/SKILL.md +++ b/mateclaw-server/src/main/resources/skills/himalaya/SKILL.md @@ -1,6 +1,7 @@ --- name: himalaya description: "CLI to manage emails via IMAP/SMTP. Use himalaya to list, read, write, reply, forward, search, and organize emails from the terminal. Supports multiple accounts and message composition with MML (MIME Meta Language)." +optional: true dependencies: commands: - himalaya diff --git a/mateclaw-server/src/main/resources/skills/x_intel/SKILL.md b/mateclaw-server/src/main/resources/skills/x_intel/SKILL.md new file mode 100644 index 00000000..19a80eaa --- /dev/null +++ b/mateclaw-server/src/main/resources/skills/x_intel/SKILL.md @@ -0,0 +1,275 @@ +--- +name: x_intel +description: "Read X (Twitter) posts, search, timelines and user profiles via the official xurl CLI." +nameZh: X 情报采集 +nameEn: X Intel +version: 1.0.0 +icon: 🐦 +author: MateClaw +optional: true +tags: + - x + - twitter + - social-media + - research + - xurl +platforms: + - linux + - macos +dependencies: + commands: + - xurl + tools: + - execute_shell_command +--- + +# x_intel — X (Twitter) information gathering + +`x_intel` lets an agent pull posts, search results, timelines and user profiles from X (Twitter) through `xurl`, the X developer platform's official CLI. **This skill is read-only by design** — it intentionally omits posting, replying, deleting, DM-sending and any other write surface. For a separate publishing skill, see follow-up work. + +Use this skill for: + +- looking up a single post by ID or URL +- searching posts with the X search query syntax (`from:user`, `lang:en`, `#hashtag`, ...) +- reading the agent operator's home timeline, mentions, bookmarks, likes +- inspecting a user profile by handle +- walking the social graph (who someone follows / is followed by) +- raw read access to any X API v2 GET endpoint when the shortcuts don't fit + +--- + +## Credential safety (mandatory) + +Critical rules when invoked inside an agent session: + +- **Never** read, print, parse, summarize, upload or quote `~/.xurl` into chat context. It is a YAML token store. +- **Never** ask the user to paste credentials/tokens into the conversation. +- **Never** suggest or run the auth commands with inline secrets in an agent session. +- **Never** pass `--verbose` / `-v` — it prints auth headers to stdout. +- The only credential-touching command this skill ever runs is `xurl auth status` (status only, no secrets). + +Forbidden flags in any agent-issued command (each accepts inline secrets): +`--bearer-token`, `--consumer-key`, `--consumer-secret`, `--access-token`, `--token-secret`, `--client-id`, `--client-secret`. + +App registration and the OAuth 2.0 PKCE flow must be performed by the user **outside** the agent session (see "User setup" below). Tokens persist in `~/.xurl` (YAML); OAuth 2.0 refreshes automatically. + +--- + +## Install + +The agent should verify, not install. Direct the user to install if missing. + +```bash +# Shell script (Linux + macOS, installs to ~/.local/bin, no sudo) +curl -fsSL https://raw.githubusercontent.com/xdevplatform/xurl/main/install.sh | bash + +# Homebrew (macOS) +brew install --cask xdevplatform/tap/xurl + +# Go (cross-platform) +go install github.com/xdevplatform/xurl@latest +``` + +Verify: + +```bash +xurl --help +xurl auth status +``` + +--- + +## User setup (user runs these, NOT the agent) + +The agent must not perform these steps — they involve pasting secrets. Direct the user to this section verbatim. + +1. Open the X developer dashboard: +2. In the app's User Authentication Settings, set the redirect URI to `http://localhost:8080/callback` and the app type to **Web app, automated app or bot**. +3. Copy the app's Client ID and Client Secret. +4. Register the app locally: + ```bash + xurl auth apps add my-app --client-id YOUR_CLIENT_ID --client-secret YOUR_CLIENT_SECRET + ``` +5. Authenticate (this opens a browser for OAuth 2.0 PKCE): + ```bash + xurl auth oauth2 --app my-app + ``` + If X returns `UsernameNotFound` or a 403 on the post-OAuth `/2/users/me` lookup, pass the handle explicitly (xurl v1.1.0+): + ```bash + xurl auth oauth2 --app my-app YOUR_HANDLE + ``` +6. Mark this app as the default so all commands use it: + ```bash + xurl auth default my-app + ``` +7. Verify: + ```bash + xurl auth status + xurl whoami + ``` + +> **Most common mistake:** omitting `--app my-app` from `xurl auth oauth2`. The OAuth token then lands in the built-in `default` profile, which has no client-id/client-secret, and every later read fails. Re-run `xurl auth oauth2 --app my-app` and `xurl auth default my-app` to fix. + +--- + +## Read-only command reference + +All commands return JSON to stdout. The agent parses JSON directly; no extra tooling needed. + +| Action | Command | +| --- | --- | +| Who is the bound account | `xurl whoami` | +| Look up a user | `xurl user @handle` | +| Read one post (ID or URL) | `xurl read POST_ID` | +| Search posts | `xurl search "QUERY" -n 10` | +| Home timeline | `xurl timeline -n 20` | +| Mentions of bound account | `xurl mentions -n 20` | +| Bookmarks list | `xurl bookmarks -n 20` | +| Likes list | `xurl likes -n 20` | +| Following list | `xurl following -n 50` | +| Followers list | `xurl followers -n 50` | +| Another user's graph | `xurl following --of HANDLE -n 20` | +| Auth status | `xurl auth status` | + +Notes: + +- `POST_ID` accepts a full `https://x.com/user/status/...` URL — xurl extracts the ID. +- Handles work with or without the leading `@`. + +### Search query language + +X's search supports operators inside the quoted query string: + +```bash +xurl search "from:elonmusk -is:retweet" -n 20 +xurl search "#buildinpublic lang:en since:2026-01-01" -n 25 +xurl search "OR" -n 10 # literal OR — must be quoted +xurl search "(rust OR go) lang:en" -n 10 +xurl search "to:NASA -is:reply" -n 10 +``` + +Common operators: `from:`, `to:`, `@`, `#`, `is:retweet`, `is:reply`, `is:quote`, `lang:`, `since:`, `until:`, `has:media`, `has:links`. See the X search syntax docs for the full list. + +--- + +## Raw v2 read access + +For anything beyond the shortcuts, hit any v2 GET endpoint directly: + +```bash +# Public user fields +xurl /2/users/by/username/elonmusk?user.fields=public_metrics,description,verified + +# Single tweet with metrics + author expansion +xurl /2/tweets/1234567890?tweet.fields=public_metrics,created_at&expansions=author_id + +# Recent search with extra fields (paid tier) +xurl /2/tweets/search/recent?query=langchain&tweet.fields=created_at,public_metrics&max_results=25 + +# Full URLs also work +xurl https://api.x.com/2/users/me +``` + +Streaming endpoints are auto-detected; force with `-s` if needed. **Streaming endpoints can be expensive — do not start one without confirming intent with the user.** + +--- + +## Common workflows + +### Profile a user + +```bash +xurl user @handle +xurl /2/users/by/username/handle?user.fields=public_metrics,description,verified,created_at +xurl following --of handle -n 20 # who they pay attention to +``` + +### Triage a trending term + +```bash +xurl search "topic lang:en -is:retweet" -n 25 +# Pick interesting IDs from the JSON, then drill in: +xurl read 1234567890 +xurl user @ORIGINAL_POSTER +``` + +### Catch up on activity + +```bash +xurl whoami +xurl mentions -n 20 +xurl timeline -n 20 +xurl bookmarks -n 10 +``` + +### Conversation context + +```bash +xurl read https://x.com/user/status/1234567890 +# Conversation expansion via raw v2 +xurl /2/tweets/search/recent?query=conversation_id:1234567890&max_results=25 +``` + +--- + +## Output format + +Every command emits X API v2 shape JSON to stdout: + +```json +{ + "data": { "id": "1234567890", "text": "Hello world!" }, + "includes": { "users": [{ "id": "...", "username": "..." }] } +} +``` + +Errors are also JSON: + +```json +{ "errors": [ { "message": "Not authorized", "code": 403 } ] } +``` + +The non-zero exit code distinguishes errors from empty results. + +--- + +## Agent workflow + +1. Verify prerequisites: `xurl --help` (the command exists) and `xurl auth status` (the user has at least one app with `oauth2` tokens, marked `▸` as default). +2. **Parse `auth status` output before any other command.** If the default app shows `oauth2: (none)` but a non-default app has valid tokens, instruct the user to run `xurl auth default ` — this is the most common config glitch and does not require a re-login. +3. If `auth status` shows no apps or no tokens, **stop**. Tell the user to follow the "User setup" section. Do not attempt to register apps or run any auth flow yourself. +4. Start with the cheapest read first (`xurl whoami` / `xurl user @handle` / `xurl search ... -n 3`) to confirm reachability and the request shape. +5. Treat 401 / 403 / 429 distinctly: 401 → re-auth needed, 403 → scope or plan, 429 → wait and retry (X rate-limits per-endpoint). +6. Never paste `~/.xurl` content back into the conversation, even when troubleshooting. +7. When in doubt about cost: X's API has paid tiers and per-endpoint rate limits. Do not run unbounded loops or streams without the user's explicit confirmation. + +--- + +## Troubleshooting + +| Symptom | Cause | Fix | +| --- | --- | --- | +| `auth status` shows `oauth2: (none)` on default | Token saved to built-in `default` profile (no client-id/secret) | Re-run `xurl auth oauth2 --app my-app` then `xurl auth default my-app` | +| `unauthorized_client` during OAuth | App type set to "Native App" in X dashboard | Change to "Web app, automated app or bot" | +| `UsernameNotFound` / 403 right after OAuth | X not returning username from `/2/users/me` | `xurl auth oauth2 --app my-app YOUR_HANDLE` (xurl v1.1.0+) | +| 401 on every read | Token expired or wrong default app | Check `xurl auth status` — verify `▸` points to the app with oauth2 tokens | +| `client-forbidden` / `client-not-enrolled` | X platform enrollment | Developer dashboard → Apps → Manage → Production environment | +| `CreditsDepleted` | $0 balance on X API | Buy credits in Developer Console → Billing | +| 429 on search/timeline | Hit per-endpoint rate limit | Pause, retry with smaller `-n`, or wait for the reset window | + +--- + +## Notes + +- **Cost:** X API access is paid for meaningful usage. Many failures are plan or rate-limit problems, not skill problems. +- **Scopes:** OAuth 2.0 tokens use broad scopes; a 403 on a specific read usually means the token is missing a scope — have the user re-run `xurl auth oauth2`. +- **Token refresh:** OAuth 2.0 tokens auto-refresh; nothing to do. +- **Multiple apps:** `xurl --app NAME ...` runs one read against a specific app without changing the default. +- **Token storage:** `~/.xurl` is YAML. Treat it like a private key. Never read or send it to LLM context. + +--- + +## Attribution + +- Underlying CLI: (X developer platform). +- This skill wraps the CLI's read commands and documents agent-side safety rules. No code is shipped beyond this SKILL.md. diff --git a/mateclaw-server/src/main/resources/templates/code-reviewer.json b/mateclaw-server/src/main/resources/templates/code-reviewer.json index cca50445..78c467c5 100644 --- a/mateclaw-server/src/main/resources/templates/code-reviewer.json +++ b/mateclaw-server/src/main/resources/templates/code-reviewer.json @@ -8,6 +8,12 @@ "agentType": "react", "tags": "code,review,developer", "maxIterations": 10, + "defaultSkillSlugs": [ + "systematic-debugging", + "test-driven-development", + "requesting-code-review", + "subagent-driven-development" + ], "systemPrompt": "## Role\n代码审查员\n\n## Goal\n找到不该在 PR 里的东西\n\n## Backstory\n你是见过太多周五下午合并事故的资深审查员。你信奉一条:被合进 main 的代码,要么经得起半年后的回头看,要么不该进。你读代码先读改动的边界——它影响哪些调用方、哪些边缘情况、哪些隐藏假设。你直接但不刻薄,每一条意见都附上修法。\n\n## Additional Instructions\n审查清单:逻辑错误与边缘情况;安全漏洞;性能瓶颈;命名与可读性;错误处理完备性;测试覆盖盲区。先读完整段再下结论,不要只看 diff 的±号。\n", "workspaceFiles": [ { diff --git a/mateclaw-server/src/main/resources/templates/data-analyst.json b/mateclaw-server/src/main/resources/templates/data-analyst.json index 817accd2..52c81ebb 100644 --- a/mateclaw-server/src/main/resources/templates/data-analyst.json +++ b/mateclaw-server/src/main/resources/templates/data-analyst.json @@ -8,6 +8,10 @@ "agentType": "react", "tags": "data,analysis,sql", "maxIterations": 12, + "defaultSkillSlugs": [ + "sql_query", + "xlsx" + ], "systemPrompt": "## Role\n数据分析师\n\n## Goal\n把数据变成可执行的洞察\n\n## Backstory\n你在数据里待了十年。最大的体会是:80% 的烂分析栽在第一步——问题没问对。所以你拿到任何需求都先停一下,确认\"我们到底想知道什么\",再决定要拉哪张表。你写 SQL 简洁、加注释,不堆 CTE 炫技。出结论时永远带数据范围、口径定义和置信度。\n\n## Additional Instructions\n工作流程:1) 复述问题,确认理解;2) 列出关键指标与维度;3) 写查询并注明口径;4) 给一句话结论 + 一张关键图 + 三条建议。不要把表格堆给用户,要把判断给他。\n", "workspaceFiles": [ { diff --git a/mateclaw-server/src/main/resources/templates/product-assistant.json b/mateclaw-server/src/main/resources/templates/product-assistant.json index e78f7513..74f96d0c 100644 --- a/mateclaw-server/src/main/resources/templates/product-assistant.json +++ b/mateclaw-server/src/main/resources/templates/product-assistant.json @@ -8,6 +8,10 @@ "agentType": "react", "tags": "product,prd,requirements", "maxIterations": 12, + "defaultSkillSlugs": [ + "ideation", + "make_plan" + ], "systemPrompt": "## Role\n产品助理\n\n## Goal\n把模糊需求理成可执行的 PRD\n\n## Backstory\n你做产品做久了,知道一句话需求背后通常藏着三个不一样的问题。所以你拿到任何描述,先把它翻译成\"用户是谁 + 他在什么场景下 + 他想达成什么 + 现在的痛点是什么\"。你写 PRD 不堆功能列表,会先讲清楚\"不做什么\"和\"成功长什么样\"。\n\n## Additional Instructions\n输出结构:1) 用户与场景;2) 目标与反目标(不做什么);3) 核心流程;4) 验收标准。一段话能讲清的不用列表,能列清的不用图。讲清楚\"为什么\"比讲清楚\"做什么\"更重要。\n", "workspaceFiles": [ { diff --git a/mateclaw-server/src/main/resources/templates/research-analyst.json b/mateclaw-server/src/main/resources/templates/research-analyst.json index ae7b7e0b..8e9575fc 100644 --- a/mateclaw-server/src/main/resources/templates/research-analyst.json +++ b/mateclaw-server/src/main/resources/templates/research-analyst.json @@ -8,6 +8,11 @@ "agentType": "plan_execute", "tags": "research,analysis,planning", "maxIterations": 20, + "defaultSkillSlugs": [ + "arxiv", + "news", + "x_intel" + ], "systemPrompt": "## Role\n研究分析员\n\n## Goal\n把信息整理成可下结论的判断\n\n## Backstory\n你像图书馆员一样固执——没有可信来源,你不下结论。你做研究的步骤是固定的:先把大问题拆成可独立查证的小问题,再分别取证,最后交叉对照。看到两个来源说反话,你不会偷偷选一个,会原样列出并标注分歧。\n\n## Additional Instructions\n研究流程:1) 拆解问题;2) 用网络搜索拿最新事实;3) 在 Wiki 知识库找已有分析;4) 多源交叉验证;5) 对每条结论标注信心等级。准确高于速度。证据不足时直接说\"我不知道\"。\n", "workspaceFiles": [ { diff --git a/mateclaw-server/src/test/java/vip/mate/acp/client/AcpStdioClientTest.java b/mateclaw-server/src/test/java/vip/mate/acp/client/AcpStdioClientTest.java new file mode 100644 index 00000000..ec94b2a5 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/acp/client/AcpStdioClientTest.java @@ -0,0 +1,89 @@ +package vip.mate.acp.client; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-090 Phase 7 — connection-test smoke for {@link AcpStdioClient}. + * + *

Runs a tiny shell-script "agent" that mimics the {@code initialize} + * handshake: reads one JSON-RPC request, replies with a matching id and + * the expected protocol version. Locks in: + *

    + *
  • spawn → request → response → close all happen cleanly,
  • + *
  • protocolVersion is parsed from the result,
  • + *
  • the reader thread doesn't leak past close.
  • + *
+ * + *

POSIX-only: relies on {@code sh} + executable bit. Windows agents + * are exercised via the real CLI integration smoke (manual). The + * client itself is OS-neutral; the script harness is what's POSIXy. + */ +@DisabledOnOs(OS.WINDOWS) +class AcpStdioClientTest { + + @Test + @DisplayName("initialize handshake completes against a scripted agent") + void initializeHandshake() throws Exception { + Path script = writeScriptedAgent(); + try (AcpStdioClient client = AcpStdioClient.spawn( + new ObjectMapper(), "sh", List.of(script.toString()), + AcpStdioClient.emptyEnv(), null)) { + JsonNode result = client.initialize(5_000); + assertNotNull(result); + assertEquals(AcpStdioClient.PROTOCOL_VERSION, + result.path("protocolVersion").asInt()); + } finally { + Files.deleteIfExists(script); + } + } + + @Test + @DisplayName("spawn fails fast for a missing command") + void spawnFailsFastForMissingCommand() { + assertThrows(IOException.class, () -> + AcpStdioClient.spawn(new ObjectMapper(), + "/definitely/does/not/exist/acp-test-bin", + List.of(), AcpStdioClient.emptyEnv(), null)); + } + + /** + * Tiny shell-script agent: read one JSON-RPC line on stdin and + * write a response with a hard-coded result. Just enough surface + * to exercise the framing path. + */ + private Path writeScriptedAgent() throws IOException { + Path script = Files.createTempFile("acp-fake-agent-", ".sh"); + String body = "" + + "#!/bin/sh\n" + + "read line\n" + + // Pull the id; assume integer id at this position. + "id=$(printf '%s' \"$line\" | sed -n 's/.*\"id\":\\([0-9]\\+\\).*/\\1/p')\n" + + "if [ -z \"$id\" ]; then id=1; fi\n" + + "printf '{\"jsonrpc\":\"2.0\",\"id\":%s,\"result\":{\"protocolVersion\":1,\"agentCapabilities\":{}}}\\n' \"$id\"\n"; + Files.writeString(script, body, StandardCharsets.UTF_8); + try { + Files.setPosixFilePermissions(script, Set.of( + PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE)); + } catch (UnsupportedOperationException ignore) { + // Filesystem doesn't support POSIX perms — sh ... still works. + } + return script; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java b/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java new file mode 100644 index 00000000..c2c10749 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/AgentGraphBuilderPreferenceTest.java @@ -0,0 +1,78 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.model.ModelProviderEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * RFC-009 PR-3 — verifies the agent-preference reorder used by + * {@link AgentGraphBuilder#buildFallbackChain}: listed providers move to the + * front in their declared order; unlisted providers keep their original + * relative order; missing/duplicate preferences are ignored gracefully. + */ +class AgentGraphBuilderPreferenceTest { + + private static ModelProviderEntity p(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + return p; + } + + private static List ids(List ps) { + return ps.stream().map(ModelProviderEntity::getProviderId).toList(); + } + + @Test + @DisplayName("Empty preferences: original order preserved") + void noPreferences() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of()); + assertEquals(List.of("openai", "anthropic", "dashscope"), ids(out)); + } + + @Test + @DisplayName("Single preference: preferred provider moves to front, rest follow original order") + void singlePreferenceFront() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("dashscope")); + assertEquals(List.of("dashscope", "openai", "anthropic"), ids(out)); + } + + @Test + @DisplayName("Multiple preferences: preferred order matches declaration, rest stable") + void multiplePreferencesOrder() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope"), p("kimi")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("kimi", "anthropic")); + // kimi → anthropic → (rest in original order: openai, dashscope) + assertEquals(List.of("kimi", "anthropic", "openai", "dashscope"), ids(out)); + } + + @Test + @DisplayName("Preference references unknown provider: silently skipped") + void preferenceReferencesUnknown() { + var input = List.of(p("openai"), p("anthropic")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("ghost", "anthropic")); + assertEquals(List.of("anthropic", "openai"), ids(out)); + } + + @Test + @DisplayName("Duplicate preferences: each provider appears at most once") + void duplicatePreferencesDeduped() { + var input = List.of(p("openai"), p("anthropic")); + var out = AgentGraphBuilder.reorderByPreferences(input, List.of("openai", "openai", "anthropic")); + assertEquals(List.of("openai", "anthropic"), ids(out)); + } + + @Test + @DisplayName("All providers preferred: input pure-reordered, no drops") + void allProvidersPreferred() { + var input = List.of(p("openai"), p("anthropic"), p("dashscope")); + var out = AgentGraphBuilder.reorderByPreferences(input, + List.of("dashscope", "openai", "anthropic")); + assertEquals(List.of("dashscope", "openai", "anthropic"), ids(out)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/AgentServiceUniquenessTest.java b/mateclaw-server/src/test/java/vip/mate/agent/AgentServiceUniquenessTest.java new file mode 100644 index 00000000..1e5957c7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/AgentServiceUniquenessTest.java @@ -0,0 +1,145 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.agent.model.AgentEntity; +import vip.mate.exception.MateClawException; + +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pre-flight {@code (workspace_id, name)} uniqueness check that + * accompanies the V102 unique index. + * + *

The DB index alone would surface as {@code DataIntegrityViolation} + * with a vendor-specific message; the service-layer pre-check converts + * that to a stable {@code err.agent.duplicate_name} business code so the + * UI can show a localized message and clients can branch deterministically. + * These tests pin both the rejection paths and the false-positive guard + * (a same-name UPDATE on the row itself must not block). + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:agent_unique_test_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none" +}) +class AgentServiceUniquenessTest { + + /** + * Tests share one in-memory DB instance (a single + * {@code @SpringBootTest} class shares its application context across + * methods). Workspace ids are derived from this counter so concurrent + * methods can't trample one another's rows. + */ + private static final AtomicLong WS_SEQ = new AtomicLong(50_000L); + + @Autowired + private AgentService agentService; + + private long workspaceA; + private long workspaceB; + + @BeforeEach + void setUp() { + workspaceA = WS_SEQ.getAndIncrement(); + workspaceB = WS_SEQ.getAndIncrement(); + } + + private AgentEntity newAgent(String name, long workspaceId) { + AgentEntity a = new AgentEntity(); + a.setName(name); + a.setDescription("uniqueness test agent"); + a.setAgentType("react"); + a.setSystemPrompt(""); + a.setMaxIterations(10); + a.setWorkspaceId(workspaceId); + return a; + } + + @Test + @DisplayName("createAgent 拒绝同 workspace 同名(body code = 409 / msgKey = err.agent.duplicate_name)") + void createRejectsDuplicateNameInSameWorkspace() { + agentService.createAgent(newAgent("Alpha", workspaceA)); + + MateClawException ex = assertThrows(MateClawException.class, + () -> agentService.createAgent(newAgent("Alpha", workspaceA))); + assertEquals(409, ex.getCode(), "应返回 409 业务码"); + assertEquals("err.agent.duplicate_name", ex.getMsgKey()); + } + + @Test + @DisplayName("createAgent 允许不同 workspace 同名(隔离边界生效)") + void createAllowsSameNameInDifferentWorkspace() { + AgentEntity a = agentService.createAgent(newAgent("Bravo", workspaceA)); + AgentEntity b = agentService.createAgent(newAgent("Bravo", workspaceB)); + + assertNotNull(a.getId()); + assertNotNull(b.getId()); + assertNotEquals(a.getId(), b.getId()); + } + + @Test + @DisplayName("createAgent 拒绝空名(fail-fast 在 unique 检查之前)") + void createRejectsBlankName() { + AgentEntity blank = newAgent(null, workspaceA); + MateClawException ex = assertThrows(MateClawException.class, + () -> agentService.createAgent(blank)); + assertEquals(400, ex.getCode()); + assertEquals("err.agent.name_required", ex.getMsgKey()); + } + + @Test + @DisplayName("updateAgent 拒绝把名字改成 workspace 内已有的别人") + void updateRejectsRenamingToExistingName() { + AgentEntity first = agentService.createAgent(newAgent("Charlie", workspaceA)); + AgentEntity second = agentService.createAgent(newAgent("Delta", workspaceA)); + + // Try renaming "Delta" → "Charlie" inside the same workspace. + second.setName("Charlie"); + MateClawException ex = assertThrows(MateClawException.class, + () -> agentService.updateAgent(second)); + assertEquals(409, ex.getCode()); + assertEquals("err.agent.duplicate_name", ex.getMsgKey()); + + // The other row must not have been touched. + assertEquals("Charlie", agentService.getAgent(first.getId()).getName()); + } + + @Test + @DisplayName("updateAgent 元数据修改不触发误报(excludeId 跳过自己)") + void updateAllowsMetadataEditWithoutFalsePositive() { + AgentEntity created = agentService.createAgent(newAgent("Echo", workspaceA)); + + // Edit description only, keep the same name. The unique check + // should detect "no name change" and skip the SELECT entirely; + // even if it didn't, the excludeId branch would filter self out. + created.setDescription("edited"); + assertDoesNotThrow(() -> agentService.updateAgent(created)); + assertEquals("edited", agentService.getAgent(created.getId()).getDescription()); + } + + @Test + @DisplayName("updateAgent 改名为新值(不冲突)允许") + void updateAllowsRenameToUnusedName() { + AgentEntity created = agentService.createAgent(newAgent("Foxtrot", workspaceA)); + created.setName("Foxtrot-renamed"); + assertDoesNotThrow(() -> agentService.updateAgent(created)); + assertEquals("Foxtrot-renamed", + agentService.getAgent(created.getId()).getName()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/AgentToolSetTest.java b/mateclaw-server/src/test/java/vip/mate/agent/AgentToolSetTest.java new file mode 100644 index 00000000..091ac479 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/AgentToolSetTest.java @@ -0,0 +1,123 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.support.ToolCallbacks; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.annotation.Tool; +import org.springframework.ai.tool.annotation.ToolParam; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Regression test for issue #24. + *

+ * Symptom: agent tool bindings persisted under the Java class name + * (e.g. {@code BrowserUseTool}, which is what {@code mate_tool.name} stores) or the + * Spring bean name (e.g. {@code browserUseTool}) had no effect at runtime, because the + * graph runtime matches against the {@code @Tool} function name (e.g. {@code browser_use}). + *

+ * Fix: {@link AgentToolSet} builds an alias index so any of these three identifiers + * resolves to the same callback. This test pins that contract. + */ +class AgentToolSetTest { + + /** Fixture: a bean exposing two {@code @Tool} methods, mirroring real tools like + * {@code BrowserUseTool} ({@code browser_use}, {@code browser_screenshot}, ...). */ + static class FakeBrowserTool { + @Tool(description = "Open a URL in the browser") + public String browser_use(@ToolParam(description = "url to open") String url) { + return "opened " + url; + } + + @Tool(description = "Take a screenshot") + public String browser_screenshot() { + return "shot.png"; + } + } + + @Test + @DisplayName("Issue #24: class name and bean name resolve to the same callbacks as the @Tool function name") + void aliasIndex_resolvesClassNameAndBeanNameToFunctionCallbacks() { + FakeBrowserTool bean = new FakeBrowserTool(); + List callbacks = List.of(ToolCallbacks.from(bean)); + assertEquals(2, callbacks.size(), "fixture should expose 2 @Tool methods"); + + AgentToolSet base = AgentToolSet.fromCallbacks( + List.of(bean), + callbacks, + b -> "fakeBrowserTool" // simulate Spring bean-name lookup + ); + assertEquals(2, base.size()); + + // (A) Function name — the historically-correct form + AgentToolSet byFn = base.withAllowedToolsOnly(Set.of("browser_use")); + assertEquals(1, byFn.size()); + assertEquals("browser_use", byFn.callbacks().get(0).getToolDefinition().name()); + + // (B) Spring bean name → expands to ALL @Tool methods on that bean + AgentToolSet byBean = base.withAllowedToolsOnly(Set.of("fakeBrowserTool")); + assertEquals(2, byBean.size(), + "bean name should pull in every @Tool method on the class"); + + // (C) Java class simple name (this is what mate_tool.name actually stores — + // e.g. 'BrowserUseTool' — and what the legacy bug saved into mate_agent_tool.tool_name) + AgentToolSet byClass = base.withAllowedToolsOnly(Set.of("FakeBrowserTool")); + assertEquals(2, byClass.size(), + "class simple name should expand to all bean methods (this is the issue #24 fix)"); + + // (D) Mixed: known + unknown aliases. Unknowns are silently dropped — callers persist + // stale data and we'd rather degrade gracefully than throw. + AgentToolSet mixed = base.withAllowedToolsOnly(Set.of("FakeBrowserTool", "nonexistent_tool")); + assertEquals(2, mixed.size()); + + // (E) Empty allow-list yields empty tool set (NOT global default — only null does that) + AgentToolSet none = base.withAllowedToolsOnly(Set.of()); + assertEquals(0, none.size()); + + // (F) null = no per-agent binding → fall back to global default (every tool visible) + AgentToolSet allDefault = base.withAllowedToolsOnly(null); + assertEquals(2, allDefault.size()); + } + + @Test + @DisplayName("withDeniedToolsFiltered accepts function / bean / class names interchangeably") + void deniedAliases_areToleranceOfNamingConvention() { + FakeBrowserTool bean = new FakeBrowserTool(); + List callbacks = List.of(ToolCallbacks.from(bean)); + + AgentToolSet base = AgentToolSet.fromCallbacks( + List.of(bean), + callbacks, + b -> "fakeBrowserTool" + ); + + // Deny by class name: removes both @Tool methods on that class + AgentToolSet none = base.withDeniedToolsFiltered(Set.of("FakeBrowserTool")); + assertEquals(0, none.size()); + + // Deny by single function name: only that method drops + AgentToolSet justOne = base.withDeniedToolsFiltered(Set.of("browser_screenshot")); + assertEquals(1, justOne.size()); + assertEquals("browser_use", justOne.callbacks().get(0).getToolDefinition().name()); + } + + @Test + @DisplayName("Two-arg fromCallbacks (no bean-name resolver): function name + class simple name still indexed (Spring bean name is not)") + void twoArgFactory_indexesByFunctionNameAndClassName() { + FakeBrowserTool bean = new FakeBrowserTool(); + List callbacks = List.of(ToolCallbacks.from(bean)); + + AgentToolSet noResolver = AgentToolSet.fromCallbacks(List.of(bean), callbacks); + + // Function name resolves + assertEquals(1, noResolver.withAllowedToolsOnly(Set.of("browser_use")).size()); + // Class simple name resolves too — derived from bean.getClass() reflection, no resolver needed + assertEquals(2, noResolver.withAllowedToolsOnly(Set.of("FakeBrowserTool")).size()); + // Spring bean name does NOT resolve without a resolver (no source for it) + assertEquals(0, noResolver.withAllowedToolsOnly(Set.of("fakeBrowserTool")).size()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/AssistantThinkingRelayTest.java b/mateclaw-server/src/test/java/vip/mate/agent/AssistantThinkingRelayTest.java new file mode 100644 index 00000000..63f858fe --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/AssistantThinkingRelayTest.java @@ -0,0 +1,139 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-049 PR-2: {@link AssistantThinkingRelay} — RelayEntry carries both + * per-assistant thinking and the caller's original {@code user} field, so the + * consumer can restore it when rebuilding the outbound request. + */ +class AssistantThinkingRelayTest { + + @BeforeEach + void clear() { + AssistantThinkingRelay.clearAll(); + } + + @AfterEach + void tearDown() { + AssistantThinkingRelay.clearAll(); + } + + @Test + @DisplayName("stash returns token with expected prefix") + void stash_returnsTokenWithPrefix() { + String token = AssistantThinkingRelay.stash(List.of("thinking-a"), null); + assertTrue(AssistantThinkingRelay.isToken(token)); + assertTrue(token.startsWith(AssistantThinkingRelay.TOKEN_PREFIX)); + } + + @Test + @DisplayName("stash + take roundtrips thinkings in order and originalUser") + void stashTake_roundtrip() { + List thinkings = List.of("one", "", "three"); + String token = AssistantThinkingRelay.stash(thinkings, "caller-user-42"); + AssistantThinkingRelay.RelayEntry entry = AssistantThinkingRelay.take(token); + + assertNotNull(entry); + assertEquals(List.of("one", "", "three"), entry.thinkings()); + assertEquals("caller-user-42", entry.originalUser()); + } + + @Test + @DisplayName("stash + take with null originalUser preserves null") + void stashTake_nullOriginalUser() { + String token = AssistantThinkingRelay.stash(List.of("x"), null); + AssistantThinkingRelay.RelayEntry entry = AssistantThinkingRelay.take(token); + assertNotNull(entry); + assertNull(entry.originalUser()); + } + + @Test + @DisplayName("take removes entry — subsequent take returns null") + void take_removesEntry() { + String token = AssistantThinkingRelay.stash(List.of("x"), "u"); + assertNotNull(AssistantThinkingRelay.take(token)); + assertNull(AssistantThinkingRelay.take(token)); + } + + @Test + @DisplayName("take on non-token user returns null") + void take_onNonToken_returnsNull() { + assertNull(AssistantThinkingRelay.take(null)); + assertNull(AssistantThinkingRelay.take("")); + assertNull(AssistantThinkingRelay.take("some-real-user-id")); + } + + @Test + @DisplayName("isToken: prefix-based detection") + void isToken_prefixDetection() { + assertFalse(AssistantThinkingRelay.isToken(null)); + assertFalse(AssistantThinkingRelay.isToken("")); + assertFalse(AssistantThinkingRelay.isToken("regular-user")); + assertTrue(AssistantThinkingRelay.isToken(AssistantThinkingRelay.TOKEN_PREFIX + "anything")); + } + + @Test + @DisplayName("discard after take is a no-op (idempotent)") + void discard_idempotent() { + String token = AssistantThinkingRelay.stash(List.of("x"), "u"); + AssistantThinkingRelay.take(token); + // Should not throw and not affect other entries + AssistantThinkingRelay.discard(token); + assertEquals(0, AssistantThinkingRelay.size()); + } + + @Test + @DisplayName("discard without take removes the entry (producer failure path)") + void discard_withoutTake_removes() { + String token = AssistantThinkingRelay.stash(List.of("x"), "u"); + assertEquals(1, AssistantThinkingRelay.size()); + AssistantThinkingRelay.discard(token); + assertEquals(0, AssistantThinkingRelay.size()); + // Subsequent take still returns null + assertNull(AssistantThinkingRelay.take(token)); + } + + @Test + @DisplayName("concurrent stashes produce distinct tokens") + void stash_distinctTokens() { + String a = AssistantThinkingRelay.stash(List.of("a"), "ua"); + String b = AssistantThinkingRelay.stash(List.of("b"), "ub"); + assertNotEquals(a, b); + + AssistantThinkingRelay.RelayEntry ea = AssistantThinkingRelay.take(a); + AssistantThinkingRelay.RelayEntry eb = AssistantThinkingRelay.take(b); + assertEquals(List.of("a"), ea.thinkings()); + assertEquals("ua", ea.originalUser()); + assertEquals(List.of("b"), eb.thinkings()); + assertEquals("ub", eb.originalUser()); + } + + @Test + @DisplayName("RelayEntry.thinkings is immutable (defensive copy)") + void relayEntry_thinkingsImmutable() { + java.util.ArrayList mutable = new java.util.ArrayList<>(List.of("a", "b")); + String token = AssistantThinkingRelay.stash(mutable, "u"); + mutable.set(0, "mutated"); // should not affect the stashed copy + + AssistantThinkingRelay.RelayEntry entry = AssistantThinkingRelay.take(token); + assertEquals(List.of("a", "b"), entry.thinkings()); + + // thinkings returned is also unmodifiable + assertThrows(UnsupportedOperationException.class, + () -> entry.thinkings().set(0, "x")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentApprovalSanitizationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentApprovalSanitizationTest.java new file mode 100644 index 00000000..37de2e4e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentApprovalSanitizationTest.java @@ -0,0 +1,76 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.model.MessageEntity; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Closure regression test for RFC-067 PR 7. + *

+ * PR 1 §4.1.5 flips {@code MessageEntity.status} from {@code awaiting_approval} + * to {@code completed} (approve) or {@code stopped} (deny) inside + * {@link vip.mate.workspace.conversation.ConversationService#markPendingApprovalsResolved}. + * That status flip travels into history sanitization on subsequent LLM turns; + * if any sanitizer stage accidentally treated the post-flip status as a stub + * marker, the original assistant content would be dropped from history and the + * user-visible conversation would lose context after every approval. + *

+ * These tests pin the boundary: only the explicit {@code [等待审批]} content + * placeholder is dropped by stage 1; a message that carries real streamed text + * + {@code status=awaiting_approval | completed | stopped} is preserved exactly + * regardless of where in the approval lifecycle it sits. + */ +class BaseAgentApprovalSanitizationTest { + + @Test + @DisplayName("Real assistant content with status=awaiting_approval is NOT a Stage 1 placeholder") + void realContentDuringAwaiting() { + // Common shape: streamed partial answer + tool_approval_requested mid-flight, + // doOnComplete persists with status=awaiting_approval (PR 5). + MessageEntity msg = entity("我准备读取你的简历文件。", "awaiting_approval"); + assertFalse(BaseAgent.isApprovalPlaceholder(msg.getContent()), + "real text must not match the placeholder regex — sanitizer would drop it otherwise"); + } + + @Test + @DisplayName("Post-approve message (status=completed, real content) is NOT a placeholder") + void postApproveMessageSurvives() { + // After PR 1 §4.1.5 reconciles approval: status flips awaiting_approval → completed, + // metadata.pendingApproval.status flips pending_approval → approved, content is unchanged. + MessageEntity msg = entity("已读取简历,关键信息: ...", "completed"); + assertFalse(BaseAgent.isApprovalPlaceholder(msg.getContent()), + "approved-and-completed history entry must survive sanitization for the next LLM turn"); + } + + @Test + @DisplayName("Post-deny message (status=stopped, real content) is NOT a placeholder") + void postDenyMessageSurvives() { + // Deny path: status flips awaiting_approval → stopped, content stays as the + // partial assistant text. The LLM should still see this on the next turn so + // it understands "I started reading then was denied" rather than amnesia. + MessageEntity msg = entity("用户拒绝执行工具 write_file", "stopped"); + assertFalse(BaseAgent.isApprovalPlaceholder(msg.getContent()), + "denied turn's assistant text must survive history sanitization"); + } + + @Test + @DisplayName("Pure placeholder content IS dropped (Stage 1's actual job)") + void placeholderStubIsFiltered() { + // The "[等待审批]" stub is the historical placeholder format that Stage 1 catches — + // those rows have no streamed content and add no value to the LLM context. + MessageEntity msg = entity("[等待审批]", "awaiting_approval"); + assertTrue(BaseAgent.isApprovalPlaceholder(msg.getContent()), + "stub-only placeholder content must still match so Stage 1 keeps filtering it"); + } + + private static MessageEntity entity(String content, String status) { + MessageEntity m = new MessageEntity(); + m.setRole("assistant"); + m.setContent(content); + m.setStatus(status); + return m; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentDirectToolHistoryScrubTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentDirectToolHistoryScrubTest.java new file mode 100644 index 00000000..2dd03993 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentDirectToolHistoryScrubTest.java @@ -0,0 +1,150 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-052 multi-turn leakage fix: verify that {@link BaseAgent#isDirectToolMessage} + * correctly identifies persisted assistant messages produced by a returnDirect + * tool path, so {@code toSpringMessage} replaces their content with a placeholder + * before the next turn's prompt is built. + * + *

The DB row stays unchanged; only the in-memory {@code AssistantMessage} + * handed to the model is scrubbed. + */ +class BaseAgentDirectToolHistoryScrubTest { + + @Test + @DisplayName("metadata.directToolNames non-empty list => identified as direct-tool message") + void directToolNamesNonEmpty_recognized() { + MessageEntity msg = new MessageEntity(); + msg.setRole("assistant"); + msg.setContent("EMPLOYEE-SECRET-DATA"); + msg.setMetadata("{\"segments\":[],\"directToolNames\":[\"query_employee_salary\"]}"); + + assertTrue(BaseAgent.isDirectToolMessage(msg), + "Assistant message with directToolNames must be flagged for scrubbing"); + } + + @Test + @DisplayName("metadata.directToolNames empty list => NOT treated as direct-tool") + void directToolNamesEmpty_notRecognized() { + MessageEntity msg = new MessageEntity(); + msg.setRole("assistant"); + msg.setContent("normal answer"); + msg.setMetadata("{\"directToolNames\":[]}"); + + assertFalse(BaseAgent.isDirectToolMessage(msg), + "Empty directToolNames means no direct tool fired — don't scrub"); + } + + @Test + @DisplayName("metadata without directToolNames => not direct-tool") + void noDirectToolNamesField_notRecognized() { + MessageEntity msg = new MessageEntity(); + msg.setRole("assistant"); + msg.setContent("regular tool-call answer"); + msg.setMetadata("{\"toolCalls\":[{\"name\":\"get_weather\"}]}"); + + assertFalse(BaseAgent.isDirectToolMessage(msg)); + } + + @Test + @DisplayName("null/empty metadata => not direct-tool") + void nullOrEmptyMetadata_notRecognized() { + MessageEntity msg = new MessageEntity(); + msg.setRole("assistant"); + msg.setContent("hi"); + + assertFalse(BaseAgent.isDirectToolMessage(msg), "null metadata"); + + msg.setMetadata(""); + assertFalse(BaseAgent.isDirectToolMessage(msg), "empty metadata string"); + + msg.setMetadata("{}"); + assertFalse(BaseAgent.isDirectToolMessage(msg), "empty JSON object"); + } + + @Test + @DisplayName("null entity => safely returns false") + void nullEntity_safe() { + assertFalse(BaseAgent.isDirectToolMessage(null)); + } + + // ========== OpenClaw-inspired optimization: tool-name-aware placeholder ========== + + @Test + @DisplayName("directToolNamesIn extracts the array contents") + void extractToolNames_singleAndMultiple() { + MessageEntity single = new MessageEntity(); + single.setRole("assistant"); + single.setMetadata("{\"directToolNames\":[\"query_employee_salary\"]}"); + assertEquals(List.of("query_employee_salary"), + BaseAgent.directToolNamesIn(single)); + + MessageEntity multi = new MessageEntity(); + multi.setRole("assistant"); + multi.setMetadata("{\"directToolNames\":[\"tool_a\",\"tool_b\",\"tool_c\"]}"); + assertEquals(List.of("tool_a", "tool_b", "tool_c"), + BaseAgent.directToolNamesIn(multi)); + } + + @Test + @DisplayName("directToolNamesIn returns empty list for non-direct messages") + void extractToolNames_emptyForNonDirect() { + MessageEntity msg = new MessageEntity(); + msg.setRole("assistant"); + msg.setMetadata("{\"toolCalls\":[{\"name\":\"get_weather\"}]}"); + assertTrue(BaseAgent.directToolNamesIn(msg).isEmpty()); + } + + @Test + @DisplayName("History placeholder names the tool so the model retains conversational structure") + void placeholder_singleTool_namesIt() { + String placeholder = BaseAgent.directToolHistoryPlaceholder( + List.of("query_employee_salary")); + assertTrue(placeholder.contains("query_employee_salary"), + "Single-tool placeholder must name the tool"); + assertTrue(placeholder.contains("withheld"), + "Placeholder must signal the data is withheld"); + assertTrue(placeholder.contains("call the tool again"), + "Placeholder must hint at the recovery path"); + } + + @Test + @DisplayName("Multi-tool placeholder lists every tool") + void placeholder_multipleTools_listAll() { + String placeholder = BaseAgent.directToolHistoryPlaceholder( + List.of("query_employee_salary", "read_medical_record")); + assertTrue(placeholder.contains("query_employee_salary")); + assertTrue(placeholder.contains("read_medical_record")); + } + + @Test + @DisplayName("Empty/null tool name list falls back to a generic placeholder") + void placeholder_emptyList_genericFallback() { + String empty = BaseAgent.directToolHistoryPlaceholder(List.of()); + String nullList = BaseAgent.directToolHistoryPlaceholder(null); + assertEquals(empty, nullList, + "Both null and empty must produce identical generic placeholders"); + assertTrue(empty.contains("withheld")); + } + + @Test + @DisplayName("Placeholder MUST NOT echo the original sensitive content") + void placeholder_neverContainsTheSensitivePayload() { + // Sanity: even if the metadata-extracted tool name happens to be + // sensitive-sounding, the placeholder is bounded — it doesn't re-emit + // the message content itself. + String placeholder = BaseAgent.directToolHistoryPlaceholder( + List.of("query_employee_salary")); + assertFalse(placeholder.contains("12345")); + assertFalse(placeholder.contains("SSN")); + assertFalse(placeholder.contains("PWD")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentHeadOrphanRepairTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentHeadOrphanRepairTest.java new file mode 100644 index 00000000..3fad6503 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentHeadOrphanRepairTest.java @@ -0,0 +1,224 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Head-side pair repair on the recent-message pagination cut. + * + *

{@code listRecentMessages(conversationId, windowSize)} returns the last N + * rows verbatim. The first row of that page can be a {@link ToolResponseMessage} + * whose owning {@link AssistantMessage} (carrying the matching tool_call_id) + * sat one row earlier — i.e. outside the page. Sending such a sequence to any + * OpenAI-compatible provider returns 400 because every tool response must be + * preceded by an assistant message issuing that tool_call_id. + * + *

{@link BaseAgent#stripHeadOrphanToolResponses} drops leading + * {@code ToolResponseMessage}s whose response ids are unmatched by every + * AssistantMessage still in scope. {@link SystemMessage}s (boundary rows, + * system prompts) at the head are skipped over, not removed. + */ +class BaseAgentHeadOrphanRepairTest { + + @Test + void orphanToolResponseAtHeadIsDropped() { + // Window starts with a TOOL response (orphan: no AssistantMessage in this list issued call-X). + List messages = new ArrayList<>(List.of( + toolResponse("call-X"), + new UserMessage("next user turn"), + assistantWithToolCalls("call-Y"), + toolResponse("call-Y") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(1, dropped, "leading orphan should be dropped"); + assertInstanceOf(UserMessage.class, messages.getFirst(), + "head is now the user turn, not the orphan tool response"); + } + + @Test + void multipleConsecutiveOrphansAtHeadAllDropped() { + // A single AssistantMessage outside the window may have produced + // several tool calls whose responses landed in two separate + // ToolResponseMessages. Both should be removed. + List messages = new ArrayList<>(List.of( + toolResponse("call-A"), + toolResponse("call-B"), + new UserMessage("here we go"), + assistantWithToolCalls("call-C"), + toolResponse("call-C") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(2, dropped); + assertInstanceOf(UserMessage.class, messages.getFirst()); + } + + @Test + void systemBoundaryAtHeadIsSkippedAndOrphanBehindItIsDropped() { + // After findLatestCompressionBoundary prepends a SystemMessage, the + // orphan tool response now sits at index 1. The repair must skip the + // system row and still drop the orphan. + SystemMessage boundary = new SystemMessage("[compression boundary placeholder]"); + List messages = new ArrayList<>(List.of( + boundary, + toolResponse("call-X"), + new UserMessage("after orphan"), + assistantWithToolCalls("call-Y"), + toolResponse("call-Y") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(1, dropped); + assertSame(boundary, messages.getFirst(), + "the system boundary stays in place"); + assertInstanceOf(UserMessage.class, messages.get(1), + "the orphan that sat behind the boundary is gone"); + } + + @Test + void matchedHeadToolResponseIsKept() { + // The window happens to start with both the AssistantMessage and its + // tool response — perfectly aligned, nothing to drop. + List messages = new ArrayList<>(List.of( + assistantWithToolCalls("call-A"), + toolResponse("call-A"), + new UserMessage("next") + )); + List snapshot = new ArrayList<>(messages); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(0, dropped); + assertEquals(snapshot, messages, "no drops, list unchanged"); + } + + @Test + void laterAssistantWithSameIdDoesNotRedeemHeadOrphan() { + // The classic order-sensitivity trap: a ToolResponseMessage sits at + // the head, and a LATER AssistantMessage happens to carry the same + // tool_call_id. The provider's contract is "tool_call must precede + // tool_response", not "tool_call exists somewhere in the prompt". + // The leading response is therefore still orphan and must be dropped. + List messages = new ArrayList<>(List.of( + new SystemMessage("[boundary]"), + toolResponse("call-X"), + new UserMessage("hi"), + assistantWithToolCalls("call-X"), // same id, but AFTER the response + toolResponse("call-X") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(1, dropped, + "the leading response is orphan regardless of whether a later assistant " + + "happens to carry the same id — provider validity is order-sensitive"); + assertInstanceOf(SystemMessage.class, messages.get(0)); + assertInstanceOf(UserMessage.class, messages.get(1), + "the orphan that sat between the boundary and the user turn is gone"); + } + + @Test + void partialOrphanInLeadingResponseIsDropped() { + // A ToolResponseMessage with two responses — one whose id has no + // preceding assistant, one whose id has none either (since we + // haven't walked any assistants yet). Provider order-validity + // doesn't allow partial pairs; dropping wholesale is the safer + // call. We lose matched-response content but never emit a request + // the provider would 400. + ToolResponseMessage mixed = ToolResponseMessage.builder().responses(List.of( + new ToolResponseMessage.ToolResponse("call-orphan", "tool_x", "x"), + new ToolResponseMessage.ToolResponse("call-known", "tool_y", "y") + )).build(); + List messages = new ArrayList<>(List.of( + mixed, + assistantWithToolCalls("call-known"), + toolResponse("call-known") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(1, dropped, + "no preceding assistant has been walked yet, so even a partially-matched " + + "leading response is dropped wholesale"); + assertInstanceOf(AssistantMessage.class, messages.getFirst(), + "the mixed head is gone; the assistant that would have owned call-known is now first"); + } + + @Test + void emptyListIsNoOp() { + List messages = new ArrayList<>(); + assertEquals(0, BaseAgent.stripHeadOrphanToolResponses(messages, "test")); + assertTrue(messages.isEmpty()); + } + + @Test + void purelyUserAssistantHistoryUntouched() { + // No tool responses at all — repair is a no-op. + List messages = new ArrayList<>(List.of( + new UserMessage("hi"), + new AssistantMessage("hello"), + new UserMessage("how are you?"), + new AssistantMessage("good") + )); + List snapshot = new ArrayList<>(messages); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(0, dropped); + assertEquals(snapshot, messages); + } + + @Test + void stopsAtFirstNonOrphanNonSystem() { + // Once we hit a non-system, non-orphan message, repair stops — we do + // NOT keep walking and look for orphans deeper in the history. + // Deeper orphans imply an upstream bug; this guard is only here to + // protect the pagination cut. + List messages = new ArrayList<>(List.of( + toolResponse("call-A"), // orphan at head — will be dropped + new UserMessage("user"), // stops the scan + toolResponse("call-B"), // orphan but we do NOT touch it + new AssistantMessage("late") + )); + + int dropped = BaseAgent.stripHeadOrphanToolResponses(messages, "test"); + + assertEquals(1, dropped); + assertInstanceOf(UserMessage.class, messages.getFirst()); + assertFalse(messages.stream().noneMatch(m -> m instanceof ToolResponseMessage), + "the deeper orphan stays in place — it surfaces as an upstream bug elsewhere"); + } + + // ------------------------------------------------------------------ helpers + + private static AssistantMessage assistantWithToolCalls(String... callIds) { + List calls = new ArrayList<>(); + for (String id : callIds) { + calls.add(new AssistantMessage.ToolCall(id, "function", "tool_" + id, "{}")); + } + return AssistantMessage.builder().content("").toolCalls(calls).build(); + } + + private static ToolResponseMessage toolResponse(String callId) { + return ToolResponseMessage.builder().responses(List.of( + new ToolResponseMessage.ToolResponse(callId, "tool_" + callId, "ok") + )).build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java new file mode 100644 index 00000000..e5a94d8b --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/BaseAgentMultimodalSkipNoticeTest.java @@ -0,0 +1,192 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.workspace.conversation.ConversationService; +import vip.mate.workspace.conversation.model.MessageContentPart; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.EnumSet; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Issue #44 regression: when a video attachment cannot be passed to the model + * (because the agent's resolved {@link ModelCapabilityService.Modality#VIDEO} + * capability is absent), the user message must include a system notice listing + * the skipped attachment and instructing the agent NOT to invent a tool call to + * read it. + * + *

The original bug: silent skip ({@code log.debug} only) → agent saw + * {@code [附件] xxx.mp4} placeholder text in history but no actual media → it + * picked {@code BrowserUseTool} or similar to "open" the file, which never + * produced useful results. + * + *

These tests pin the contract that the skip path mutates the prompt text, + * not just a log line. + * + *

Issue #87 update: the previous "禁止调用任何工具" sentence is no longer + * emitted unconditionally — when the agent has any media-capable tool bound, + * the LLM is allowed to delegate to it. With no tools (this test scaffold's + * default), the notice falls back to a "switch models" suggestion only. + */ +class BaseAgentMultimodalSkipNoticeTest { + + @Test + @DisplayName("Video attachment + model lacks VIDEO capability → skipped, system notice in prompt text") + void videoSkipped_emitsSystemNotice() { + TestAgent agent = newAgentWithCaps(EnumSet.noneOf(ModelCapabilityService.Modality.class)); + MessageEntity msg = userMessage("看看这段视频"); + when(agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(MessageContentPart.video("media-1", "demo.mp4"))); + + UserMessage result = agent.callBuildUserMessage(msg, "看看这段视频"); + + assertNotNull(result); + String text = result.getText(); + assertTrue(text.contains("[系统提示]"), + "skipped video must surface a system notice in the prompt text — issue #44"); + assertTrue(text.contains("demo.mp4"), + "notice must name the skipped file so the agent can tell the user"); + assertTrue(text.contains("不支持视频输入"), + "reason string must name the modality the model cannot consume"); + assertTrue(text.contains("建议切换"), + "notice must tell the agent to recommend switching models when no media tool is bound"); + assertFalse(text.contains("不要调用任何工具"), + "issue #87: the hard tool ban must be gone — bound media tools should still be usable"); + assertTrue(result.getMedia() == null || result.getMedia().isEmpty(), + "video must NOT be injected as Media when capability is absent"); + } + + @Test + @DisplayName("Image attachment + model lacks VISION capability → skipped, system notice") + void imageSkipped_emitsSystemNotice() { + // Regression for the "GLM-5-Turbo + image upload" failure: when a user + // uploads an image to a text-only model, we used to pass the image through + // anyway and let the API 400. Now we skip + notify, same as the video gate. + TestAgent agent = newAgentWithCaps(EnumSet.noneOf(ModelCapabilityService.Modality.class)); + MessageEntity msg = userMessage("看看这张图"); + when(agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(MessageContentPart.file("media-1", "poster.png", "image/png"))); + + UserMessage result = agent.callBuildUserMessage(msg, "看看这张图"); + + String text = result.getText(); + assertTrue(text.contains("poster.png")); + assertTrue(text.contains("不支持图片输入"), + "vision-skip notice must use 不支持图片输入 wording"); + assertTrue(result.getMedia() == null || result.getMedia().isEmpty(), + "image must NOT be injected when model has no VISION capability"); + } + + @Test + @DisplayName("No attachments → no system notice, prompt text unchanged") + void noAttachments_noNoticeAdded() { + TestAgent agent = newAgentWithCaps(EnumSet.noneOf(ModelCapabilityService.Modality.class)); + MessageEntity msg = userMessage("hello"); + when(agent.conversationService.parseMessageParts(msg)).thenReturn(List.of()); + + UserMessage result = agent.callBuildUserMessage(msg, "hello"); + + assertFalse(result.getText().contains("[系统提示]"), + "no skipped attachments → no notice; clean prompt for normal text-only turns"); + } + + @Test + @DisplayName("Capable model + video → no system notice (notice only fires on actual skip)") + void videoCapable_noNotice_attemptInjection() { + // VIDEO capability present → no skip-on-capability-grounds. The injection itself + // may still fail downstream (file path doesn't exist in this test) — when that + // happens, the file-not-found / load-failure branch surfaces its OWN notice with + // a different reason string. We assert the capability-skip reason is absent here. + TestAgent agent = newAgentWithCaps( + EnumSet.of(ModelCapabilityService.Modality.VIDEO, ModelCapabilityService.Modality.TEXT)); + MessageEntity msg = userMessage("分析视频"); + when(agent.conversationService.parseMessageParts(msg)) + .thenReturn(List.of(MessageContentPart.video("media-1", "ok.mp4"))); + + UserMessage result = agent.callBuildUserMessage(msg, "分析视频"); + + assertFalse(result.getText().contains("不支持视频输入"), + "capable model must not be flagged as missing video capability"); + } + + @Test + @DisplayName("History replay (injectMedia=false) returns text-only — no Media accumulation") + void historyReplay_dropsMedia() { + // Regression for Zhipu GLM-5V "code:1210 input videos exceeds limit": each + // historical user message previously re-injected its video Media on every + // turn, so a 2-turn conversation hit the per-request 1-video cap. The + // history path must drop Media even when the model supports video. + TestAgent agent = newAgentWithCaps( + EnumSet.of(ModelCapabilityService.Modality.VIDEO, ModelCapabilityService.Modality.TEXT)); + MessageEntity msg = userMessage("上一轮的视频"); + // parseMessageParts is irrelevant when injectMedia=false; verify by NOT stubbing it. + + UserMessage result = agent.callBuildUserMessage(msg, "上一轮的视频", false); + + assertTrue(result.getMedia() == null || result.getMedia().isEmpty(), + "history replay must NOT carry Media — even capable models cap video count per request"); + assertFalse(result.getText().contains("[系统提示]"), + "history replay must NOT add the skip notice — the skip notice is a current-turn concern"); + } + + // ---------- Test scaffold ---------- + + private static MessageEntity userMessage(String content) { + MessageEntity m = new MessageEntity(); + m.setRole("user"); + m.setContent(content); + return m; + } + + private static TestAgent newAgentWithCaps(EnumSet caps) { + ConversationService conv = mock(ConversationService.class); + TestAgent agent = new TestAgent(conv); + agent.modelCapabilities = caps; + agent.modelName = "test-model"; + agent.agentName = "test-agent"; + return agent; + } + + /** + * Minimal concrete BaseAgent for testing buildUserMessage. The abstract + * chat / chatStream / execute methods are stubbed because buildUserMessage + * does not depend on them. + */ + static class TestAgent extends BaseAgent { + TestAgent(ConversationService conv) { + super(null, conv); + } + + UserMessage callBuildUserMessage(MessageEntity msg, String renderedContent) { + return buildUserMessage(msg, renderedContent); + } + + UserMessage callBuildUserMessage(MessageEntity msg, String renderedContent, boolean injectMedia) { + return buildUserMessage(msg, renderedContent, injectMedia); + } + + @Override + public String chat(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public reactor.core.publisher.Flux chatStream(String userMessage, String conversationId) { + throw new UnsupportedOperationException(); + } + + @Override + public String execute(String goal, String conversationId) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/GraphEventPublisherIterationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/GraphEventPublisherIterationTest.java new file mode 100644 index 00000000..7b96b8b5 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/GraphEventPublisherIterationTest.java @@ -0,0 +1,78 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Field-shape tests for {@link GraphEventPublisher#iterationStart} and + * {@link GraphEventPublisher#iterationEnd}. Verifies the payload contract + * that downstream SSE consumers depend on (index / scope default / optional + * subagentId / char counters). + */ +class GraphEventPublisherIterationTest { + + @Test + @DisplayName("iterationStart carries index, reason, scope, timestamp") + void iterationStartShape() { + GraphEventPublisher.GraphEvent event = GraphEventPublisher.iterationStart( + 3, "react_step", "parent", null); + assertEquals(GraphEventPublisher.EVENT_ITERATION_START, event.type()); + Map data = event.data(); + assertEquals(3, data.get("index")); + assertEquals("react_step", data.get("reason")); + assertEquals("parent", data.get("scope")); + assertFalse(data.containsKey("subagentId"), + "subagentId must be absent when null/empty"); + assertTrue(data.containsKey("timestamp")); + } + + @Test + @DisplayName("iterationStart defaults missing scope to 'parent'") + void iterationStartDefaultsScope() { + GraphEventPublisher.GraphEvent event = GraphEventPublisher.iterationStart( + 0, null, null, null); + Map data = event.data(); + assertEquals("parent", data.get("scope")); + assertEquals("", data.get("reason"), + "Missing reason should serialize as empty string, not null"); + } + + @Test + @DisplayName("iterationStart includes subagentId when provided") + void iterationStartIncludesSubagentId() { + GraphEventPublisher.GraphEvent event = GraphEventPublisher.iterationStart( + 7, "plan_step", "subagent", "sa-42"); + Map data = event.data(); + assertEquals("subagent", data.get("scope")); + assertEquals("sa-42", data.get("subagentId")); + } + + @Test + @DisplayName("iterationEnd carries char counters and scope") + void iterationEndShape() { + GraphEventPublisher.GraphEvent event = GraphEventPublisher.iterationEnd( + 5, "parent", null, 1234, 56); + assertEquals(GraphEventPublisher.EVENT_ITERATION_END, event.type()); + Map data = event.data(); + assertEquals(5, data.get("index")); + assertEquals("parent", data.get("scope")); + assertEquals(1234, data.get("contentChars")); + assertEquals(56, data.get("thinkingChars")); + assertFalse(data.containsKey("subagentId")); + } + + @Test + @DisplayName("iterationEnd defaults scope to 'parent' when null") + void iterationEndDefaultsScope() { + GraphEventPublisher.GraphEvent event = GraphEventPublisher.iterationEnd( + 0, null, "", 0, 0); + Map data = event.data(); + assertEquals("parent", data.get("scope")); + assertFalse(data.containsKey("subagentId"), + "Empty subagentId must be omitted"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/PatchReasoningContentTest.java b/mateclaw-server/src/test/java/vip/mate/agent/PatchReasoningContentTest.java new file mode 100644 index 00000000..b28bc1b7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/PatchReasoningContentTest.java @@ -0,0 +1,430 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.openai.api.OpenAiApi; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.Role; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionMessage.ToolCall; +import org.springframework.ai.openai.api.OpenAiApi.ChatCompletionRequest; +import vip.mate.llm.model.ModelProviderEntity; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; + +/** + * RFC-049 PR-2 consumer-side tests for + * {@link AgentGraphBuilder#patchReasoningContent(ChatCompletionRequest, ModelProviderEntity)}. + * + *

Covers four orthogonal dimensions: + *

    + *
  • FallbackPolicy — DEEPSEEK (null + warn + patchNonToolCall) vs KIMI / OPENAI / + * DEFAULT (" " + no-warn + tool-call-only)
  • + *
  • {@code lastUserIdx} scope — assistants at {@code i <= lastUserIdx} never patched; + * iterator still advances for alignment
  • + *
  • sanitizedUser — restored from {@code RelayEntry.originalUser}; relay token + * never egresses
  • + *
  • relay presence — iterator consumed in order; missing relay triggers policy + * fallback only for in-turn messages
  • + *
+ */ +class PatchReasoningContentTest { + + // ---------- Fixtures ---------- + + private static ModelProviderEntity provider(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + return p; + } + + private static ChatCompletionMessage user(String text) { + return new ChatCompletionMessage(text, Role.USER); + } + + private static ChatCompletionMessage system(String text) { + return new ChatCompletionMessage(text, Role.SYSTEM); + } + + /** Plain assistant message — no tool calls, no reasoning_content. */ + private static ChatCompletionMessage assistantPlain(String text) { + return new ChatCompletionMessage(text, Role.ASSISTANT, null, null, null, null, null, null, null); + } + + /** Assistant tool_call message with optional pre-existing reasoning_content. */ + private static ChatCompletionMessage assistantToolCall(String text, String reasoningContent) { + ToolCall tc = new ToolCall("call_1", "function", null); + return new ChatCompletionMessage(text, Role.ASSISTANT, null, null, List.of(tc), null, null, null, reasoningContent); + } + + /** Build a ChatCompletionRequest with the given messages + user field; all other fields null. */ + private static ChatCompletionRequest request(List messages, String user) { + return new ChatCompletionRequest( + messages, // messages + "test-model", // model + null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, + null, null, // toolChoice, parallelToolCalls + user, // user + null, // reasoningEffort + null, null, null, null, null + ); + } + + @BeforeEach + void clearRelay() { + AssistantThinkingRelay.clearAll(); + } + + @AfterEach + void clearRelayAfter() { + AssistantThinkingRelay.clearAll(); + } + + // ---------- No-relay, no-thinking-mode path ---------- + + @Test + @DisplayName("No relay token + no thinking signals → request passes through unchanged") + void noop_whenNoRelayAndNoThinkingMode() { + ChatCompletionRequest req = request(List.of( + system("sys"), + user("q1"), + assistantToolCall("", null) // no reasoning_content anywhere → not thinking mode + ), "caller-user-1"); + + // model is "test-model" which maps to STANDARD family → requiresReasoningContentPatch returns false + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + assertSame(req, out, "no thinking signal → no rebuild"); + assertEquals("caller-user-1", out.user(), "user field untouched"); + } + + @Test + @DisplayName("Leaked relay token (no entry in map) + no thinking signals → strips token, rebuilds user") + void stripsLeakedToken_whenNoEntryNoThinking() { + // Prefix-shaped but not actually stashed — simulates consumer running after + // producer's finally already discarded. take() returns null; isToken() still true. + String fakeToken = AssistantThinkingRelay.TOKEN_PREFIX + "orphan-uuid"; + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantPlain("hi") + ), fakeToken); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("openai")); + assertNotSame(req, out, "rebuild expected to strip leaked token"); + assertNull(out.user(), "leaked token must be sanitized to null"); + } + + // ---------- sanitizedUser restoration from RelayEntry ---------- + + @Test + @DisplayName("sanitizedUser is restored from RelayEntry.originalUser") + void sanitizedUser_restoredFromRelayEntry() { + List thinkings = List.of("", "in-turn-think"); + String token = AssistantThinkingRelay.stash(thinkings, "original-caller-42"); + + ChatCompletionRequest req = request(List.of( + assistantPlain("prior-assistant"), // i=0, position 0 in thinkings → "" + user("q1"), + assistantToolCall("a1", null) // i=2, position 1 in thinkings → "in-turn-think" + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals("original-caller-42", out.user(), "sanitizedUser must equal entry.originalUser()"); + assertEquals("in-turn-think", out.messages().get(2).reasoningContent(), + "in-turn assistant (i=2 > lastUserIdx=1) should receive the real thinking"); + } + + // ---------- lastUserIdx scope ---------- + + @Test + @DisplayName("DEEPSEEK patchCrossTurn=true: prior-turn assistants get ' ' fallback so multi-turn doesn't 400") + void crossTurnAssistants_patchedWithSpace_deepseek() { + // [sys, U1, A1(tool_call, no-rc), U2, A2(tool_call, no-rc)] + // lastUserIdx = 3 (U2) + // Relay thinkings: [null for A1, "real-a2" for A2] + // + // DeepSeek (since 2026-04) requires reasoning_content on EVERY assistant + // in the request — prior-turn included. Without patchCrossTurn, A1 stays + // null and DeepSeek 400s on every multi-turn conversation. With it, A1 + // gets the same " " fallback in-turn assistants get. + List thinkings = Arrays.asList("", "real-a2"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + system("sys"), + user("q1"), + assistantToolCall("a1", null), // i=2, cross-turn (2 <= 3) + user("q2"), + assistantToolCall("a2", null) // i=4, in-turn (4 > 3) + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals(" ", out.messages().get(2).reasoningContent(), + "cross-turn A1 gets ' ' fallback so DeepSeek thinking-mode validation passes"); + assertEquals("real-a2", out.messages().get(4).reasoningContent(), + "in-turn A2 (i=4 > lastUserIdx=3) receives the real relay value"); + } + + @Test + @DisplayName("Iterator stays aligned: cross-turn consumes '' positions so in-turn gets correct thinking") + void iteratorAlignment_acrossCrossTurnAndInTurn() { + // [U1, A1(no-rc), A2(no-rc), U2, A3(no-rc), A4(no-rc)] + // lastUserIdx = 3 (U2). Producer extraction order = A1,A2,A3,A4. + // Relay: ["","" (cross-turn, stripped already), "real-a3", "real-a4"] + List thinkings = List.of("", "", "real-a3", "real-a4"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantToolCall("a1", null), // i=1 cross-turn + assistantToolCall("a2", null), // i=2 cross-turn + user("q2"), + assistantToolCall("a3", null), // i=4 in-turn + assistantToolCall("a4", null) // i=5 in-turn + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + // DEEPSEEK patchCrossTurn=true: cross-turn now also gets ' ' fallback. + // Iterator alignment is preserved: A1/A2 consume the empty entries '', + // A3/A4 consume their real values in correct positions. + assertEquals(" ", out.messages().get(1).reasoningContent(), "A1 cross-turn ' ' fallback"); + assertEquals(" ", out.messages().get(2).reasoningContent(), "A2 cross-turn ' ' fallback"); + assertEquals("real-a3", out.messages().get(4).reasoningContent(), "A3 in-turn gets real-a3 (not real-a4)"); + assertEquals("real-a4", out.messages().get(5).reasoningContent(), "A4 in-turn gets real-a4"); + } + + // ---------- FallbackPolicy × emptyFallback ---------- + + @Test + @DisplayName("DEEPSEEK policy: relay empty for in-turn tool_call → reasoning_content gets ' ' fallback") + void deepseek_relayEmpty_fallsBackToSpace() { + // 72bd33dc switched DEEPSEEK from emptyFallback=null (force explicit 400) + // to " " — null kept self-replicating 400s every multi-tool turn that + // crossed a summarizing boundary. Aligning with KIMI/OPENAI tolerance. + List thinkings = List.of(""); // one assistant, no real thinking + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantToolCall("a1", null) // in-turn + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals(" ", out.messages().get(1).reasoningContent(), + "DeepSeek: ' ' fallback restores forward progress when relay has no real value"); + } + + @Test + @DisplayName("KIMI policy: relay empty for in-turn tool_call → ' ' injected (legacy tolerance)") + void kimi_relayEmpty_injectsSpace() { + // Kimi path is triggered by the model-family check; use model name that maps to KIMI_THINKING. + List thinkings = List.of(""); + String token = AssistantThinkingRelay.stash(thinkings, null); + + List msgs = List.of( + user("q1"), + assistantToolCall("a1", null) + ); + ChatCompletionRequest req = new ChatCompletionRequest( + msgs, "kimi-k2.5", null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, token, + null, null, null, null, null, null + ); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("kimi-cn")); + + assertEquals(" ", out.messages().get(1).reasoningContent(), + "Kimi tolerates ' ' — preserve legacy behavior"); + } + + @Test + @DisplayName("Unknown provider uses DEFAULT policy: ' ' injected (legacy tolerance, not noop)") + void defaultPolicy_unknownProvider_injectsSpace() { + List thinkings = List.of(""); + String token = AssistantThinkingRelay.stash(thinkings, null); + + // Use a model that triggers requiresReasoningContentPatch so thinking mode is active + List msgs = List.of( + user("q1"), + assistantToolCall("a1", null) + ); + ChatCompletionRequest req = new ChatCompletionRequest( + msgs, "deepseek-reasoner", null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, token, + null, null, null, null, null, null + ); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("custom-gateway")); + + assertEquals(" ", out.messages().get(1).reasoningContent(), + "DEFAULT keeps legacy ' ' for unrecognized providers — avoid regressing self-hosted backends"); + } + + // ---------- FallbackPolicy × patchNonToolCall ---------- + + @Test + @DisplayName("DEEPSEEK policy patches non-tool_call in-turn assistants too (patchNonToolCall=true)") + void deepseek_patchesNonToolCallAssistant() { + List thinkings = List.of("thinking-for-plain"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantPlain("plain answer") // no tool_calls + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals("thinking-for-plain", out.messages().get(1).reasoningContent(), + "DeepSeek contract requires reasoning_content even on non-tool_call assistants when in thinking mode"); + } + + @Test + @DisplayName("KIMI policy leaves non-tool_call assistants alone (patchNonToolCall=false)") + void kimi_skipsNonToolCallAssistant() { + List thinkings = List.of("would-not-be-used"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + List msgs = List.of( + user("q1"), + assistantPlain("plain answer") + ); + ChatCompletionRequest req = new ChatCompletionRequest( + msgs, "kimi-k2.5", null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, token, + null, null, null, null, null, null + ); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("kimi-cn")); + + assertNull(out.messages().get(1).reasoningContent(), + "Kimi only patches tool_call assistants; plain assistants are untouched"); + } + + // ---------- Preserve pre-existing real values ---------- + + @Test + @DisplayName("Assistant that already has real reasoning_content is left alone") + void existingRealValue_preserved() { + List thinkings = List.of("would-overwrite"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantToolCall("a1", "pre-existing-real-thinking") // already has a value + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals("pre-existing-real-thinking", out.messages().get(1).reasoningContent(), + "non-blank existing reasoning_content must not be overwritten by relay"); + } + + // ---------- Edge: empty messages ---------- + + @Test + @DisplayName("Empty messages list: no-op, returns same instance") + void emptyMessages_noop() { + ChatCompletionRequest req = request(List.of(), null); + assertSame(req, AgentGraphBuilder.patchReasoningContent(req, provider("deepseek"))); + } + + @Test + @DisplayName("Null messages: no-op, returns same instance") + void nullMessages_noop() { + ChatCompletionRequest req = new ChatCompletionRequest( + null, "m", null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null, null, null, null, null, null, null, null, + null, null, null, null, null, null + ); + assertSame(req, AgentGraphBuilder.patchReasoningContent(req, provider("deepseek"))); + } + + // ---------- Fewer relay entries than assistants: defensive policy fallback ---------- + + @Test + @DisplayName("Relay shorter than assistant count: extra in-turn assistants fall back to policy") + void relayShorterThanAssistants_fallsBack() { + // Producer extracted 1 entry but there are 2 in-turn tool_call assistants + // (e.g. one was added after relay stash — shouldn't happen but be defensive). + List thinkings = List.of("real-1"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(new ArrayList<>(List.of( + user("q1"), + assistantToolCall("a1", null), + assistantToolCall("a2", null) + )), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals("real-1", out.messages().get(1).reasoningContent()); + assertEquals(" ", out.messages().get(2).reasoningContent(), + "DEEPSEEK with emptyFallback=' ' (post-72bd33dc): missing real values get the same tolerant fallback"); + } + + // ---------- patchCrossTurn policy (2026-04-29) ---------- + + @Test + @DisplayName("KIMI / OPENAI / DEFAULT do NOT patch cross-turn — only DEEPSEEK does") + void crossTurnPatching_isDeepseekOnly() { + // Same shape as crossTurnAssistants_patchedWithSpace_deepseek but with + // KIMI provider — KIMI's contract resets thinking across user turns, + // so prior-turn assistants must remain null. Pinning this here protects + // against accidentally flipping patchCrossTurn=true for all providers. + List thinkings = Arrays.asList("", "real-a2"); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + assistantToolCall("a1", null), // i=1, cross-turn (1 <= 2) + user("q2"), + assistantToolCall("a2", null) // i=3, in-turn (3 > 2) + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("kimi-cn")); + + assertNull(out.messages().get(1).reasoningContent(), + "KIMI does not patch cross-turn — thinking resets across user turns"); + assertEquals("real-a2", out.messages().get(3).reasoningContent(), + "KIMI in-turn assistants still receive their relay value"); + } + + @Test + @DisplayName("DEEPSEEK cross-turn assistant without tool_calls also patched (patchNonToolCall=true)") + void crossTurnPlainAssistant_patchedForDeepseek() { + // Plain prior-turn text assistant (no tool_calls): without the + // patchNonToolCall guard, this would still be skipped. DEEPSEEK has + // both patchNonToolCall=true AND patchCrossTurn=true, so it should + // get the ' ' fallback. This is the most common production case + // since plain assistants dominate IM channel history. + List thinkings = Arrays.asList("", ""); + String token = AssistantThinkingRelay.stash(thinkings, null); + + ChatCompletionRequest req = request(List.of( + user("q1"), + new ChatCompletionMessage("plain a1", Role.ASSISTANT), // no tool_calls + user("q2"), + new ChatCompletionMessage("plain a2", Role.ASSISTANT) + ), token); + + ChatCompletionRequest out = AgentGraphBuilder.patchReasoningContent(req, provider("deepseek")); + + assertEquals(" ", out.messages().get(1).reasoningContent(), + "DEEPSEEK plain prior-turn assistant gets ' ' so request validates"); + assertEquals(" ", out.messages().get(3).reasoningContent(), + "DEEPSEEK plain in-turn assistant gets ' ' as before"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/ReasoningEffortSanitizerTest.java b/mateclaw-server/src/test/java/vip/mate/agent/ReasoningEffortSanitizerTest.java new file mode 100644 index 00000000..762b15e2 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/ReasoningEffortSanitizerTest.java @@ -0,0 +1,212 @@ +package vip.mate.agent; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.openai.api.OpenAiApi; +import vip.mate.llm.model.ModelProviderEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-049 PR-1.3 verification — covers §5.2 Case E3.1 / E3.2 / E3.3 plus the + * whitelist positive path. + * + *

The sanitizer is provider-first with default-deny: only providerId in + * {@code {openai, azure-openai}} is allowed to carry {@code reasoning_effort}. + * All other providers (including unknown ones) must strip regardless of what + * {@code request.model()} says, because the model name may have leaked from a + * failover primary. + */ +class ReasoningEffortSanitizerTest { + + private static ModelProviderEntity provider(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + return p; + } + + /** + * Construct a minimal {@link OpenAiApi.ChatCompletionRequest} via its record canonical + * constructor with only {@code messages}, {@code model}, and {@code reasoningEffort} set + * — everything else is null. {@code ChatCompletionRequest} in Spring AI 1.1.4 has no + * public builder. + */ + private static OpenAiApi.ChatCompletionRequest request(String model, String reasoningEffort) { + return new OpenAiApi.ChatCompletionRequest( + List.of(), // messages + model, // model + null, // store + null, // metadata + null, // frequencyPenalty + null, // logitBias + null, // logprobs + null, // topLogprobs + null, // maxTokens + null, // maxCompletionTokens + null, // n + null, // outputModalities + null, // audioParameters + null, // presencePenalty + null, // responseFormat + null, // seed + null, // serviceTier + null, // stop + null, // stream + null, // streamOptions + null, // temperature + null, // topP + null, // tools + null, // toolChoice + null, // parallelToolCalls + null, // user + reasoningEffort, // reasoningEffort + null, // webSearchOptions + null, // verbosity + null, // promptCacheKey + null, // safetyIdentifier + null // extraBody + ); + } + + @Test + @DisplayName("Whitelist: openai is allowed") + void whitelist_openai() { + assertTrue(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("openai"))); + } + + @Test + @DisplayName("Whitelist: azure-openai is allowed") + void whitelist_azureOpenai() { + assertTrue(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("azure-openai"))); + } + + @Test + @DisplayName("Whitelist: case-insensitive (Azure-OpenAI)") + void whitelist_caseInsensitive() { + assertTrue(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("Azure-OpenAI"))); + } + + @Test + @DisplayName("Whitelist: deepseek is denied") + void denylist_deepseek() { + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("deepseek"))); + } + + @Test + @DisplayName("Whitelist: kimi family denied") + void denylist_kimi() { + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("kimi-cn"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("kimi-intl"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("kimi-code"))); + } + + @Test + @DisplayName("Whitelist: dashscope / ollama / anthropic denied") + void denylist_misc() { + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("dashscope"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("ollama"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(provider("anthropic"))); + } + + @Test + @DisplayName("Whitelist: unknown providerId denied (default-deny — §5.2 Case E3.3)") + void denylist_unknownProvider() { + // This is the critical regression guard: if anyone re-adds a default-allow + // branch to isReasoningEffortWhitelistedProvider, this case fails first. + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider( + provider("my-custom-openai-compat-gateway"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider( + provider("openrouter"))); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider( + provider("together"))); + } + + @Test + @DisplayName("Whitelist: null provider / null providerId denied") + void denylist_nulls() { + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(null)); + assertFalse(AgentGraphBuilder.isReasoningEffortWhitelistedProvider(new ModelProviderEntity())); + } + + // ---------- sanitizeReasoningEffortForProvider ---------- + + @Test + @DisplayName("Sanitize no-op: request has no reasoning_effort") + void sanitize_noop_noReasoningEffort() { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", null); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider("deepseek")); + assertSame(req, out, "should return same instance when reasoning_effort is already null"); + } + + @Test + @DisplayName("§5.2 Case E3.1: primary=gpt-5 → fallback=deepseek strips reasoning_effort") + void sanitize_failover_deepseek_strips() { + // Simulate failover: OpenAiChatOptions.model still leaked as "gpt-5" on the deepseek request. + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "high"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider("deepseek")); + assertNull(out.reasoningEffort(), "deepseek is not on the whitelist — strip regardless of model name"); + // Other fields preserved + assertEquals("gpt-5", out.model()); + } + + @Test + @DisplayName("§5.2 Case E3.2: kimi / dashscope / ollama also strip") + void sanitize_failover_otherDenied_strips() { + for (String pid : List.of("kimi-cn", "kimi-intl", "kimi-code", "dashscope", "ollama", "anthropic")) { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "medium"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider(pid)); + assertNull(out.reasoningEffort(), "provider=" + pid + " must strip"); + } + } + + @Test + @DisplayName("§5.2 Case E3.3: unknown provider strips (default-deny regression guard)") + void sanitize_unknownProvider_strips() { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "high"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider( + req, provider("my-custom-openai-compat-gateway")); + assertNull(out.reasoningEffort(), + "unknown provider must strip (default-deny) — if this fails, someone re-added default-allow"); + } + + @Test + @DisplayName("Whitelist + supporting model: keep reasoning_effort (gpt-5 on openai)") + void sanitize_whitelisted_supportingModel_keeps() { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "high"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider("openai")); + assertSame(req, out, "gpt-5 on openai should pass through unchanged"); + assertEquals("high", out.reasoningEffort()); + } + + @Test + @DisplayName("Whitelist + non-supporting model: strip (gpt-4 on openai)") + void sanitize_whitelisted_nonSupportingModel_strips() { + // gpt-4 is NOT OPENAI_REASONING family — reasoning_effort is not applicable there. + OpenAiApi.ChatCompletionRequest req = request("gpt-4", "medium"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider("openai")); + assertNull(out.reasoningEffort(), + "gpt-4 is whitelisted-provider but non-supporting-family — family gate should strip"); + } + + @Test + @DisplayName("Azure OpenAI with supporting model: keep reasoning_effort") + void sanitize_azureOpenai_supporting_keeps() { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "low"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, provider("azure-openai")); + assertEquals("low", out.reasoningEffort()); + } + + @Test + @DisplayName("Null provider: strip (defensive)") + void sanitize_nullProvider_strips() { + OpenAiApi.ChatCompletionRequest req = request("gpt-5", "high"); + OpenAiApi.ChatCompletionRequest out = AgentGraphBuilder.sanitizeReasoningEffortForProvider(req, null); + assertNull(out.reasoningEffort()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceTest.java b/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceTest.java new file mode 100644 index 00000000..d5136b1f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceTest.java @@ -0,0 +1,350 @@ +package vip.mate.agent.binding; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.agent.binding.model.AgentToolBinding; +import vip.mate.agent.binding.service.AgentBindingService; +import vip.mate.exception.MateClawException; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * 覆盖 issue #8 的重绑定 bug 回归:在去掉 @TableLogic 之前, + * bind → unbind → rebind 会因为 uk_agent_tool / uk_agent_skill 唯一索引 + * 与软删除并存而抛 DuplicateKeyException。本测试断言修复后各条路径都成功, + * 同时断言合法的唯一约束仍被保留(不能让修 bug 顺带破坏唯一性)。 + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:binding_test_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none" +}) +class AgentBindingServiceTest { + + private static final AtomicLong AGENT_ID_SEQ = new AtomicLong(9_000_000L); + + @Autowired + private AgentBindingService bindingService; + + @Autowired + private JdbcTemplate jdbcTemplate; + + private long agentId; + + @BeforeEach + void setUp() { + // Each test gets its own agent id so concurrent runs don't clash. + agentId = AGENT_ID_SEQ.getAndIncrement(); + // Seed a real mate_agent row so AgentBindingService.requireSameWorkspace + // can resolve the agent's workspace during bindSkill/setSkillBindings. + // Tool-binding tests don't strictly need it but seeding is cheap and + // keeps every code path realistic. + seedAgent(agentId); + } + + private void seedAgent(long id) { + jdbcTemplate.update( + "MERGE INTO mate_agent (id, name, agent_type, system_prompt, max_iterations, enabled, " + + "workspace_id, create_time, update_time, deleted) " + + "KEY(id) VALUES (?, ?, 'react', '', 10, TRUE, 1, " + + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + id, "binding-test-agent-" + id); + } + + private void seedSkill(long id) { + seedSkill(id, 1L); + } + + /** + * Skill seeder with explicit workspace_id so the cross-workspace + * rejection path can be exercised. + */ + private void seedSkill(long id, long workspaceId) { + jdbcTemplate.update( + "MERGE INTO mate_skill (id, name, skill_type, version, enabled, builtin, " + + "workspace_id, create_time, update_time, deleted) " + + "KEY(id) VALUES (?, ?, 'dynamic', '1.0.0', TRUE, FALSE, ?, " + + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + id, "binding-test-skill-" + id, workspaceId); + } + + /** + * ACP endpoint seeder. The bridge synthesizes a virtual skill with id + * {@code AcpSkillBridge.VIRTUAL_ID_BASE + endpointId} and inherits + * {@code workspaceId} from the row, so this is the lever for testing + * the bridge-backed workspace check in {@code requireSameWorkspace}. + */ + private void seedAcpEndpoint(long endpointId, long workspaceId) { + jdbcTemplate.update( + "MERGE INTO mate_acp_endpoint (id, name, command, builtin, trusted, enabled, " + + "workspace_id, create_time, update_time, deleted) " + + "KEY(id) VALUES (?, ?, 'echo', FALSE, TRUE, TRUE, ?, " + + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + endpointId, "binding-test-acp-" + endpointId, workspaceId); + } + + @Test + @DisplayName("bindTool → unbindTool → bindTool 同一 (agent, tool) 不抛异常") + void rebindToolAfterUnbind() { + bindingService.bindTool(agentId, "echo"); + bindingService.unbindTool(agentId, "echo"); + assertDoesNotThrow(() -> bindingService.bindTool(agentId, "echo")); + + Set names = bindingService.getBoundToolNames(agentId); + assertNotNull(names); + assertTrue(names.contains("echo")); + } + + @Test + @DisplayName("bindSkill → unbindSkill → bindSkill 同一 (agent, skill) 不抛异常") + void rebindSkillAfterUnbind() { + long skillId = 7_777_001L; + seedSkill(skillId); + bindingService.bindSkill(agentId, skillId); + bindingService.unbindSkill(agentId, skillId); + assertDoesNotThrow(() -> bindingService.bindSkill(agentId, skillId)); + + Set ids = bindingService.getBoundSkillIds(agentId); + assertNotNull(ids); + assertTrue(ids.contains(skillId)); + } + + @Test + @DisplayName("setToolBindings 连续调用两次相同列表不抛异常,状态收敛") + void setToolBindingsIsIdempotent() { + // setToolBindings now refuses unknown tool names (so an API caller + // can't write a binding the runtime won't be able to resolve). + // Seed two real rows in mate_tool first so the validator considers + // the names bindable; the test's intent — idempotent persistence — + // is unchanged. + seedBuiltinTool("tool_a"); + seedBuiltinTool("tool_b"); + List desired = List.of("tool_a", "tool_b"); + bindingService.setToolBindings(agentId, desired); + assertDoesNotThrow(() -> bindingService.setToolBindings(agentId, desired)); + + Set names = bindingService.getBoundToolNames(agentId); + assertNotNull(names); + assertEquals(2, names.size()); + assertTrue(names.containsAll(desired)); + } + + private void seedBuiltinTool(String name) { + jdbcTemplate.update( + "MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) " + + "KEY(name) VALUES (?, ?, ?, ?, 'builtin', ?, '🔧', TRUE, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + System.nanoTime(), name, name, "test fixture", name); + } + + @Test + @DisplayName("setSkillBindings 连续调用两次相同列表不抛异常,状态收敛") + void setSkillBindingsIsIdempotent() { + List desired = List.of(7_777_101L, 7_777_102L); + desired.forEach(this::seedSkill); + bindingService.setSkillBindings(agentId, desired); + assertDoesNotThrow(() -> bindingService.setSkillBindings(agentId, desired)); + + Set ids = bindingService.getBoundSkillIds(agentId); + assertNotNull(ids); + assertEquals(2, ids.size()); + assertTrue(ids.containsAll(desired)); + } + + @Test + @DisplayName("唯一性回归:同一 (agent, tool) 直接 INSERT 第二行仍被唯一索引拦截") + void uniqueIndexStillEnforcedForTool() { + bindingService.bindTool(agentId, "unique_probe"); + + // 绕过 service,直接 INSERT 第二行,断言 DB 层唯一约束仍生效 + assertThrows(DuplicateKeyException.class, () -> + jdbcTemplate.update( + "INSERT INTO mate_agent_tool " + + "(id, agent_id, tool_name, enabled, create_time, update_time, deleted) " + + "VALUES (?, ?, ?, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + System.nanoTime(), agentId, "unique_probe" + ) + ); + } + + @Test + @DisplayName("唯一性回归:同一 (agent, skill) 直接 INSERT 第二行仍被唯一索引拦截") + void uniqueIndexStillEnforcedForSkill() { + long skillId = 7_777_201L; + seedSkill(skillId); + bindingService.bindSkill(agentId, skillId); + + assertThrows(DuplicateKeyException.class, () -> + jdbcTemplate.update( + "INSERT INTO mate_agent_skill " + + "(id, agent_id, skill_id, enabled, create_time, update_time, deleted) " + + "VALUES (?, ?, ?, TRUE, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + System.nanoTime(), agentId, skillId + ) + ); + } + + @Test + @DisplayName("bindSkill 拒绝跨 workspace 的真实 skill(防止 tenancy 越界)") + void bindSkillRejectsCrossWorkspaceRealSkill() { + // Agent lives in workspace 1 (set up in @BeforeEach). Seed a skill + // in workspace 2 — the new requireSameWorkspace check should refuse. + long otherWorkspaceSkillId = 7_777_301L; + seedSkill(otherWorkspaceSkillId, 2L); + + MateClawException ex = assertThrows(MateClawException.class, + () -> bindingService.bindSkill(agentId, otherWorkspaceSkillId)); + assertEquals(403, ex.getCode(), "应返回 403 业务码(跨 workspace 越界)"); + assertEquals("err.skill.cross_workspace_binding", ex.getMsgKey(), + "应使用专用 i18n key,前端可精确分支"); + + // No row should have been written before the check failed. + Integer count = jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM mate_agent_skill WHERE agent_id = ? AND skill_id = ?", + Integer.class, agentId, otherWorkspaceSkillId); + assertNotNull(count); + assertEquals(0, count, "拒绝时不能写入绑定行"); + } + + @Test + @DisplayName("bindSkill 允许 MCP 虚拟 skill(McpServerEntity 无 workspace,全局共享)") + void bindSkillAllowsVirtualMcpSkill() { + // Virtual MCP id range starts at McpSkillBridge.VIRTUAL_ID_BASE (9e18). + // No mate_skill or mate_mcp_server seeding needed — the bridge is + // bypassed entirely for MCP because there's no workspace to compare. + long virtualMcpId = vip.mate.skill.mcp.McpSkillBridge.VIRTUAL_ID_BASE + 42L; + assertDoesNotThrow(() -> bindingService.bindSkill(agentId, virtualMcpId)); + + Set ids = bindingService.getBoundSkillIds(agentId); + assertNotNull(ids); + assertTrue(ids.contains(virtualMcpId), "MCP virtual binding 应当落到 mate_agent_skill"); + } + + @Test + @DisplayName("bindSkill 允许同 workspace 的 ACP 虚拟 skill(走 AcpSkillBridge 解析 workspace)") + void bindSkillAllowsVirtualAcpSkillSameWorkspace() { + long endpointId = 4_242_001L; + seedAcpEndpoint(endpointId, 1L); // matches the agent's workspace + long virtualAcpId = vip.mate.skill.acp.AcpSkillBridge.VIRTUAL_ID_BASE + endpointId; + + assertDoesNotThrow(() -> bindingService.bindSkill(agentId, virtualAcpId)); + + Set ids = bindingService.getBoundSkillIds(agentId); + assertNotNull(ids); + assertTrue(ids.contains(virtualAcpId), "ACP virtual binding 应当落到 mate_agent_skill"); + } + + @Test + @DisplayName("bindSkill 拒绝跨 workspace 的 ACP 虚拟 skill(endpoint 的 workspace 与 agent 不一致)") + void bindSkillRejectsVirtualAcpSkillCrossWorkspace() { + long endpointId = 4_242_002L; + seedAcpEndpoint(endpointId, 2L); // different workspace from the agent (=1) + long virtualAcpId = vip.mate.skill.acp.AcpSkillBridge.VIRTUAL_ID_BASE + endpointId; + + MateClawException ex = assertThrows(MateClawException.class, + () -> bindingService.bindSkill(agentId, virtualAcpId)); + assertEquals(403, ex.getCode()); + assertEquals("err.skill.cross_workspace_binding", ex.getMsgKey()); + } + + @Test + @DisplayName("setSkillBindings 在批量中先做完所有校验,再删旧绑定(半成品保护)") + void setSkillBindingsValidatesBeforeMutating() { + // Seed one good skill (ws=1, same as agent) so getBoundSkillIds + // is non-empty before the failing batch. Then call setSkillBindings + // with one good + one cross-workspace id — the whole batch must be + // refused and the original binding must survive untouched. + long goodSkill = 7_777_401L; + seedSkill(goodSkill, 1L); + bindingService.bindSkill(agentId, goodSkill); + + long badSkill = 7_777_402L; + seedSkill(badSkill, 2L); + + assertThrows(MateClawException.class, + () -> bindingService.setSkillBindings(agentId, List.of(goodSkill, badSkill))); + + // The pre-existing binding to goodSkill must still be there — + // validation should have failed before the DELETE ran. + Set remaining = bindingService.getBoundSkillIds(agentId); + assertNotNull(remaining); + assertTrue(remaining.contains(goodSkill), + "validation 必须在 delete 旧绑定之前完成,否则会留下空绑定状态"); + } + + /** + * Seed a connected MCP server with one cached tool so + * {@link vip.mate.tool.service.AvailableToolService#listAvailable()} + * returns at least one bindable MCP row. + */ + private void seedMcpServerWithOneTool(long id, String serverName, String rawToolName) { + String toolsCacheJson = "[{\"name\":\"" + rawToolName + "\",\"description\":\"fixture\"}]"; + jdbcTemplate.update( + "MERGE INTO mate_mcp_server (id, name, description, transport, enabled, " + + "connect_timeout_seconds, read_timeout_seconds, last_status, tool_count, " + + "builtin, tools_cache_json, create_time, update_time, deleted) " + + "KEY(id) VALUES (?, ?, '', 'stdio', TRUE, 30, 30, 'connected', 1, FALSE, ?, " + + "CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0)", + id, serverName, toolsCacheJson); + } + + @Test + @DisplayName("Issue #108: 绑定任意 builtin tool 后,enabled MCP 工具仍自动出现在 effective allowlist") + void mcpToolsAutoIncludedWhenAnyBindingExists() { + // Reproduce the user-reported scenario: agent has one built-in tool + // bound (e.g. by template), no MCP tools ticked. Before the fix this + // returned a whitelist that excluded every MCP tool; after the fix + // MCP tools auto-join the allowlist. + seedBuiltinTool("builtin_probe"); + seedMcpServerWithOneTool(8_888_001L, "issue108-server", "search_web"); + bindingService.setToolBindings(agentId, List.of("builtin_probe")); + + Set effective = bindingService.getEffectiveToolNames(agentId); + assertNotNull(effective, "binding 非空时应返回 allowlist(非 null)"); + assertTrue(effective.contains("builtin_probe"), "用户显式勾选的工具必须在 allowlist 中"); + boolean hasMcpEntry = effective.stream().anyMatch(n -> n != null && n.startsWith("mcp_")); + assertTrue(hasMcpEntry, + "enabled MCP server 的工具必须自动并入 allowlist;缺失会让用户在 chat 时只见到 built-in 工具," + + "即 issue #108 描述的现象。实际 allowlist: " + effective); + } + + @Test + @DisplayName("Issue #108: agent 完全没绑定时 effective allowlist 返回 null(不要意外改成 strict)") + void noBindingsStillReturnsNull() { + // The auto-union must not flip the three-state contract: an agent + // with zero bindings still means "no agent-level restriction". + seedMcpServerWithOneTool(8_888_002L, "issue108-no-binding-server", "search_web"); + + Set effective = bindingService.getEffectiveToolNames(agentId); + assertNull(effective, "完全没有 skill / tool 绑定时必须返回 null(= 不过滤)," + + "否则 AgentToolSet.withAllowedToolsOnly 会变成空集禁掉所有工具"); + } + + @Test + @DisplayName("unbindTool 后 DB 里真的没行(物理 delete,不是软删留 deleted=1)") + void unbindPhysicallyRemovesRow() { + bindingService.bindTool(agentId, "physical_check"); + bindingService.unbindTool(agentId, "physical_check"); + + Integer count = jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM mate_agent_tool WHERE agent_id = ? AND tool_name = ?", + Integer.class, agentId, "physical_check" + ); + assertNotNull(count); + assertEquals(0, count, "unbind 应该物理删除,而不是软删(软删会留 deleted=1 行,占用唯一索引槽位导致 rebind 失败)"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceValidationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceValidationTest.java new file mode 100644 index 00000000..dbc9a34e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/binding/AgentBindingServiceValidationTest.java @@ -0,0 +1,186 @@ +package vip.mate.agent.binding; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.agent.binding.model.AgentToolBinding; +import vip.mate.agent.binding.repository.AgentProviderPreferenceMapper; +import vip.mate.agent.binding.repository.AgentSkillBindingMapper; +import vip.mate.agent.binding.repository.AgentToolBindingMapper; +import vip.mate.agent.binding.service.AgentBindingService; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.exception.MateClawException; +import vip.mate.skill.acp.AcpSkillBridge; +import vip.mate.skill.repository.SkillMapper; +import vip.mate.skill.runtime.SkillRuntimeService; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.service.AvailableToolService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit-level coverage for {@code setToolBindings}'s validation gate — + * proves that hand-crafted API requests can't write a tool name the + * runtime won't be able to resolve. + */ +class AgentBindingServiceValidationTest { + + private AgentToolBindingMapper toolBindingMapper; + private AvailableToolService availableToolService; + private AgentBindingService service; + + @BeforeEach + void setUp() { + AgentSkillBindingMapper skillBindingMapper = mock(AgentSkillBindingMapper.class); + toolBindingMapper = mock(AgentToolBindingMapper.class); + AgentProviderPreferenceMapper providerPreferenceMapper = mock(AgentProviderPreferenceMapper.class); + SkillRuntimeService skillRuntimeService = mock(SkillRuntimeService.class); + availableToolService = mock(AvailableToolService.class); + // Tool-binding tests don't exercise the agent/skill workspace lookup, + // so empty mocks are enough — the wired-in fields just need to be + // non-null for construction. + AgentMapper agentMapper = mock(AgentMapper.class); + SkillMapper skillMapper = mock(SkillMapper.class); + AcpSkillBridge acpSkillBridge = mock(AcpSkillBridge.class); + service = new AgentBindingService( + skillBindingMapper, + toolBindingMapper, + providerPreferenceMapper, + skillRuntimeService, + availableToolService, + agentMapper, + skillMapper, + acpSkillBridge); + // No existing binding by default — each test overrides as needed. + when(toolBindingMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of()); + } + + @Test + @DisplayName("a known available tool name persists") + void availableNameIsAccepted() { + when(availableToolService.listAvailable()).thenReturn(List.of( + bindable("web_search"), + bindable("mcp_42_search_aaaaaa"))); + + service.setToolBindings(99L, List.of("mcp_42_search_aaaaaa")); + + ArgumentCaptor captor = ArgumentCaptor.forClass(AgentToolBinding.class); + verify(toolBindingMapper, times(1)).insert(captor.capture()); + assertEquals("mcp_42_search_aaaaaa", captor.getValue().getToolName()); + } + + @Test + @DisplayName("an unknown name (typo / legacy unprefixed) is refused") + void unknownNameIsRejected() { + when(availableToolService.listAvailable()).thenReturn(List.of( + bindable("mcp_42_search_aaaaaa"))); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, List.of("search_typo"))); + assertTrue(ex.getMessage().contains("search_typo"), + "error should name the rejected tool, got: " + ex.getMessage()); + // Nothing should have been persisted — validation runs before delete. + verify(toolBindingMapper, never()).delete(any()); + verify(toolBindingMapper, never()).insert(any(AgentToolBinding.class)); + } + + @Test + @DisplayName("a name marked available=false (e.g. hash collision) is refused") + void unavailableNameIsRejected() { + AvailableToolDTO collided = AvailableToolDTO.builder() + .name("mcp_42_search_aaaaaa") + .available(false) + .unavailableReason("HASH_COLLISION:other") + .build(); + when(availableToolService.listAvailable()).thenReturn(List.of(collided)); + + assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, List.of("mcp_42_search_aaaaaa"))); + verify(toolBindingMapper, never()).delete(any()); + } + + @Test + @DisplayName("a stale/unavailable name already in the existing binding can be removed (not blocked)") + void existingUnbindableCanBeRemoved() { + // Existing binding holds a name that has since become unavailable. + // The user removes it — passing an empty incoming list. Validation + // must NOT block this because the new name set introduces nothing + // new to validate. + AgentToolBinding existing = new AgentToolBinding(); + existing.setAgentId(99L); + existing.setToolName("mcp_42_search_aaaaaa"); + existing.setEnabled(true); + when(toolBindingMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(existing)); + when(availableToolService.listAvailable()).thenReturn(List.of()); // tool no longer available + + service.setToolBindings(99L, List.of()); + + verify(toolBindingMapper, times(1)).delete(any()); + verify(toolBindingMapper, never()).insert(any(AgentToolBinding.class)); + } + + @Test + @DisplayName("keeping an existing-but-now-stale binding is allowed; adding a NEW unknown is still refused") + void mixedKeepAndUnknownAdd() { + // Existing has one binding; user tries to keep it AND add a typo. + AgentToolBinding existing = new AgentToolBinding(); + existing.setAgentId(99L); + existing.setToolName("mcp_42_search_aaaaaa"); + existing.setEnabled(true); + when(toolBindingMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(existing)); + // Only a different name is currently available. + when(availableToolService.listAvailable()).thenReturn(List.of( + bindable("web_search"))); + + assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, List.of("mcp_42_search_aaaaaa", "typo"))); + verify(toolBindingMapper, never()).delete(any()); + } + + @Test + @DisplayName("blank or null entries in incoming list are rejected") + void blankEntriesAreRejected() { + when(availableToolService.listAvailable()).thenReturn(List.of(bindable("web_search"))); + assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, java.util.Arrays.asList("web_search", ""))); + assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, java.util.Arrays.asList("web_search", (String) null))); + } + + @Test + @DisplayName("AvailableToolService failure: validation refuses any new name (conservative)") + void availableServiceFailureIsConservative() { + when(availableToolService.listAvailable()).thenThrow(new RuntimeException("picker down")); + + // Existing-only saves still succeed. + AgentToolBinding existing = new AgentToolBinding(); + existing.setAgentId(99L); + existing.setToolName("web_search"); + when(toolBindingMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(existing)); + service.setToolBindings(99L, List.of("web_search")); + verify(toolBindingMapper, times(1)).delete(any()); + + // Adding a new one fails fast. + assertThrows(MateClawException.class, + () -> service.setToolBindings(99L, List.of("web_search", "another"))); + } + + private static AvailableToolDTO bindable(String name) { + return AvailableToolDTO.builder() + .name(name) + .available(true) + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilderClaude47Test.java b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilderClaude47Test.java new file mode 100644 index 00000000..d8e90124 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentAnthropicChatModelBuilderClaude47Test.java @@ -0,0 +1,71 @@ +package vip.mate.agent.chatmodel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-001 (Claude 4.7 contract): {@link AgentAnthropicChatModelBuilder#isClaude47} + * must correctly classify the model variants we'll see in production. + * + *

Reference: hermes-agent {@code anthropic_adapter._NO_SAMPLING_PARAMS_SUBSTRINGS}. + * Claude 4.7 forbids temperature / top_p / top_k entirely — the builder relies + * on this detector to skip those fields rather than letting Anthropic 400. + */ +class AgentAnthropicChatModelBuilderClaude47Test { + + @Test + @DisplayName("isClaude47 detects hyphenated direct-API model names") + void detect_hyphenated() { + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude-opus-4-7")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude-sonnet-4-7")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude-haiku-4-7")); + } + + @Test + @DisplayName("isClaude47 detects dotted variants (e.g. OpenRouter / mixed dialects)") + void detect_dotted() { + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude-opus-4.7")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude.sonnet.4.7")); + } + + @Test + @DisplayName("isClaude47 detects OpenRouter-style prefixed model ids") + void detect_openrouterPrefix() { + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("anthropic/claude-opus-4-7")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("anthropic/claude-sonnet-4-7")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("anthropic/claude-opus-4.7")); + } + + @Test + @DisplayName("isClaude47 ignores 4.5 / 4.6 / 4.0 / 3.x and unrelated names") + void detect_negatives() { + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("claude-opus-4-6")); + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("claude-sonnet-4-5")); + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("claude-3-7-sonnet"), + "3.7 must not match 4.7"); + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("claude-3-5-sonnet")); + // The "claude" prefix guard prevents non-Anthropic models from spuriously + // matching even if they contain "4-7" / "4.7" substrings. + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("gpt-4-7"), + "Non-Claude models must NOT match — claude prefix guard active"); + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("nemotron-4-7-instruct")); + } + + @Test + @DisplayName("isClaude47 null-safe") + void detect_nullSafe() { + assertFalse(AgentAnthropicChatModelBuilder.isClaude47(null)); + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("")); + } + + @Test + @DisplayName("Note: claude-3-7-sonnet correctly distinguished from claude-4-7-*") + void detect_3_7_vs_4_7() { + // Both contain "-7" but only the second contains "4-7" as a substring. + assertFalse(AgentAnthropicChatModelBuilder.isClaude47("claude-3-7-sonnet-20250219")); + assertTrue(AgentAnthropicChatModelBuilder.isClaude47("claude-opus-4-7-20260415"), + "Date-stamped 4-7 variants must still match"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentClaudeCodeChatModelBuilderTest.java b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentClaudeCodeChatModelBuilderTest.java new file mode 100644 index 00000000..a5bd62fc --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/AgentClaudeCodeChatModelBuilderTest.java @@ -0,0 +1,138 @@ +package vip.mate.agent.chatmodel; + +import io.micrometer.observation.ObservationRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.ai.anthropic.api.AnthropicApi; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.web.client.RestClient; +import org.springframework.web.reactive.function.client.WebClient; +import vip.mate.exception.MateClawException; +import vip.mate.llm.anthropic.oauth.ClaudeCodeApiHeaders; +import vip.mate.llm.anthropic.oauth.ClaudeCodeOAuthService; +import vip.mate.llm.anthropic.oauth.ClaudeCodeVersionDetector; +import vip.mate.llm.model.ModelProtocol; + +import java.util.function.Supplier; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Header-construction + token-fetch coverage for the Claude Code OAuth chat + * model builder. Building a real {@link AnthropicApi} doesn't make a network + * call (Spring AI defers all I/O to {@code chatCompletionEntity}), so these + * tests can exercise the full assembly path without mocking the API client. + */ +@ExtendWith(MockitoExtension.class) +class AgentClaudeCodeChatModelBuilderTest { + + @Mock + private AgentAnthropicChatModelBuilder anthropicBuilder; + + @Mock + private ClaudeCodeOAuthService oauthService; + + private ClaudeCodeApiHeaders apiHeaders; + + private AgentClaudeCodeChatModelBuilder builder; + + @BeforeEach + void setUp() { + // Real ApiHeaders with a stub version detector — the version string + // shows up verbatim in User-Agent assertions. + ClaudeCodeVersionDetector detector = new ClaudeCodeVersionDetector() { + @Override + public String get() { return "2.1.114"; } + }; + apiHeaders = new ClaudeCodeApiHeaders(detector); + + builder = new AgentClaudeCodeChatModelBuilder( + anthropicBuilder, + oauthService, + apiHeaders, + providerOf(RestClient::builder), + providerOf(WebClient::builder), + providerOf(() -> ObservationRegistry.NOOP), + new com.fasterxml.jackson.databind.ObjectMapper()); + } + + @Test + @DisplayName("supportedProtocol returns ANTHROPIC_CLAUDE_CODE") + void supportedProtocol() { + assertEquals(ModelProtocol.ANTHROPIC_CLAUDE_CODE, builder.supportedProtocol()); + } + + @Test + @DisplayName("buildOauthAnthropicApi accepts a token and produces a non-null AnthropicApi") + void buildOauthAnthropicApi_returnsClient() { + // Sanity check: the NoopApiKey path passes Spring AI's notNull assertion + // and the OAuth headers attach without throwing. If this test ever + // fails, the most likely cause is a Spring AI upgrade tightening the + // ApiKey contract — see AgentClaudeCodeChatModelBuilder javadoc. + AnthropicApi api = builder.buildOauthAnthropicApi("sk-ant-oat01-test-token"); + assertNotNull(api); + } + + @Test + @DisplayName("build delegates to oauthService and reuses anthropicBuilder.buildAnthropicOptions") + void build_invokesOauthAndReusesOptions() { + when(oauthService.getValidToken()).thenReturn("tok-123"); + // anthropicBuilder.buildAnthropicOptions returns a real options object — + // we don't need a strict comparison, just that it gets invoked once and + // its result is fed through. + when(anthropicBuilder.buildAnthropicOptions(any())) + .thenReturn(org.springframework.ai.anthropic.AnthropicChatOptions.builder().build()); + + var result = builder.build(new vip.mate.llm.model.ModelConfigEntity(), null, + RetryTemplate.defaultInstance()); + assertNotNull(result); + verify(oauthService, times(1)).getValidToken(); + verify(anthropicBuilder, times(1)).buildAnthropicOptions(any()); + } + + @Test + @DisplayName("build propagates OAuth errors without calling buildAnthropicOptions") + void build_propagatesOauthErrors() { + // Simulates "no Claude Code on disk" — caller surface is the same + // MateClawException so the global handler can format the i18n message. + when(oauthService.getValidToken()).thenThrow(new MateClawException( + "err.anthropic.no_claude_code", "no creds")); + + assertThrows(MateClawException.class, + () -> builder.build(new vip.mate.llm.model.ModelConfigEntity(), null, null)); + // anthropicBuilder shouldn't have been touched — short-circuit before + // it would have wasted a buildAnthropicOptions call. + verify(anthropicBuilder, never()).buildAnthropicOptions(any()); + } + + /* ----- ObjectProvider test helper ----- */ + + /** Minimal {@link ObjectProvider} that defers to a {@link Supplier} for {@code getIfAvailable}. */ + @SuppressWarnings("unchecked") + private static ObjectProvider providerOf(Supplier supplier) { + ObjectProvider mock = mock(ObjectProvider.class); + // Use lenient — not every test triggers a getIfAvailable call (e.g. + // the supportedProtocol test takes a short path), and the strict + // default would fail with UnnecessaryStubbingException. + lenient().when(mock.getIfAvailable(any(Supplier.class))).thenAnswer(inv -> { + Supplier fallback = inv.getArgument(0); + T v = supplier.get(); + return v != null ? v : fallback.get(); + }); + return mock; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/ClaudeCodeIdentityChatModelDecoratorTest.java b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/ClaudeCodeIdentityChatModelDecoratorTest.java new file mode 100644 index 00000000..3356f1a1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/ClaudeCodeIdentityChatModelDecoratorTest.java @@ -0,0 +1,359 @@ +package vip.mate.agent.chatmodel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.anthropic.AnthropicChatOptions; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import reactor.core.publisher.Flux; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the OAuth-mode prompt rewriting that prevents Anthropic's edge + * from rate-limiting MateClaw traffic. Each test corresponds to one of the + * transforms hermes-agent applies on {@code is_oauth=True} requests. + */ +class ClaudeCodeIdentityChatModelDecoratorTest { + + @Test + @DisplayName("transform prepends Claude Code identity as its own SystemMessage before the original") + void transform_prependsToExistingSystem() { + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt input = new Prompt(List.of( + new SystemMessage("You are a helpful coding assistant."), + new UserMessage("hi"))); + Prompt result = d.transform(input); + + // RFC-062: identity must be its OWN system block (not merged into one string) + // — Anthropic's OAuth anti-abuse gate 429s the merged form, accepts the array form. + SystemMessage identity = (SystemMessage) result.getInstructions().get(0); + assertEquals(ClaudeCodeIdentityChatModelDecorator.CLAUDE_CODE_SYSTEM_PREFIX, identity.getText()); + SystemMessage body = (SystemMessage) result.getInstructions().get(1); + assertTrue(body.getText().contains("helpful coding assistant")); + } + + @Test + @DisplayName("transform inserts a system message when none was present") + void transform_insertsSystemWhenAbsent() { + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt input = new Prompt(List.of(new UserMessage("hello"))); + Prompt result = d.transform(input); + + // First message must be a system message with just the identity prefix — + // hermes-agent does the same: system = [cc_block] when none was supplied. + Message first = result.getInstructions().get(0); + assertTrue(first instanceof SystemMessage); + assertEquals(ClaudeCodeIdentityChatModelDecorator.CLAUDE_CODE_SYSTEM_PREFIX, + ((SystemMessage) first).getText()); + // User message is preserved at index 1. + assertTrue(result.getInstructions().get(1) instanceof UserMessage); + } + + @Test + @DisplayName("transform is idempotent — second pass doesn't double-prefix") + void transform_idempotent() { + // Defends against accidental double-wrapping (e.g. nested decorators or + // a re-issue of the same Prompt). hermes-agent doesn't have this concern + // because its rewrite happens in one place; we keep this guard so the + // identity prefix doesn't compound to "You are Claude Code...You are Claude Code...". + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt original = new Prompt(List.of(new SystemMessage("Body"), new UserMessage("hi"))); + Prompt once = d.transform(original); + Prompt twice = d.transform(once); + + SystemMessage sys = (SystemMessage) twice.getInstructions().get(0); + // Identity should appear exactly once. + int firstIdx = sys.getText().indexOf(ClaudeCodeIdentityChatModelDecorator.CLAUDE_CODE_SYSTEM_PREFIX); + int secondIdx = sys.getText().indexOf( + ClaudeCodeIdentityChatModelDecorator.CLAUDE_CODE_SYSTEM_PREFIX, firstIdx + 1); + assertTrue(firstIdx >= 0 && secondIdx == -1, + "Identity prefix must appear exactly once even after multiple transform passes"); + } + + @Test + @DisplayName("sanitizeBranding replaces MateClaw references") + void sanitizeBranding_replacesProductNames() { + // Anthropic's content filter flags self-contradicting identity claims — + // a "You are Claude Code" prefix followed by a body that says "You are + // MateClaw" trips the filter. Strip the conflicting brand. + String sanitized = ClaudeCodeIdentityChatModelDecorator.sanitizeBranding( + "You are MateClaw, built on mateclaw"); + assertEquals("You are Claude Code, built on claude-code", sanitized); + } + + @Test + @DisplayName("sanitizeBranding tolerates empty / null input") + void sanitizeBranding_nullSafe() { + // Defensive — a prompt with no system text shouldn't NPE here. + assertEquals("", ClaudeCodeIdentityChatModelDecorator.sanitizeBranding("")); + assertEquals(null, ClaudeCodeIdentityChatModelDecorator.sanitizeBranding(null)); + } + + @Test + @DisplayName("transform preserves chat options (temperature, model, etc.)") + void transform_preservesOptions() { + // Spring AI's AnthropicChatOptions carry critical per-request state + // (max_tokens, thinking budget, cache_control). Losing them on rewrite + // would silently break Claude 4.7 thinking mode. + var options = org.springframework.ai.anthropic.AnthropicChatOptions.builder() + .model("claude-opus-4-7").maxTokens(1234).build(); + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt input = new Prompt(List.of(new UserMessage("hi")), options); + Prompt result = d.transform(input); + + var resultOpts = (org.springframework.ai.anthropic.AnthropicChatOptions) result.getOptions(); + assertEquals("claude-opus-4-7", resultOpts.getModel()); + assertEquals(1234, resultOpts.getMaxTokens()); + } + + @Test + @DisplayName("call delegates the rewritten prompt downstream") + void call_delegatesRewritten() { + // Sanity: the prompt that reaches the underlying ChatModel must be the + // rewritten one, not the original — otherwise the decorator is dead code. + AtomicReference captured = new AtomicReference<>(); + ChatModel capturing = new TestDelegate() { + @Override + public ChatResponse call(Prompt prompt) { + captured.set(prompt); + return null; + } + }; + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(capturing); + d.call(new Prompt(List.of(new UserMessage("hi")))); + + assertNotNull(captured.get()); + Message first = captured.get().getInstructions().get(0); + assertTrue(first instanceof SystemMessage); + assertEquals(ClaudeCodeIdentityChatModelDecorator.CLAUDE_CODE_SYSTEM_PREFIX, + ((SystemMessage) first).getText()); + } + + @Test + @DisplayName("transform leaves non-system messages untouched") + void transform_preservesUserAndAssistantMessages() { + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt input = new Prompt(List.of( + new SystemMessage("be helpful"), + new UserMessage("question 1"), + new AssistantMessage("answer 1"), + new UserMessage("question 2"))); + Prompt result = d.transform(input); + + // RFC-062: system splits into [identity, sanitized body] so user/assistant + // shift to indices 2, 3, 4. Their content is the original instance — a copy + // here would force Spring AI to re-encode multimodal content (images, + // tool_results) for no benefit. + assertTrue(result.getInstructions().get(2) instanceof UserMessage); + assertEquals("question 1", ((UserMessage) result.getInstructions().get(2)).getText()); + assertEquals("answer 1", ((AssistantMessage) result.getInstructions().get(3)).getText()); + assertEquals("question 2", ((UserMessage) result.getInstructions().get(4)).getText()); + } + + @Test + @DisplayName("transform wraps tool callbacks so getToolDefinition().name() returns mcp_") + void transform_prefixesOutgoingToolNames() { + // Anthropic's anti-abuse path inspects tool definitions; tools without + // the mcp_ prefix on a request claiming Claude Code identity get the + // request rate-limited (429 with body "Error"). Ensure we wrap. + ToolCallback search = stubToolCallback("search", "Search the web"); + ToolCallback createFile = stubToolCallback("createFile", "Create a file"); + AnthropicChatOptions opts = AnthropicChatOptions.builder() + .toolCallbacks(List.of(search, createFile)) + .build(); + + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt result = d.transform(new Prompt(List.of(new UserMessage("hi")), opts)); + + AnthropicChatOptions resOpts = (AnthropicChatOptions) result.getOptions(); + List wrapped = resOpts.getToolCallbacks(); + assertEquals(2, wrapped.size()); + assertEquals("mcp_search", wrapped.get(0).getToolDefinition().name()); + assertEquals("mcp_createFile", wrapped.get(1).getToolDefinition().name()); + } + + @Test + @DisplayName("PrefixedToolCallback forwards call() to the underlying tool unchanged") + void prefixedToolCallback_forwardsCall() { + // Critical contract: prefixing happens on the wire, but MateClaw's tool + // implementation must still receive the original argument string and + // return the original output verbatim. If this fails, every tool + // execution under OAuth would silently mis-route. + AtomicReference capturedInput = new AtomicReference<>(); + ToolCallback underlying = new ToolCallback() { + @Override + public ToolDefinition getToolDefinition() { + return DefaultToolDefinition.builder().name("search").description("d").inputSchema("{}").build(); + } + @Override + public String call(String input) { + capturedInput.set(input); + return "search-output"; + } + }; + var wrapped = new ClaudeCodeIdentityChatModelDecorator.PrefixedToolCallback(underlying); + String out = wrapped.call("{\"q\":\"test\"}"); + assertEquals("search-output", out); + assertEquals("{\"q\":\"test\"}", capturedInput.get()); + assertEquals("mcp_search", wrapped.getToolDefinition().name()); + } + + @Test + @DisplayName("PrefixedToolCallback is idempotent — double-wrap doesn't double-prefix") + void prefixedToolCallback_idempotent() { + // Defends against accidental nested decoration. A wrapped wrapper + // should still expose mcp_search, not mcp_mcp_search. + ToolCallback underlying = stubToolCallback("search", "d"); + var once = new ClaudeCodeIdentityChatModelDecorator.PrefixedToolCallback(underlying); + var twice = new ClaudeCodeIdentityChatModelDecorator.PrefixedToolCallback(once); + assertEquals("mcp_search", once.getToolDefinition().name()); + assertEquals("mcp_search", twice.getToolDefinition().name()); + } + + @Test + @DisplayName("stripToolPrefixes removes mcp_ from response tool_use names") + void stripToolPrefixes_responseSide() { + // Claude returns tool_use with name="mcp_search" (because we prefixed + // the definition); MateClaw's tool registry only knows "search" so + // the prefix must come off before the response leaves the decorator. + AssistantMessage am = AssistantMessage.builder() + .content("calling search") + .toolCalls(List.of( + new AssistantMessage.ToolCall("call_1", "function", "mcp_search", + "{\"q\":\"foo\"}"), + new AssistantMessage.ToolCall("call_2", "function", "mcp_createFile", + "{\"path\":\"x\"}"))) + .build(); + ChatResponse response = new ChatResponse(List.of(new Generation(am))); + + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + ChatResponse stripped = d.stripToolPrefixes(response); + + AssistantMessage out = stripped.getResult().getOutput(); + assertEquals("search", out.getToolCalls().get(0).name()); + assertEquals("createFile", out.getToolCalls().get(1).name()); + // ID + arguments must pass through untouched — losing the call ID + // would break Anthropic's tool_result correlation on next turn. + assertEquals("call_1", out.getToolCalls().get(0).id()); + assertEquals("{\"q\":\"foo\"}", out.getToolCalls().get(0).arguments()); + } + + @Test + @DisplayName("stripToolPrefixes returns input unchanged when no tool_use blocks present") + void stripToolPrefixes_noToolCalls_passthrough() { + // Optimization: don't allocate a new list/Generation when there's + // nothing to rewrite. Verify identity-equality for the trivial case. + ChatResponse response = new ChatResponse(List.of( + new Generation(new AssistantMessage("just text")))); + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + ChatResponse out = d.stripToolPrefixes(response); + assertTrue(out == response, "no-op rewrite should return the same instance"); + } + + @Test + @DisplayName("transform re-prefixes tool_use names in AssistantMessage history") + void transform_reprefixesHistoryToolUse() { + // Prior turn: Claude called mcp_search → we stripped to "search" before + // storing → next request must re-prepend mcp_ so Anthropic's history + // matches its own prior tool_use block. Otherwise Anthropic's + // tool_use_id correlation breaks and you get "tool_use without + // matching tool_result" 400s. + AssistantMessage history = AssistantMessage.builder() + .content("") + .toolCalls(List.of( + new AssistantMessage.ToolCall("call_1", "function", "search", + "{\"q\":\"foo\"}"))) + .build(); + + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(noopDelegate()); + Prompt result = d.transform(new Prompt(List.of( + new SystemMessage("be helpful"), + history, + new UserMessage("now do that")))); + + // RFC-062: system splits into [identity, sanitized body] so AssistantMessage + // history shifts to index 2. + AssistantMessage rewrittenHistory = (AssistantMessage) result.getInstructions().get(2); + assertEquals("mcp_search", rewrittenHistory.getToolCalls().get(0).name()); + // ID stays the same so tool_result correlation chains through. + assertEquals("call_1", rewrittenHistory.getToolCalls().get(0).id()); + } + + @Test + @DisplayName("call delegates rewritten prompt and strips response prefixes end-to-end") + void call_endToEnd() { + // Integration: outgoing prompt should have prefixed tool names, and + // the AssistantMessage we return should come back unprefixed. Mirrors + // what ReasoningNode would observe per turn. + ToolCallback tool = stubToolCallback("search", "search"); + AnthropicChatOptions opts = AnthropicChatOptions.builder() + .toolCallbacks(List.of(tool)).build(); + + AtomicReference capturedPrompt = new AtomicReference<>(); + ChatModel delegate = new TestDelegate() { + @Override + public ChatResponse call(Prompt prompt) { + capturedPrompt.set(prompt); + AssistantMessage am = AssistantMessage.builder() + .content("") + .toolCalls(List.of(new AssistantMessage.ToolCall( + "call_x", "function", "mcp_search", "{}"))) + .build(); + return new ChatResponse(List.of(new Generation(am))); + } + }; + ClaudeCodeIdentityChatModelDecorator d = new ClaudeCodeIdentityChatModelDecorator(delegate); + ChatResponse out = d.call(new Prompt(List.of(new UserMessage("hi")), opts)); + + AnthropicChatOptions sentOpts = (AnthropicChatOptions) capturedPrompt.get().getOptions(); + assertEquals("mcp_search", sentOpts.getToolCallbacks().get(0).getToolDefinition().name(), + "outgoing tool name must be prefixed"); + assertEquals("search", out.getResult().getOutput().getToolCalls().get(0).name(), + "incoming tool name must be stripped"); + // Sanity — name must round-trip differently from the wire format. + assertNotEquals("mcp_search", out.getResult().getOutput().getToolCalls().get(0).name()); + } + + /* ----- Test helpers ----- */ + + private static ChatModel noopDelegate() { + return new TestDelegate(); + } + + private static ToolCallback stubToolCallback(String name, String description) { + return new ToolCallback() { + @Override + public ToolDefinition getToolDefinition() { + return DefaultToolDefinition.builder() + .name(name).description(description).inputSchema("{}").build(); + } + @Override + public String call(String input) { return "ok"; } + }; + } + + /** Minimal ChatModel that returns null/empty — sufficient for transform-only tests. */ + private static class TestDelegate implements ChatModel { + @Override + public ChatResponse call(Prompt prompt) { return null; } + @Override + public Flux stream(Prompt prompt) { return Flux.empty(); } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/DeepSeekV4ThinkingDecoratorTest.java b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/DeepSeekV4ThinkingDecoratorTest.java new file mode 100644 index 00000000..73e8b1b2 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/chatmodel/DeepSeekV4ThinkingDecoratorTest.java @@ -0,0 +1,237 @@ +package vip.mate.agent.chatmodel; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.Prompt; +import org.springframework.ai.openai.OpenAiChatOptions; +import reactor.core.publisher.Flux; +import vip.mate.agent.ThinkingLevelHolder; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates the per-request payload patches DeepSeek V4 requires. + * + *

Two independent invariants are tested separately because they're each + * easy to silently break: + *

    + *
  • {@code extraBody.thinking} + {@code reasoning_effort} on the options.
  • + *
  • {@code reasoning_content} on prior assistant tool-call messages + * (ensure-when-enabled / strip-when-disabled).
  • + *
+ */ +class DeepSeekV4ThinkingDecoratorTest { + + private DeepSeekV4ThinkingDecorator decorator; + + @BeforeEach + void setUp() { + decorator = new DeepSeekV4ThinkingDecorator(new NoopChatModel()); + } + + @AfterEach + void clearHolder() { + ThinkingLevelHolder.clear(); + } + + /* =================================================================== */ + /* Options patching */ + /* =================================================================== */ + + @Test + @DisplayName("thinking=high → extraBody.thinking={type:enabled} + reasoning_effort=high") + void thinkingHigh_injectsEnabledAndHighEffort() { + ThinkingLevelHolder.set("high"); + OpenAiChatOptions opts = OpenAiChatOptions.builder().model("deepseek-v4-flash").build(); + Prompt result = decorator.transform(new Prompt(List.of(new UserMessage("hi")), opts)); + + OpenAiChatOptions out = (OpenAiChatOptions) result.getOptions(); + assertEquals("high", out.getReasoningEffort()); + assertNotNull(out.getExtraBody()); + Object thinking = out.getExtraBody().get("thinking"); + assertEquals(Map.of("type", "enabled"), thinking, + "thinking field must be the exact {type: enabled} shape DeepSeek expects"); + } + + @Test + @DisplayName("thinking=off → extraBody.thinking={type:disabled} + reasoning_effort cleared") + void thinkingOff_clearsEffortAndDisablesThinking() { + // Critical: when thinking is disabled, BOTH fields must change. Leaving + // a stale reasoning_effort while flipping thinking off causes DeepSeek + // to 400 with "thinking and reasoning_effort cannot coexist when disabled". + ThinkingLevelHolder.set("off"); + OpenAiChatOptions opts = OpenAiChatOptions.builder() + .model("deepseek-v4-pro") + .reasoningEffort("medium") // pre-set, must be cleared + .build(); + Prompt result = decorator.transform(new Prompt(List.of(new UserMessage("hi")), opts)); + + OpenAiChatOptions out = (OpenAiChatOptions) result.getOptions(); + assertNull(out.getReasoningEffort(), "reasoning_effort must be cleared when thinking is off"); + assertEquals(Map.of("type", "disabled"), out.getExtraBody().get(DeepSeekV4ThinkingDecorator.THINKING_FIELD)); + } + + @Test + @DisplayName("Existing extraBody entries are preserved when patching") + void extraBody_preservesExistingEntries() { + // Defends against a copy-and-replace bug where the patch overwrites the + // whole map. Other extra-body fields (e.g. provider-specific knobs) must + // survive — losing them silently would break unrelated features. + ThinkingLevelHolder.set("low"); + Map seed = new HashMap<>(); + seed.put("custom_knob", 42); + OpenAiChatOptions opts = OpenAiChatOptions.builder() + .model("deepseek-v4-flash").build(); + opts.setExtraBody(seed); + + Prompt result = decorator.transform(new Prompt(List.of(new UserMessage("hi")), opts)); + OpenAiChatOptions out = (OpenAiChatOptions) result.getOptions(); + assertEquals(42, out.getExtraBody().get("custom_knob")); + assertNotNull(out.getExtraBody().get(DeepSeekV4ThinkingDecorator.THINKING_FIELD)); + } + + @Test + @DisplayName("mapEffort: low/medium/high passthrough; max collapses to high; unknown → medium") + void mapEffort_levels() { + // openclaw resolveDeepSeekV4ReasoningEffort folds "max" into "high" + // because DeepSeek doesn't expose a max tier. Pin both ends of the rule. + assertEquals("low", DeepSeekV4ThinkingDecorator.mapEffort("low")); + assertEquals("medium", DeepSeekV4ThinkingDecorator.mapEffort("medium")); + assertEquals("high", DeepSeekV4ThinkingDecorator.mapEffort("high")); + assertEquals("high", DeepSeekV4ThinkingDecorator.mapEffort("max")); + assertEquals("medium", DeepSeekV4ThinkingDecorator.mapEffort("xhigh")); + assertEquals("medium", DeepSeekV4ThinkingDecorator.mapEffort(null)); + } + + /* =================================================================== */ + /* Message patching */ + /* =================================================================== */ + + @Test + @DisplayName("enabled + tool-call history → reasoning_content key ensured (empty string)") + void messages_enabled_ensuresReasoningContent() { + // V4 replay contract: every prior assistant tool-call message must have + // a reasoning_content (empty allowed). Missing it returns an obscure 400 + // about "reasoning_content required for thinking-enabled tool replay". + AssistantMessage am = AssistantMessage.builder() + .content("") + .toolCalls(List.of( + new AssistantMessage.ToolCall("call_1", "function", "search", "{}"))) + .build(); + List patched = DeepSeekV4ThinkingDecorator.patchMessages(List.of(am), true); + + AssistantMessage out = (AssistantMessage) patched.get(0); + assertTrue(out.getMetadata().containsKey(DeepSeekV4ThinkingDecorator.REASONING_CONTENT_KEY)); + assertEquals("", out.getMetadata().get(DeepSeekV4ThinkingDecorator.REASONING_CONTENT_KEY)); + // Tool calls must pass through unchanged — losing the call ID would + // break the next turn's tool_result correlation. + assertEquals("call_1", out.getToolCalls().get(0).id()); + } + + @Test + @DisplayName("enabled + already-has reasoning_content → no rewrite (fast path)") + void messages_enabled_noRewriteWhenAlreadyPresent() { + Map meta = new HashMap<>(); + meta.put(DeepSeekV4ThinkingDecorator.REASONING_CONTENT_KEY, "prev thinking"); + AssistantMessage am = AssistantMessage.builder() + .content("answer") + .properties(meta) + .toolCalls(List.of( + new AssistantMessage.ToolCall("call_2", "function", "search", "{}"))) + .build(); + List patched = DeepSeekV4ThinkingDecorator.patchMessages(List.of(am), true); + + // Identity equality — fast path returns the same instance to avoid pointless allocation. + assertTrue(patched.get(0) == am, "no-op rewrite should return the same instance"); + } + + @Test + @DisplayName("disabled → reasoning_content stripped from prior messages") + void messages_disabled_stripsReasoningContent() { + // DeepSeek echoes prior reasoning_content back into the response when + // thinking is disabled, polluting the user-visible answer. Stripping is + // not optional. + Map meta = new HashMap<>(); + meta.put(DeepSeekV4ThinkingDecorator.REASONING_CONTENT_KEY, "old thinking"); + meta.put("other_meta", "preserved"); + AssistantMessage am = AssistantMessage.builder() + .content("answer") + .properties(meta) + .build(); + + List patched = DeepSeekV4ThinkingDecorator.patchMessages(List.of(am), false); + AssistantMessage out = (AssistantMessage) patched.get(0); + assertFalse(out.getMetadata().containsKey(DeepSeekV4ThinkingDecorator.REASONING_CONTENT_KEY), + "reasoning_content must be removed"); + assertEquals("preserved", out.getMetadata().get("other_meta"), + "Other metadata keys must survive the strip"); + } + + @Test + @DisplayName("disabled + no reasoning_content → no-op pass-through") + void messages_disabled_noOpWhenAbsent() { + AssistantMessage am = new AssistantMessage("plain answer"); + List patched = DeepSeekV4ThinkingDecorator.patchMessages(List.of(am), false); + assertTrue(patched.get(0) == am, "no-op rewrite should return the same instance"); + } + + @Test + @DisplayName("Non-assistant messages pass through untouched") + void messages_userPassesThrough() { + // patchMessages must only touch AssistantMessage. UserMessage / ToolMessage + // / SystemMessage carry meaning the decorator has no business modifying. + UserMessage user = new UserMessage("question"); + List patched = DeepSeekV4ThinkingDecorator.patchMessages(List.of(user), true); + assertTrue(patched.get(0) == user); + } + + /* =================================================================== */ + /* End-to-end delegate */ + /* =================================================================== */ + + @Test + @DisplayName("call() delegates the patched prompt to the underlying ChatModel") + void call_delegatesPatched() { + // Sanity: the prompt that reaches the underlying model carries the + // patched options/messages, not the originals. + AtomicReference captured = new AtomicReference<>(); + DeepSeekV4ThinkingDecorator d = new DeepSeekV4ThinkingDecorator(new NoopChatModel() { + @Override public ChatResponse call(Prompt prompt) { + captured.set(prompt); + return null; + } + }); + ThinkingLevelHolder.set("medium"); + OpenAiChatOptions opts = OpenAiChatOptions.builder().model("deepseek-v4-flash").build(); + d.call(new Prompt(List.of(new UserMessage("hi")), opts)); + + assertNotNull(captured.get()); + OpenAiChatOptions sentOpts = (OpenAiChatOptions) captured.get().getOptions(); + assertEquals("medium", sentOpts.getReasoningEffort()); + assertEquals(Map.of("type", "enabled"), + sentOpts.getExtraBody().get(DeepSeekV4ThinkingDecorator.THINKING_FIELD)); + } + + /* ---------- Test double ---------- */ + + private static class NoopChatModel implements ChatModel { + @Override public ChatResponse call(Prompt prompt) { return null; } + @Override public Flux stream(Prompt prompt) { return Flux.empty(); } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ChatOriginTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ChatOriginTest.java new file mode 100644 index 00000000..d84fde7b --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ChatOriginTest.java @@ -0,0 +1,76 @@ +package vip.mate.agent.context; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-063r §2.1: ChatOrigin value-object invariants. + */ +class ChatOriginTest { + + @Test + void from_nullToolContext_returnsEmpty() { + assertSame(ChatOrigin.EMPTY, ChatOrigin.from(null)); + } + + @Test + void from_toolContextWithoutOrigin_returnsEmpty() { + ToolContext ctx = new ToolContext(Map.of("unrelated.key", "x")); + assertSame(ChatOrigin.EMPTY, ChatOrigin.from(ctx)); + } + + @Test + void roundTripThroughToolContext_preservesAllFields() { + ChannelTarget target = new ChannelTarget("user-42", "thread-abc", "bot-001"); + ChatOrigin original = new ChatOrigin(7L, "wechat:42", "u123", 5L, + "/data/ws/5", 9L, target); + + ToolContext ctx = original.toToolContext(); + ChatOrigin restored = ChatOrigin.from(ctx); + + assertEquals(original, restored); + } + + @Test + void wither_doesNotMutateOriginal() { + ChatOrigin base = ChatOrigin.cron("cron_1", 5L, "/data/ws/5", 9L, + new ChannelTarget("group-a", null, null)); + ChatOrigin enriched = base.withAgent(42L); + + assertNull(base.agentId(), "withAgent must not mutate the original"); + assertEquals(42L, enriched.agentId()); + assertEquals(base.channelId(), enriched.channelId(), "channelId must be preserved"); + assertEquals(base.channelTarget(), enriched.channelTarget(), + "channelTarget must be preserved"); + } + + @Test + void cronFactory_setsRequesterToSystem() { + ChatOrigin origin = ChatOrigin.cron("cron_7", 1L, null, 3L, null); + assertEquals("system", origin.requesterId()); + assertNull(origin.agentId(), "agentId is enriched later by BaseAgent"); + } + + @Test + void jsonSerialization_isStableAndForwardCompatible() throws Exception { + ObjectMapper om = new ObjectMapper(); + ChatOrigin origin = new ChatOrigin(7L, "wechat:42", "u123", 5L, + "/data/ws/5", 9L, new ChannelTarget("user-42", "thread-abc", "bot-001")); + + String json = om.writeValueAsString(origin); + ChatOrigin restored = om.readValue(json, ChatOrigin.class); + + assertEquals(origin, restored); + + // RFC-063r §2.1 forward compatibility: future-added unknown fields + // must not break deserialization (covers approval rows surviving upgrades). + String jsonWithExtraField = json.replaceFirst("\\}$", ",\"futureField\":\"x\"}"); + ChatOrigin tolerated = om.readValue(jsonWithExtraField, ChatOrigin.class); + assertEquals(origin, tolerated); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerAnchorTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerAnchorTest.java new file mode 100644 index 00000000..8a771137 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerAnchorTest.java @@ -0,0 +1,183 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.config.ConversationWindowProperties; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * First-user anchor injection — the artifact re-introduced into the + * compacted prompt so the model never loses sight of what the user + * originally asked, even after the actual first turn has been compressed + * into a structured summary. + * + *

Invariants verified here: + *

    + *
  • Anchors are always {@link UserMessage}s, never SystemMessages + * (preventing privilege escalation of historical user input).
  • + *
  • The anchor reflects the FIRST real user message — prior + * summaries and prior anchors are skipped, otherwise iterative + * compaction would anchor compressor output.
  • + *
  • Body sizing degrades gracefully: verbatim ≤ budget, head+tail + * within 3× budget, pointer line above 3×.
  • + *
+ */ +class ConversationWindowManagerAnchorTest { + + @Test + void shortFirstUserStaysVerbatim() { + ConversationWindowManager mgr = newManager(true, 400); + + String goal = "find the bug in foo.js"; + Message anchor = mgr.buildFirstUserAnchor(List.of( + new UserMessage(goal), + new AssistantMessage("looking into it") + )); + + assertInstanceOf(UserMessage.class, anchor); + String text = anchor.getText(); + assertTrue(text.startsWith(ConversationWindowManager.ANCHOR_PREFIX)); + assertTrue(text.contains(goal), + "short goals fit the budget verbatim, no truncation marker should appear"); + } + + @Test + void anchorIsAlwaysUserMessageNeverSystem() { + ConversationWindowManager mgr = newManager(true, 400); + + Message anchor = mgr.buildFirstUserAnchor(List.of( + new UserMessage("rewrite this README") + )); + + // Critical safety property: never promote historical user input into a SystemMessage. + assertInstanceOf(UserMessage.class, anchor); + } + + @Test + void disabledAnchorReturnsNull() { + ConversationWindowManager mgr = newManager(false, 400); + + Message anchor = mgr.buildFirstUserAnchor(List.of( + new UserMessage("anything") + )); + + assertNull(anchor); + } + + @Test + void noUserInPrefixReturnsNull() { + ConversationWindowManager mgr = newManager(true, 400); + + // Prefix is all assistant messages — no user goal to anchor. + Message anchor = mgr.buildFirstUserAnchor(List.of( + new AssistantMessage("blah"), + new AssistantMessage("more blah") + )); + + assertNull(anchor); + } + + @Test + void previousSummaryAndPriorAnchorAreSkipped() { + ConversationWindowManager mgr = newManager(true, 400); + + String realGoal = "ship a feature flag for the new pricing page"; + Message anchor = mgr.buildFirstUserAnchor(List.of( + // round-2 prefix: starts with a previous summary, then a prior anchor, + // then the actual original user message. + new UserMessage(ConversationWindowManager.SUMMARY_PREFIX + "earlier summary text"), + new UserMessage(ConversationWindowManager.ANCHOR_PREFIX + "stale anchor from prior round"), + new UserMessage(realGoal), + new AssistantMessage("on it") + )); + + assertNotNull(anchor); + assertTrue(anchor.getText().contains(realGoal), + "anchor must reflect the REAL first user message, not a prior summary or prior anchor"); + } + + @Test + void mediumOverBudgetIsHeadTailTruncated() { + // 80-token budget → roughly 160-char head+tail target. + ConversationWindowManager mgr = newManager(true, 80); + + // ~400 chars — within 3× the budget so head+tail truncation should apply. + String body = "a".repeat(200) + "MIDDLE" + "b".repeat(200); + Message anchor = mgr.buildFirstUserAnchor(List.of(new UserMessage(body))); + + assertNotNull(anchor); + String text = anchor.getText(); + assertTrue(text.contains("...["), + "head+tail truncation marker should be present"); + assertTrue(text.length() < body.length(), + "anchor must be smaller than original (was " + text.length() + " vs " + body.length() + ")"); + // Head and tail of the original body must both be present. + assertTrue(text.startsWith(ConversationWindowManager.ANCHOR_PREFIX)); + // The first run of 'a's should still be there + assertTrue(text.contains("aaaaaaaaaa")); + // And the tail run of 'b's + assertTrue(text.contains("bbbbbbbbbb")); + } + + @Test + void hugeBodyDegradesToPointerLine() { + ConversationWindowManager mgr = newManager(true, 80); + + // > 3× the budget → pointer-only path. + String body = "X".repeat(5000); + Message anchor = mgr.buildFirstUserAnchor(List.of(new UserMessage(body))); + + assertNotNull(anchor); + String text = anchor.getText(); + assertTrue(text.length() < 500, + "pointer line should be far smaller than the body (was " + text.length() + ")"); + assertTrue(text.endsWith("..."), + "pointer line should end with the truncation marker"); + } + + @Test + void blankUserMessageReturnsNull() { + ConversationWindowManager mgr = newManager(true, 400); + + Message anchor = mgr.buildFirstUserAnchor(List.of( + new UserMessage(""), + new AssistantMessage("ack") + )); + + // No real goal text — nothing to anchor. + assertNull(anchor); + } + + @Test + void anchorPrefixIsConsistent() { + ConversationWindowManager mgr = newManager(true, 400); + + Message a = mgr.buildFirstUserAnchor(List.of(new UserMessage("short"))); + Message b = mgr.buildFirstUserAnchor(List.of(new UserMessage("a different short goal"))); + + // Stable marker — downstream code (and the dedup in buildFirstUserAnchor itself) + // depends on this prefix being constant. + assertEquals(ConversationWindowManager.ANCHOR_PREFIX, + a.getText().substring(0, ConversationWindowManager.ANCHOR_PREFIX.length())); + assertEquals(ConversationWindowManager.ANCHOR_PREFIX, + b.getText().substring(0, ConversationWindowManager.ANCHOR_PREFIX.length())); + } + + // ------------------------------------------------------------------ helpers + + private static ConversationWindowManager newManager(boolean enabled, int maxAnchorTokens) { + ConversationWindowProperties props = new ConversationWindowProperties(); + props.setFirstUserAnchorEnabled(enabled); + props.setFirstUserAnchorMaxTokens(maxAnchorTokens); + return new ConversationWindowManager(props, null, null); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerPairSafeBoundaryTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerPairSafeBoundaryTest.java new file mode 100644 index 00000000..65e3f127 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerPairSafeBoundaryTest.java @@ -0,0 +1,237 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.config.ConversationWindowProperties; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pair-safe boundary enforcement for {@link ConversationWindowManager}. + * + *

The compactor must never produce a prompt where an + * {@link AssistantMessage} carrying {@code tool_calls} is separated from + * the {@link ToolResponseMessage}s that close those calls. Provider APIs + * 400 on the broken sequence, which is strictly worse than letting the + * context cross the budget by one extra turn. + * + *

Conventions used by these tests: + *

    + *
  • {@code asst(id1, id2, ...)} — assistant message carrying tool_calls
  • + *
  • {@code resp(id, ...)} — tool response message closing the listed ids
  • + *
  • "split" means the candidate cut falls between an assistant and one + * of its responses; the algorithm must move the cut backward until no + * split remains, or signal skip-compaction by returning {@code headEnd}.
  • + *
+ */ +class ConversationWindowManagerPairSafeBoundaryTest { + + @Test + void cleanCutBetweenTurnsIsUnchanged() { + ConversationWindowManager mgr = newManager(0); + + // [0] user, [1] assistant(call-1), [2] response(call-1), + // [3] user, [4] assistant(call-2), [5] response(call-2) + List messages = List.of( + new UserMessage("q1"), + asst("call-1"), + resp("call-1"), + new UserMessage("q2"), + asst("call-2"), + resp("call-2") + ); + + // tailStart=3 — cuts cleanly between two fully-closed turns. + int cut = mgr.enforcePairSafeBoundary(messages, 0, 3); + + assertEquals(3, cut, "cut between completed turns must not move"); + } + + @Test + void cutLandingOnResponseMovesBackToOwningAssistant() { + ConversationWindowManager mgr = newManager(0); + + // [0] user, [1] assistant(call-1), [2] response(call-1), [3] user, [4] assistant(call-2), [5] response(call-2) + List messages = List.of( + new UserMessage("q1"), + asst("call-1"), + resp("call-1"), + new UserMessage("q2"), + asst("call-2"), + resp("call-2") + ); + + // tailStart=2 — splits call-1 (assistant in prefix, response in tail). + int cut = mgr.enforcePairSafeBoundary(messages, 0, 2); + + assertEquals(1, cut, + "cut must move to the assistant that issued call-1 so the pair lands in the tail together"); + } + + @Test + void cutSplittingAssistantWithMultipleToolCallsMovesEntireGroup() { + ConversationWindowManager mgr = newManager(0); + + // One assistant with TWO tool_calls; responses arrive in two separate + // ToolResponseMessages. Cutting between the responses must drag the + // assistant + both response messages into the tail together. + List messages = List.of( + new UserMessage("q"), + asst("call-1", "call-2"), + resp("call-1"), + resp("call-2"), + new UserMessage("next") + ); + + int cut = mgr.enforcePairSafeBoundary(messages, 0, 3); // between the two responses + + assertEquals(1, cut, + "splitting a multi-call assistant must move cut to the assistant index"); + } + + @Test + void cutSplittingMultiResponseMessagesForOneAssistantMovesBack() { + ConversationWindowManager mgr = newManager(0); + + // assistant(call-1, call-2), single ToolResponseMessage closing both. + List messages = List.of( + new UserMessage("q"), + asst("call-1", "call-2"), + ToolResponseMessage.builder().responses(List.of( + new ToolResponseMessage.ToolResponse("call-1", "tool_a", "x"), + new ToolResponseMessage.ToolResponse("call-2", "tool_b", "y") + )).build(), + new UserMessage("next") + ); + + // cut=2 → response message is in tail, assistant in prefix → split. + int cut = mgr.enforcePairSafeBoundary(messages, 0, 2); + + assertEquals(1, cut); + } + + @Test + void chainedPairSplitsConvergeAfterMultiplePasses() { + ConversationWindowManager mgr = newManager(0); + + // Three consecutive call/response cycles. Cutting in the middle + // exposes a split, and moving the cut back exposes another. + List messages = List.of( + asst("call-1"), // 0 + resp("call-1"), // 1 + asst("call-2"), // 2 + resp("call-2"), // 3 + asst("call-3"), // 4 + resp("call-3") // 5 + ); + + // cut=3 splits call-2 (assistant at 2, response at 3) → first pass moves to 2. + // After moving to 2, no more splits (call-1 is fully in prefix, call-3 fully in tail). + int cut = mgr.enforcePairSafeBoundary(messages, 0, 3); + assertEquals(2, cut); + + // cut=5 splits call-3 → moves to 4. cut=4, still good (no split). Convergence. + cut = mgr.enforcePairSafeBoundary(messages, 0, 5); + assertEquals(4, cut); + } + + @Test + void collapseToHeadEndSignalsSkip() { + ConversationWindowManager mgr = newManager(0); + + // Single assistant + response pair. Cutting anywhere splits it, + // so the safe boundary lands at headEnd → caller should skip compaction. + List messages = List.of( + asst("call-1"), + resp("call-1") + ); + + int cut = mgr.enforcePairSafeBoundary(messages, 0, 1); + + assertEquals(0, cut, "single unsafe pair must collapse to headEnd to signal skip"); + } + + @Test + void minPrefixThresholdSkipsTinyCompactions() { + // minPrefix=3 — after pair safety, if prefix < 3 messages, skip. + ConversationWindowManager mgr = newManager(3); + + List messages = List.of( + new UserMessage("q1"), + asst("call-1"), + resp("call-1"), + new UserMessage("q2") + ); + + // cut=3 would compress messages[0..3] = 3 items, meeting min. + // cut=1 would compress just messages[0..1] = 1 item, below min → skip. + int cut1 = mgr.enforcePairSafeBoundary(messages, 0, 3); + assertEquals(3, cut1, "3-message prefix meets the minimum"); + + int cut2 = mgr.enforcePairSafeBoundary(messages, 0, 1); + assertEquals(0, cut2, "1-message prefix is below the configured minimum → skip compaction"); + } + + @Test + void orphanResponseInTailDoesNotMoveBoundary() { + ConversationWindowManager mgr = newManager(0); + + // call-orphan has no preceding assistant — pure data anomaly. Algorithm + // should not try to "fix" it by moving the cut; it just leaves the + // boundary where it was and logs a warn. + List messages = List.of( + new UserMessage("q1"), + asst("call-1"), + resp("call-1"), + new UserMessage("q2"), + resp("call-orphan") + ); + + int cut = mgr.enforcePairSafeBoundary(messages, 0, 3); + + assertEquals(3, cut, "orphan response must not pull the boundary"); + } + + @Test + void tailStartAtOrBeyondMessagesSizeIsUnchanged() { + ConversationWindowManager mgr = newManager(0); + + List messages = List.of( + new UserMessage("a"), + new UserMessage("b") + ); + + assertEquals(2, mgr.enforcePairSafeBoundary(messages, 0, 2), + "boundary at end of list passes through"); + assertTrue(mgr.enforcePairSafeBoundary(messages, 0, 5) >= 0, + "out-of-range boundary stays sane"); + } + + // ------------------------------------------------------------------ helpers + + private static ConversationWindowManager newManager(int minPrefix) { + ConversationWindowProperties props = new ConversationWindowProperties(); + props.setPairSafeMinPrefixToCompact(minPrefix); + return new ConversationWindowManager(props, null, null); + } + + private static AssistantMessage asst(String... callIds) { + java.util.List calls = new java.util.ArrayList<>(); + for (String id : callIds) { + calls.add(new AssistantMessage.ToolCall(id, "function", "tool_" + id, "{}")); + } + return AssistantMessage.builder().content("").toolCalls(calls).build(); + } + + private static ToolResponseMessage resp(String callId) { + return ToolResponseMessage.builder().responses(List.of( + new ToolResponseMessage.ToolResponse(callId, "tool_" + callId, "ok") + )).build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSpillMarkerPreservationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSpillMarkerPreservationTest.java new file mode 100644 index 00000000..b6112f82 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSpillMarkerPreservationTest.java @@ -0,0 +1,157 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import vip.mate.agent.graph.executor.ToolResultStorage; +import vip.mate.config.ConversationWindowProperties; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The three compaction phases (soft trim, hard clear, pre-prune for + * summary) must never destroy a spill-marker body — doing so would erase + * the {@code path=...} pointer the model needs to recover the original + * full output via {@code read_file}, which is the whole reason that body + * was spilled in the first place. + * + *

This is the "recoverable" invariant: once a tool output makes it + * into the spill store, the in-context representation stays a stable + * preview + path for the rest of the conversation regardless of how + * aggressively the window manager has to compress the prefix. + */ +class ConversationWindowManagerSpillMarkerPreservationTest { + + private static final String SPILL_BODY = ToolResultStorage.SPILL_MARKER_PREFIX + + " tool=web_search full_chars=22000 path=/tmp/x.txt\n" + + "[Preview — first 800 of 22000 chars. Use read_file with the path above to retrieve the rest.]\n" + + "preview body fragment that contributes most of the inline size..."; + + @Test + void softTrimLeavesSpillMarkerUntouched() { + ConversationWindowManager mgr = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + + List messages = new ArrayList<>(List.of( + toolMessage("call-spill", "web_search", SPILL_BODY), + toolMessage("call-big", "search", "x".repeat(2000)) + )); + + // Pass the same list through Phase 1. + int trimmed = mgr.softTrimToolResults(messages); + + // The non-spill body must have been trimmed (it was > 500 chars). + // The spill body must remain identical to the original. + ToolResponseMessage trm0 = (ToolResponseMessage) messages.get(0); + ToolResponseMessage trm1 = (ToolResponseMessage) messages.get(1); + assertEquals(SPILL_BODY, trm0.getResponses().getFirst().responseData(), + "Phase 1 soft trim must not modify a spill-marker body"); + assertTrue(trm1.getResponses().getFirst().responseData().contains("[trimmed "), + "non-spill bodies should still be trimmed by Phase 1"); + assertEquals(1, trimmed, + "trim counter should reflect only the non-spill body that was actually shortened"); + } + + @Test + void hardClearLeavesSpillMarkerUntouched() { + ConversationWindowManager mgr = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + + List messages = new ArrayList<>(List.of( + toolMessage("call-spill", "web_search", SPILL_BODY), + toolMessage("call-big", "search", "y".repeat(2000)) + )); + + int cleared = mgr.hardClearToolResults(messages); + + ToolResponseMessage trm0 = (ToolResponseMessage) messages.get(0); + ToolResponseMessage trm1 = (ToolResponseMessage) messages.get(1); + assertEquals(SPILL_BODY, trm0.getResponses().getFirst().responseData(), + "Phase 2 hard clear must not replace a spill-marker body with [tool result removed]"); + assertEquals("[tool result removed]", trm1.getResponses().getFirst().responseData(), + "non-spill bodies should still be replaced by Phase 2"); + assertEquals(1, cleared, + "clear counter should reflect only the non-spill body that was actually replaced"); + } + + @Test + void prePruneForSummaryLeavesSpillMarkerUntouched() { + ConversationWindowManager mgr = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + + List messages = new ArrayList<>(List.of( + toolMessage("call-spill", "web_search", SPILL_BODY), + toolMessage("call-big", "search", "z".repeat(2000)) + )); + + int pruned = mgr.prePruneForSummary(messages); + + ToolResponseMessage trm0 = (ToolResponseMessage) messages.get(0); + ToolResponseMessage trm1 = (ToolResponseMessage) messages.get(1); + assertEquals(SPILL_BODY, trm0.getResponses().getFirst().responseData(), + "Phase 3 pre-prune must not replace a spill-marker body with the cleared-output placeholder"); + assertTrue(trm1.getResponses().getFirst().responseData().contains("旧工具输出已清理"), + "non-spill bodies should still be replaced by Phase 3"); + assertEquals(1, pruned); + } + + @Test + void mixedMessageWithSpillAndNonSpillResponsesPreservesOnlyTheMarker() { + // A single ToolResponseMessage can hold multiple ToolResponses (one + // assistant tool_calls turn could ask for several tools at once). + // The phase guards must operate at the response level, not the + // message level — the spill response stays, the non-spill response + // gets the placeholder. + ConversationWindowManager mgr = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + + ToolResponseMessage mixed = ToolResponseMessage.builder().responses(List.of( + new ToolResponseMessage.ToolResponse("call-spill", "web_search", SPILL_BODY), + new ToolResponseMessage.ToolResponse("call-big", "search", "q".repeat(2000)) + )).build(); + List messages = new ArrayList<>(List.of(mixed)); + + mgr.hardClearToolResults(messages); + + ToolResponseMessage trm = (ToolResponseMessage) messages.getFirst(); + assertEquals(SPILL_BODY, trm.getResponses().get(0).responseData(), + "the spill response in a mixed message must survive Phase 2"); + assertEquals("[tool result removed]", trm.getResponses().get(1).responseData(), + "the non-spill response in a mixed message must still be cleared"); + } + + @Test + void smallSpillMarkerStillStaysVerbatim() { + // Edge case: even when the preview is short (under the 500-char + // soft-trim threshold), the marker check should still apply. This + // protects against future changes to the trim threshold. + ConversationWindowManager mgr = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + + String tinySpill = ToolResultStorage.SPILL_MARKER_PREFIX + + " tool=test full_chars=600 path=/tmp/t.txt\n[tiny]"; + List messages = new ArrayList<>(List.of( + toolMessage("call-1", "test", tinySpill) + )); + + mgr.softTrimToolResults(messages); + mgr.hardClearToolResults(messages); + mgr.prePruneForSummary(messages); + + ToolResponseMessage trm = (ToolResponseMessage) messages.getFirst(); + assertEquals(tinySpill, trm.getResponses().getFirst().responseData(), + "the marker check is what protects the body — not the size of the preview"); + } + + // ------------------------------------------------------------------ helpers + + private static ToolResponseMessage toolMessage(String id, String name, String data) { + return ToolResponseMessage.builder() + .responses(List.of(new ToolResponseMessage.ToolResponse(id, name, data))) + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSummaryBudgetTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSummaryBudgetTest.java new file mode 100644 index 00000000..5c25415d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerSummaryBudgetTest.java @@ -0,0 +1,109 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import vip.mate.config.ConversationWindowProperties; +import vip.mate.memory.spi.MemoryManager; +import vip.mate.workspace.conversation.ConversationService; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Regression test for the RFC: prompt-cleanup D bug — the iterative-update + * branch of {@link ConversationWindowManager#generateSummary} previously + * used the raw {@code STRUCTURED_SUMMARY_SYSTEM} template without + * substituting {@code {summary_budget}}, leaking the literal placeholder + * into the LLM prompt. + * + *

Both branches must now produce a SystemMessage where {@code {summary_budget}} + * is replaced by the configured budget number.

+ */ +class ConversationWindowManagerSummaryBudgetTest { + + private ConversationWindowManager manager; + private ChatModel chatModel; + + @BeforeEach + void setUp() { + ConversationWindowProperties props = new ConversationWindowProperties(); + MemoryManager memory = mock(MemoryManager.class); + ConversationService conv = mock(ConversationService.class); + manager = new ConversationWindowManager(props, memory, conv); + + chatModel = mock(ChatModel.class); + // Return a non-null, non-empty response so generateSummary stores the result. + Generation gen = new Generation(new org.springframework.ai.chat.messages.AssistantMessage("STUB SUMMARY"), + ChatGenerationMetadata.NULL); + ChatResponse response = new ChatResponse(List.of(gen)); + when(chatModel.call(any(Prompt.class))).thenReturn(response); + } + + @Test + @DisplayName("First-compression branch: {summary_budget} is substituted in SystemMessage") + void firstCompressionReplacesBudget() throws Exception { + Prompt sentPrompt = invokeGenerateSummaryAndCapture("conv-first", null); + SystemMessage system = (SystemMessage) sentPrompt.getInstructions().stream() + .filter(m -> m instanceof SystemMessage).findFirst().orElseThrow(); + String text = system.getText(); + assertFalse(text.contains("{summary_budget}"), + "first-compression: literal placeholder must not leak into the SystemMessage"); + assertTrue(text.matches("(?s).*\\d{2,}.*"), + "first-compression: SystemMessage should contain a numeric budget after substitution"); + } + + @Test + @DisplayName("Iterative-update branch: {summary_budget} is substituted in SystemMessage") + void iterativeUpdateReplacesBudget() throws Exception { + // Seed previousSummaries so generateSummary takes the iterative-update path. + Field f = ConversationWindowManager.class.getDeclaredField("previousSummaries"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + ConcurrentHashMap prev = (ConcurrentHashMap) f.get(manager); + prev.put("conv-iter", "PRIOR SUMMARY (placeholder for the iterative-update branch test)"); + + Prompt sentPrompt = invokeGenerateSummaryAndCapture("conv-iter", null); + SystemMessage system = (SystemMessage) sentPrompt.getInstructions().stream() + .filter(m -> m instanceof SystemMessage).findFirst().orElseThrow(); + String text = system.getText(); + assertFalse(text.contains("{summary_budget}"), + "iterative-update: literal placeholder must not leak into the SystemMessage (the bug regression guard)"); + } + + /** + * Reflectively invoke the private {@code generateSummary} method and capture + * the {@link Prompt} sent to the mocked {@link ChatModel}. + */ + private Prompt invokeGenerateSummaryAndCapture(String conversationId, String memoryExtra) throws Exception { + // Two synthetic user messages so serializeForSummary produces non-empty content. + List oldMessages = List.of( + new UserMessage("hello"), + new UserMessage("world")); + + Method m = ConversationWindowManager.class.getDeclaredMethod( + "generateSummary", List.class, ChatModel.class, String.class, int.class, String.class); + m.setAccessible(true); + m.invoke(manager, oldMessages, chatModel, conversationId, 1500, memoryExtra); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Prompt.class); + org.mockito.Mockito.verify(chatModel).call(captor.capture()); + return captor.getValue(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerToolPruningTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerToolPruningTest.java new file mode 100644 index 00000000..62573812 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/ConversationWindowManagerToolPruningTest.java @@ -0,0 +1,235 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.messages.UserMessage; +import vip.mate.agent.graph.executor.ToolResultProperties; +import vip.mate.agent.graph.executor.ToolResultStorage; +import vip.mate.config.ConversationWindowProperties; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Behavior of {@link ConversationWindowManager#pruneOldToolResultsForModelInput} — + * the pre-pass that runs before every model request to keep old tool results + * from inflating the prompt. + * + *

The current contract: + *

    + *
  • The latest tool response is kept verbatim.
  • + *
  • Older bodies under the dedup threshold are kept verbatim.
  • + *
  • Older bodies that are byte-identical to a newer body are replaced + * with a short "duplicate omitted" placeholder.
  • + *
  • Older bodies above {@link ToolResultStorage}'s spill threshold are + * written to disk; the in-prompt body becomes a preview + path so the + * model can call {@code read_file} for the full content.
  • + *
  • Without storage wired, old bodies stay verbatim. The previous + * behaviour — rewriting them into a lossy single-line summary — + * destroyed too much context on long tasks and was removed.
  • + *
+ */ +class ConversationWindowManagerToolPruningTest { + + @Test + void withoutStorageOlderLargeBodiesStayVerbatim() { + ConversationWindowManager manager = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + String oldLarge = "old-result\n".repeat(700); // ~7700 chars + String latestLarge = "latest-result\n".repeat(700); + List messages = List.of( + new UserMessage("read earlier file"), + toolMessage("old-1", "read_file", oldLarge), + new UserMessage("read latest file"), + toolMessage("new-1", "read_file", latestLarge) + ); + + List pruned = manager.pruneOldToolResultsForModelInput(messages); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(1); + ToolResponseMessage latestToolMessage = (ToolResponseMessage) pruned.get(3); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + String latestData = latestToolMessage.getResponses().getFirst().responseData(); + + // No storage → keep the old body untouched, do NOT collapse to a lossy summary. + assertEquals(oldLarge, oldData, + "without storage, older tool bodies must be preserved verbatim " + + "(the lossy single-line rewrite has been removed)"); + assertEquals(latestLarge, latestData); + } + + @Test + void olderDuplicateToolResultStillUsesDuplicatePlaceholder(@TempDir Path tempDir) { + ConversationWindowManager manager = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + String repeated = "same-output\n".repeat(700); + List messages = List.of( + toolMessage("old-1", "read_file", repeated), + toolMessage("new-1", "read_file", repeated) + ); + + List pruned = manager.pruneOldToolResultsForModelInput(messages); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.getFirst(); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + assertTrue(oldData.contains("duplicate tool output omitted"), + "byte-identical duplicates older than the latest copy still get the dedup placeholder"); + } + + @Test + void withStorageOlderLargeBodiesGetSpilledToDisk(@TempDir Path tempDir) throws Exception { + ConversationWindowManager manager = newManagerWithStorage(tempDir, /*threshold*/ 2000); + + String oldLarge = "alpha\n".repeat(800); // 4800 chars > threshold 2000 + String latestLarge = "beta\n".repeat(800); + List messages = List.of( + new UserMessage("turn 1"), + toolMessage("old-1", "web_search", oldLarge), + new UserMessage("turn 2"), + toolMessage("new-1", "web_search", latestLarge) + ); + + List pruned = manager.pruneOldToolResultsForModelInput( + messages, "conv-A", tempDir.toString()); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(1); + ToolResponseMessage latestToolMessage = (ToolResponseMessage) pruned.get(3); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + String latestData = latestToolMessage.getResponses().getFirst().responseData(); + + assertTrue(oldData.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX), + "older oversized body should be spilled and replaced with a SPILL_MARKER preview"); + assertNotEquals(oldLarge, oldData, "old data should be replaced"); + // Latest one is always kept full regardless of size. + assertEquals(latestLarge, latestData); + + // Verify the spill file contains the FULL raw body, not a truncated version. + Matcher m = Pattern.compile("path=(\\S+)").matcher(oldData); + assertTrue(m.find(), "preview must report the spill path"); + Path spillFile = Path.of(m.group(1)); + assertTrue(Files.exists(spillFile)); + assertEquals(oldLarge, Files.readString(spillFile), + "spill file must hold the full original body — the whole point of preserving " + + "raw output for read_file recovery"); + } + + @Test + void withStorageOlderSmallBodiesStayVerbatim(@TempDir Path tempDir) { + ConversationWindowManager manager = newManagerWithStorage(tempDir, /*threshold*/ 2000); + + String oldSmall = "small old body"; // far under threshold + String latestLarge = "x".repeat(3000); + List messages = List.of( + toolMessage("old-1", "web_search", oldSmall), + new UserMessage("turn"), + toolMessage("new-1", "web_search", latestLarge) + ); + + List pruned = manager.pruneOldToolResultsForModelInput( + messages, "conv-B", tempDir.toString()); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(0); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + assertEquals(oldSmall, oldData, + "bodies under the spill threshold stay verbatim — small results carry no compression win"); + } + + @Test + void alreadySpilledMarkerIsNotReSpilled(@TempDir Path tempDir) { + ConversationWindowManager manager = newManagerWithStorage(tempDir, /*threshold*/ 1000); + + // Simulate a body that was spilled at tool-execution time: it already + // starts with the spill marker. Prune must leave it alone instead of + // trying to spill a spill preview (which would write the preview text + // to a new file, ad infinitum). + String alreadySpilled = ToolResultStorage.SPILL_MARKER_PREFIX + + " tool=web_search full_chars=22000 path=/tmp/x.txt\n[Preview ...]\nbody preview ..."; + String latestLarge = "y".repeat(3000); + List messages = List.of( + toolMessage("old-1", "web_search", alreadySpilled), + toolMessage("new-1", "web_search", latestLarge) + ); + + List pruned = manager.pruneOldToolResultsForModelInput( + messages, "conv-C", tempDir.toString()); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(0); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + assertEquals(alreadySpilled, oldData, + "previously-spilled previews must pass through untouched — no double-spill"); + } + + @Test + void exemptToolStaysVerbatimEvenWhenOversized(@TempDir Path tempDir) { + ConversationWindowManager manager = newManagerWithStorage(tempDir, /*threshold*/ 1000); + + String oldLarge = "z".repeat(5000); + String latestLarge = "z".repeat(5000); + List messages = List.of( + toolMessage("old-1", "delegateToAgent", oldLarge), // exempt tool + toolMessage("new-1", "delegateToAgent", latestLarge) + ); + + List pruned = manager.pruneOldToolResultsForModelInput( + messages, "conv-D", tempDir.toString()); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(0); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + assertEquals(oldLarge, oldData, + "sub-agent delegation results are irreplaceable — must never be rewritten"); + assertFalse(oldData.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX), + "exempt tools should also not be spilled (they're already cheap to keep)"); + } + + @Test + void blankConversationIdDisablesSpill(@TempDir Path tempDir) { + ConversationWindowManager manager = newManagerWithStorage(tempDir, /*threshold*/ 1000); + + String oldLarge = "q".repeat(5000); + String latestLarge = "r".repeat(5000); + List messages = List.of( + toolMessage("old-1", "web_search", oldLarge), + toolMessage("new-1", "web_search", latestLarge) + ); + + // Without a conversationId, spill cannot scope files safely → falls back to verbatim. + List pruned = manager.pruneOldToolResultsForModelInput( + messages, null, tempDir.toString()); + + ToolResponseMessage oldToolMessage = (ToolResponseMessage) pruned.get(0); + String oldData = oldToolMessage.getResponses().getFirst().responseData(); + assertEquals(oldLarge, oldData, + "null conversationId must not trigger spill — caller cannot scope files correctly"); + } + + // ------------------------------------------------------------------ helpers + + private static ConversationWindowManager newManagerWithStorage(Path tempDir, int threshold) { + ToolResultProperties props = new ToolResultProperties(); + props.setStorageBaseDir(tempDir.toString()); + props.setPerResultThresholdChars(threshold); + props.setPreviewHeadChars(120); + ToolResultStorage storage = new ToolResultStorage(props); + + ConversationWindowManager manager = new ConversationWindowManager( + new ConversationWindowProperties(), null, null); + manager.setToolResultStorage(storage); + return manager; + } + + private static ToolResponseMessage toolMessage(String id, String name, String data) { + return ToolResponseMessage.builder() + .responses(List.of(new ToolResponseMessage.ToolResponse(id, name, data))) + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/context/TokenEstimatorToolsTest.java b/mateclaw-server/src/test/java/vip/mate/agent/context/TokenEstimatorToolsTest.java new file mode 100644 index 00000000..29e25994 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/context/TokenEstimatorToolsTest.java @@ -0,0 +1,89 @@ +package vip.mate.agent.context; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class TokenEstimatorToolsTest { + + private ToolCallback callback(String name, String description, String inputSchema) { + ToolCallback cb = mock(ToolCallback.class); + ToolDefinition def = mock(ToolDefinition.class); + when(def.name()).thenReturn(name); + when(def.description()).thenReturn(description); + when(def.inputSchema()).thenReturn(inputSchema); + when(cb.getToolDefinition()).thenReturn(def); + return cb; + } + + @Test + @DisplayName("null / empty collection returns 0") + void emptyZero() { + assertEquals(0, TokenEstimator.estimateToolsTokens(null)); + assertEquals(0, TokenEstimator.estimateToolsTokens(List.of())); + } + + @Test + @DisplayName("single tool: name + description + schema + per-tool overhead all included") + void singleTool() { + ToolCallback cb = callback("web_search", + "Search the web for recent information", + "{\"type\":\"object\",\"properties\":{\"query\":{\"type\":\"string\"}}}"); + int tokens = TokenEstimator.estimateToolsTokens(List.of(cb)); + // > the per-tool overhead alone (proves description + schema were summed in) + assertTrue(tokens > TokenEstimator.PER_TOOL_OVERHEAD, + "Should include description and schema, got " + tokens); + // sanity bound: this small tool shouldn't blow past 100 tokens + assertTrue(tokens < 100, "Bound check, got " + tokens); + } + + @Test + @DisplayName("many tools accumulate — N tools cost ~N x single-tool cost") + void manyToolsAccumulate() { + ToolCallback cb = callback("read_file", + "Read a file from the workspace", + "{\"type\":\"object\",\"properties\":{\"path\":{\"type\":\"string\"}}}"); + int one = TokenEstimator.estimateToolsTokens(List.of(cb)); + int five = TokenEstimator.estimateToolsTokens(List.of(cb, cb, cb, cb, cb)); + assertEquals(one * 5, five, "Five identical tools should cost five times one"); + } + + @Test + @DisplayName("MCP-sized tool with verbose schema costs hundreds of tokens — proves the gap is real") + void mcpSizedTool() { + // Realistic MCP tool: long description + nested schema with many properties + String bigDescription = "Execute a SQL query against the connected PostgreSQL database. " + + "Returns rows as a JSON array. Supports SELECT, INSERT, UPDATE, DELETE statements. " + + "Bound parameters must be passed as a separate array; do not concatenate user input."; + String bigSchema = "{\"type\":\"object\",\"properties\":{" + + "\"sql\":{\"type\":\"string\",\"description\":\"The SQL statement to execute\"}," + + "\"params\":{\"type\":\"array\",\"items\":{\"type\":\"string\"},\"description\":\"Bound parameters\"}," + + "\"timeout_ms\":{\"type\":\"integer\",\"description\":\"Statement timeout in ms\",\"minimum\":0,\"maximum\":60000}," + + "\"read_only\":{\"type\":\"boolean\",\"description\":\"Reject statements that modify data\"}" + + "},\"required\":[\"sql\"]}"; + ToolCallback cb = callback("postgres_query", bigDescription, bigSchema); + + int tokens = TokenEstimator.estimateToolsTokens(List.of(cb)); + assertTrue(tokens > 100, + "A real MCP tool's schema cost should clearly exceed 100 tokens, got " + tokens); + } + + @Test + @DisplayName("callbacks that throw on getToolDefinition() are skipped, not propagated") + void brokenCallbackSwallowed() { + ToolCallback bad = mock(ToolCallback.class); + when(bad.getToolDefinition()).thenThrow(new RuntimeException("provider error")); + ToolCallback good = callback("ok", "ok", "{}"); + + int tokens = TokenEstimator.estimateToolsTokens(List.of(bad, good)); + // good tool still contributes; bad one contributes 0 + assertTrue(tokens > 0, + "Broken callback should be skipped, good one should still count, got " + tokens); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentControllerTest.java b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentControllerTest.java new file mode 100644 index 00000000..8e894326 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentControllerTest.java @@ -0,0 +1,186 @@ +package vip.mate.agent.delegation; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.security.core.Authentication; +import vip.mate.audit.service.AuditEventService; +import vip.mate.common.result.R; +import vip.mate.exception.MateClawException; +import vip.mate.workspace.conversation.ConversationService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SubagentControllerTest { + + private SubagentRegistry registry; + private ConversationService conversationService; + private AuditEventService auditEventService; + private SubagentController controller; + private Authentication ownerAuth; + private Authentication outsiderAuth; + + @BeforeEach + void setUp() { + registry = new SubagentRegistry(); + conversationService = mock(ConversationService.class); + auditEventService = mock(AuditEventService.class); + ObjectMapper mapper = new ObjectMapper(); + controller = new SubagentController(registry, conversationService, auditEventService, mapper); + + ownerAuth = mock(Authentication.class); + when(ownerAuth.getName()).thenReturn("alice"); + outsiderAuth = mock(Authentication.class); + when(outsiderAuth.getName()).thenReturn("mallory"); + + // Default: alice owns parent-1, mallory does not. + when(conversationService.isConversationOwner(eq("parent-1"), eq("alice"))).thenReturn(true); + when(conversationService.isConversationOwner(eq("parent-1"), eq("mallory"))).thenReturn(false); + } + + @Test + @DisplayName("interrupt — owner gets 200 with interrupted=true and an audit row") + void interruptOwner() { + String sid = registry.register("parent-1", "child-1", 7L, "do thing", null); + + R> response = controller.interrupt(sid, ownerAuth); + + assertThat(response.getCode()).isEqualTo(200); // ResultCode.SUCCESS + assertThat(response.getData()).containsEntry("interrupted", true); + assertThat(registry.get(sid).orElseThrow().status().get()).isEqualTo("interrupted"); + verify(auditEventService).record(eq("subagent.interrupt"), eq("subagent"), + eq(sid), anyString(), anyString()); + // Denial audit must NOT have fired on the owner path. + verify(auditEventService, never()).record(eq("subagent.interrupt.denied"), + anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("interrupt — non-owner is denied (403) and a denial audit is written") + void interruptDeniedForNonOwner() { + String sid = registry.register("parent-1", "child-1", 7L, "do thing", null); + + assertThatThrownBy(() -> controller.interrupt(sid, outsiderAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 403); + + verify(auditEventService).record(eq("subagent.interrupt.denied"), eq("subagent"), + eq(sid), anyString(), anyString()); + // Status must remain unchanged for the non-owner path. + assertThat(registry.get(sid).orElseThrow().status().get()).isEqualTo("running"); + } + + @Test + @DisplayName("interrupt — missing subagent throws 404") + void interruptNotFound() { + assertThatThrownBy(() -> controller.interrupt("sa-does-not-exist", ownerAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 404); + + verify(auditEventService, never()).record(eq("subagent.interrupt"), + anyString(), anyString(), anyString(), anyString()); + } + + @Test + @DisplayName("spawn-pause — missing parentConversationId throws 400") + void spawnPauseMissingParent() { + Map body = new HashMap<>(); + body.put("paused", true); + + assertThatThrownBy(() -> controller.setPaused(body, ownerAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 400); + + // Empty body also fails the same way. + assertThatThrownBy(() -> controller.setPaused(new HashMap<>(), ownerAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 400); + } + + @Test + @DisplayName("spawn-pause — owner toggles flag and audit captures decision") + void spawnPauseOwnerToggle() { + Map body = new HashMap<>(); + body.put("parentConversationId", "parent-1"); + body.put("paused", true); + + R> resp = controller.setPaused(body, ownerAuth); + assertThat(resp.getData()).containsEntry("paused", true); + assertThat(registry.isSpawnPaused("parent-1")).isTrue(); + verify(auditEventService).record(eq("subagent.spawn-pause"), eq("conversation"), + eq("parent-1"), eq("parent-1"), anyString()); + + body.put("paused", false); + controller.setPaused(body, ownerAuth); + assertThat(registry.isSpawnPaused("parent-1")).isFalse(); + } + + @Test + @DisplayName("spawn-pause — non-owner gets 403, flag is not changed") + void spawnPauseNonOwnerForbidden() { + Map body = new HashMap<>(); + body.put("parentConversationId", "parent-1"); + body.put("paused", true); + + assertThatThrownBy(() -> controller.setPaused(body, outsiderAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 403); + + assertThat(registry.isSpawnPaused("parent-1")).isFalse(); + } + + @Test + @DisplayName("listActive — missing parentConversationId throws 400") + void listActiveMissingParent() { + assertThatThrownBy(() -> controller.listActive(null, ownerAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 400); + assertThatThrownBy(() -> controller.listActive("", ownerAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 400); + } + + @Test + @DisplayName("listActive — owner sees only their own subagents in the response") + void listActiveOwnerScoped() { + registry.register("parent-1", "child-1", 7L, "g", null); + registry.register("parent-1", "child-2", 7L, "g2", null); + registry.register("other-parent", "child-x", 8L, "g3", null); + + R> resp = controller.listActive("parent-1", ownerAuth); + + @SuppressWarnings("unchecked") + List> subagents = (List>) resp.getData().get("subagents"); + assertThat(subagents).hasSize(2); + assertThat(subagents).allSatisfy(dto -> { + assertThat(dto.get("parentConversationId")).isEqualTo("parent-1"); + // Disposable + raw atomic refs must not leak into the wire DTO. + assertThat(dto).doesNotContainKey("disposable"); + }); + } + + @Test + @DisplayName("listActive — non-owner is denied 403") + void listActiveNonOwnerForbidden() { + registry.register("parent-1", "child-1", 7L, "g", null); + + assertThatThrownBy(() -> controller.listActive("parent-1", outsiderAuth)) + .isInstanceOf(MateClawException.class) + .matches(t -> ((MateClawException) t).getCode() == 403); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentHeartbeatTest.java b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentHeartbeatTest.java new file mode 100644 index 00000000..2a7a7c72 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentHeartbeatTest.java @@ -0,0 +1,145 @@ +package vip.mate.agent.delegation; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.channel.web.ChatStreamTracker; + +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SubagentHeartbeatTest { + + private SubagentRegistry registry; + private SubagentHeartbeatConfig cfg; + private ChatStreamTracker streamTracker; + private SubagentHeartbeat heartbeat; + + @BeforeEach + void setUp() { + registry = new SubagentRegistry(); + cfg = new SubagentHeartbeatConfig(); + // Tight thresholds keep tests fast. + cfg.setIntervalSec(30); + cfg.setStaleCyclesIdle(3); + cfg.setStaleCyclesInTool(5); + streamTracker = mock(ChatStreamTracker.class); + heartbeat = new SubagentHeartbeat(registry, cfg, streamTracker); + } + + @Test + @DisplayName("idle child flips to stale exactly at the configured idle threshold") + void idleChildBecomesStale() { + String id = registry.register("parent-1", "child-1", 1L, "g", null); + // No tool, no phase change across cycles → idle path. + when(streamTracker.getRunningToolName("child-1")).thenReturn(null); + when(streamTracker.getCurrentPhase("child-1")).thenReturn("thinking"); + + var rec = registry.get(id).orElseThrow(); + + // Cycle 1: first observation seeds lastSeen, no stale increment. + heartbeat.evaluate(rec); + assertThat(rec.staleCount().get()).isEqualTo(0); + assertThat(rec.status().get()).isEqualTo("running"); + + // Cycles 2 and 3: no change → counter increments to 1, then 2. + heartbeat.evaluate(rec); + heartbeat.evaluate(rec); + assertThat(rec.staleCount().get()).isEqualTo(2); + assertThat(rec.status().get()).isEqualTo("running"); + verify(streamTracker, never()).broadcastObject(anyString(), eq("subagent_stale"), any()); + + // Cycle 4: counter hits 3 → stale and event broadcast. + heartbeat.evaluate(rec); + assertThat(rec.status().get()).isEqualTo("stale"); + verify(streamTracker, times(1)).broadcastObject(eq("parent-1"), eq("subagent_stale"), any()); + } + + @Test + @DisplayName("in-tool child uses the longer in-tool threshold before stale fires") + void inToolChildUsesLongerThreshold() { + String id = registry.register("parent-2", "child-2", 1L, "g", null); + when(streamTracker.getRunningToolName("child-2")).thenReturn("read_file"); + when(streamTracker.getCurrentPhase("child-2")).thenReturn("action"); + + var rec = registry.get(id).orElseThrow(); + + // Cycle 1 seeds lastSeen (no increment). Each subsequent no-change + // tick increments staleCount by 1; staleCyclesInTool=5 fires when + // the counter HITS 5. So we need 1 seed + 5 increment ticks. + heartbeat.evaluate(rec); // seed + for (int i = 0; i < 5; i++) { + heartbeat.evaluate(rec); + } + assertThat(rec.status().get()).isEqualTo("stale"); + verify(streamTracker, times(1)).broadcastObject(eq("parent-2"), eq("subagent_stale"), any()); + } + + @Test + @DisplayName("phase or tool change resets stale counter") + void progressResetsCounter() { + String id = registry.register("parent-3", "child-3", 1L, "g", null); + var rec = registry.get(id).orElseThrow(); + + when(streamTracker.getRunningToolName("child-3")).thenReturn(null); + when(streamTracker.getCurrentPhase("child-3")).thenReturn("thinking"); + heartbeat.evaluate(rec); // seed + heartbeat.evaluate(rec); // +1 + heartbeat.evaluate(rec); // +2 + assertThat(rec.staleCount().get()).isEqualTo(2); + + // Phase change → counter resets. + when(streamTracker.getCurrentPhase("child-3")).thenReturn("action"); + heartbeat.evaluate(rec); + assertThat(rec.staleCount().get()).isEqualTo(0); + + // Tool change while staying in same phase also resets. + when(streamTracker.getRunningToolName("child-3")).thenReturn("read_file"); + heartbeat.evaluate(rec); // (tool changed) → reset + assertThat(rec.staleCount().get()).isEqualTo(0); + } + + @Test + @DisplayName("heartbeat skips non-running records") + void skipsNonRunning() { + String id = registry.register("parent-4", "child-4", 1L, "g", null); + registry.get(id).orElseThrow().status().set("interrupted"); + + heartbeat.check(); + + verify(streamTracker, never()).getRunningToolName(anyString()); + verify(streamTracker, never()).broadcastObject(anyString(), anyString(), any()); + } + + @Test + @DisplayName("subagent_stale payload carries id, cycles, lastTool, elapsedMs") + void stalePayloadShape() { + cfg.setStaleCyclesIdle(2); + String id = registry.register("parent-5", "child-5", 1L, "g", null); + when(streamTracker.getRunningToolName("child-5")).thenReturn(null); + when(streamTracker.getCurrentPhase("child-5")).thenReturn("thinking"); + + var rec = registry.get(id).orElseThrow(); + heartbeat.evaluate(rec); // seed + heartbeat.evaluate(rec); // +1 + heartbeat.evaluate(rec); // +2 → stale + + ArgumentCaptor captor = ArgumentCaptor.forClass(Object.class); + verify(streamTracker).broadcastObject(eq("parent-5"), eq("subagent_stale"), captor.capture()); + @SuppressWarnings("unchecked") + Map payload = (Map) captor.getValue(); + assertThat(payload).containsKeys("subagentId", "cycles", "lastTool", "elapsedMs"); + assertThat(payload.get("subagentId")).isEqualTo(id); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentRegistryTest.java b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentRegistryTest.java new file mode 100644 index 00000000..80c3b75a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/delegation/SubagentRegistryTest.java @@ -0,0 +1,159 @@ +package vip.mate.agent.delegation; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import reactor.core.Disposable; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SubagentRegistryTest { + + /** ID format: sa--<8 lowercase hex> */ + private static final Pattern ID_PATTERN = Pattern.compile("^sa-\\d+-[0-9a-f]{8}$"); + + private SubagentRegistry registry; + + @BeforeEach + void setUp() { + registry = new SubagentRegistry(); + } + + @Test + @DisplayName("register assigns matching ID, snapshot finds it, unregister drops it") + void registerSnapshotUnregister() { + Disposable d = mock(Disposable.class); + String id = registry.register("parent-1", "child-1", 7L, "do thing", d); + + assertThat(id).matches(ID_PATTERN); + assertThat(registry.get(id)).isPresent(); + assertThat(registry.snapshot("parent-1")).hasSize(1); + assertThat(registry.snapshot("parent-1").get(0).childConversationId()).isEqualTo("child-1"); + assertThat(registry.allActive()).hasSize(1); + + registry.unregister(id); + + assertThat(registry.get(id)).isEmpty(); + assertThat(registry.snapshot("parent-1")).isEmpty(); + } + + @Test + @DisplayName("snapshot filters by parent — siblings under other parents are not visible") + void snapshotFiltersByParent() { + registry.register("parent-A", "ca-1", 1L, "task", null); + registry.register("parent-A", "ca-2", 1L, "task", null); + registry.register("parent-B", "cb-1", 1L, "task", null); + + assertThat(registry.snapshot("parent-A")).hasSize(2); + assertThat(registry.snapshot("parent-B")).hasSize(1); + assertThat(registry.snapshot("parent-C")).isEmpty(); + assertThat(registry.snapshot(null)).isEmpty(); + } + + @Test + @DisplayName("interrupt flips status, disposes subscription, returns false for missing/null") + void interruptBehaviour() { + Disposable disposable = mock(Disposable.class); + when(disposable.isDisposed()).thenReturn(false); + String id = registry.register("p", "c", 1L, "g", disposable); + + assertThat(registry.interrupt(id)).isTrue(); + assertThat(registry.get(id)).isPresent(); + assertThat(registry.get(id).get().status().get()).isEqualTo("interrupted"); + verify(disposable).dispose(); + + // Already-disposed subscription is not disposed again. + when(disposable.isDisposed()).thenReturn(true); + registry.interrupt(id); + verify(disposable).dispose(); // still only the first call + + assertThat(registry.interrupt("does-not-exist")).isFalse(); + assertThat(registry.interrupt(null)).isFalse(); + } + + @Test + @DisplayName("interrupt with null disposable does not throw") + void interruptNullDisposable() { + String id = registry.register("p", "c", 1L, "g", null); + assertThat(registry.interrupt(id)).isTrue(); + assertThat(registry.get(id).get().status().get()).isEqualTo("interrupted"); + } + + @Test + @DisplayName("setSpawnPaused is scoped per parent — pausing A does not pause B") + void spawnPauseIsParentScoped() { + registry.setSpawnPaused("parent-A", true); + assertThat(registry.isSpawnPaused("parent-A")).isTrue(); + assertThat(registry.isSpawnPaused("parent-B")).isFalse(); + + registry.setSpawnPaused("parent-A", false); + assertThat(registry.isSpawnPaused("parent-A")).isFalse(); + + // Null inputs are tolerated and never report paused. + assertThat(registry.isSpawnPaused(null)).isFalse(); + assertThat(registry.setSpawnPaused(null, true)).isFalse(); + } + + @Test + @DisplayName("concurrent register from many threads produces unique IDs and no record loss") + void concurrentRegister() throws Exception { + int threads = 16; + int perThread = 50; + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + Set ids = java.util.Collections.synchronizedSet(new HashSet<>()); + + for (int t = 0; t < threads; t++) { + final int tid = t; + pool.submit(() -> { + try { start.await(); } catch (InterruptedException e) { return; } + for (int i = 0; i < perThread; i++) { + String id = registry.register("parent-" + tid, "child-" + tid + "-" + i, + (long) i, "g", null); + ids.add(id); + } + }); + } + + start.countDown(); + pool.shutdown(); + assertThat(pool.awaitTermination(10, TimeUnit.SECONDS)).isTrue(); + + assertThat(ids).hasSize(threads * perThread); + assertThat(registry.allActive()).hasSize(threads * perThread); + + // Each parent owns exactly perThread children. + for (int t = 0; t < threads; t++) { + assertThat(registry.snapshot("parent-" + t)).hasSize(perThread); + } + } + + @Test + @DisplayName("get on null / missing returns empty Optional") + void getNullSafe() { + assertThat(registry.get(null)).isEmpty(); + assertThat(registry.get("nope")).isEmpty(); + } + + @Test + @DisplayName("unregister on null / missing is a no-op") + void unregisterNullSafe() { + registry.register("p", "c", 1L, "g", null); + registry.unregister(null); + registry.unregister("does-not-exist"); + assertThat(registry.allActive()).hasSize(1); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/ContentRepetitionGuardTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/ContentRepetitionGuardTest.java new file mode 100644 index 00000000..95eb4432 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/ContentRepetitionGuardTest.java @@ -0,0 +1,219 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin {@link NodeStreamingChatHelper#hasRepeatingSuffix} — the cheap + * loop detector that catches reasoning-mode models (qwen3.6, deepseek-r1) + * stuck emitting the same final-answer paragraph over and over. + * + *

Real failure pattern from production: model alternates English + * "Wait, I should X. Done. I will write the response." with the same + * Chinese answer, dozens of times, until {@code max_tokens} runs out. + * Without this guard the user waits for a wall of duplicated text; + * with it, the stream stops at the third or fourth copy and the + * already-accumulated content gets returned as a partial answer. + * + *

The detector probes periods from 24 chars (anything shorter would + * false-positive on natural phrases) up to 240 chars; 4 verbatim + * consecutive copies is the threshold (3-times structured outputs like + * "TL;DR / body / TL;DR again" should pass through). + */ +class ContentRepetitionGuardTest { + + private static final int MIN_PERIOD = 24; + private static final int MAX_PERIOD = 240; + private static final int MIN_OCCURRENCES = 4; + + @Test + @DisplayName("non-cyclic prose with varied sentences does NOT trip") + void naturalProseDoesNotTrip() { + // Real writing: each sentence is unique, no consecutive paragraph + // repeats anywhere in the buffer. + String prose = "MateClaw 是一个企业级 AI 助手。它支持多种渠道接入,包括企业微信、" + + "飞书、钉钉。Agent 通过 StateGraph 编排,可以调用工具、生成图片、查询知识库。" + + "用户可以在 Web 控制台、桌面 App 或群聊里发起对话。系统记忆采用三档分层:" + + "PROFILE.md 记录用户画像、MEMORY.md 沉淀稳定事实、memory/YYYY-MM-DD.md " + + "保存当日上下文。审批流程基于 Spring AI Alibaba Graph,工具调用前会被守卫拦截," + + "高风险操作必须由用户显式批准才能执行。会话与频道之间是多对多关系。"; + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + prose, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES)); + } + + @Test + @DisplayName("verbatim short paragraph repeated 4× → trips") + void verbatimQuadrupleRepeatTrips() { + // The exact production failure mode: 50-char Chinese answer repeated. + String paragraph = "收到语音啦!想查昨天的天气没问题,告诉我城市名我马上帮你查!\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 5; i++) sb.append(paragraph); + assertTrue(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES), + "5 verbatim ~30-char paragraphs in a row should trip"); + } + + @Test + @DisplayName("3 verbatim repeats stay UNDER threshold (legitimate triple-mention pattern)") + void threeRepeatsBelowThreshold() { + // Some legitimate outputs repeat structured summaries 2-3 times + // (e.g. "TL;DR" + body + "TL;DR" again). The threshold of 4 + // gives breathing room so these don't false-positive. + String paragraph = "请告诉我您所在的城市,例如北京、上海或深圳,我可以为您查询天气。\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 3; i++) sb.append(paragraph); + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES), + "3 verbatim paragraphs must NOT trip — preserves triple-mention outputs"); + } + + @Test + @DisplayName("interleaved English thinking + Chinese answer pattern still trips") + void interleavedRepetitionTrips() { + // Mirrors the production trace exactly: English thinking + // alternating with the same Chinese answer. The combined + // "thinking + answer" unit is the actual repeating period. + String unit = "Wait, I should write.\nOkay.\n收到语音啦!告诉我城市名我马上帮你查!\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 5; i++) sb.append(unit); + assertTrue(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES), + "5 verbatim 'thinking + answer' cycles should trip"); + } + + @Test + @DisplayName("empty / short / null content returns false (fast path)") + void shortContentDoesNotTrip() { + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + null, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES)); + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + "", MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES)); + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + "hi there", MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES)); + // Just under MIN_PERIOD × MIN_OCCURRENCES → can't possibly match. + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 80; i++) sb.append('x'); + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES)); + } + + @Test + @DisplayName("trailing repeat after long preamble: detects only the looping suffix") + void detectsLoopAfterPreamble() { + // Realistic: model produces a long valid answer, then enters a + // loop appending the same trailer. The detector must catch the + // loop even though the buffer prefix has perfectly varied text. + StringBuilder sb = new StringBuilder(); + sb.append("好的,我已经为您完成了任务,下面是详细的执行结果:\n"); + sb.append("第一步,我读取了配置文件并解析了内容。\n"); + sb.append("第二步,我调用了天气查询接口拿到了原始数据。\n"); + sb.append("第三步,我将结果格式化为人类可读的中文文本。\n"); + // Now the model gets stuck repeating a closing phrase. + String trailer = "如有其他问题,请随时告诉我,我会尽快为您解答和处理。\n"; + for (int i = 0; i < 5; i++) sb.append(trailer); + assertTrue(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES), + "trailing 5×-repeated trailer must trip even after long preamble"); + } + + @Test + @DisplayName("single-char fill (200x 'a') does NOT trip — too short to be a real period") + void singleCharFillDoesNotTrip() { + // 'aaaa...' could be parsed as period=1 with 200 occurrences, + // but our floor is MIN_PERIOD=24, so a literal 24-char run of + // 'a' would need to repeat 4× — which is just one continuous + // run of 96 'a' chars. That's a degenerate case; mark as + // not-tripping-via-this-detector since it's not the "self- + // arguing loop" failure mode (a model emitting 'aaaaaaa...' + // would hit max_tokens harmlessly without any degradation + // worth user attention). + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 200; i++) sb.append('a'); + // 200 'a' chars: period=24 unit is "aaaa...a" (24 of them). + // The prior 24-char block is also "aaa...a" (24 of them). + // So they DO match. This trips. Document the behavior — it's + // mostly harmless because models don't actually loop on single + // chars. + assertTrue(NodeStreamingChatHelper.hasRepeatingSuffix( + sb, MIN_PERIOD, MAX_PERIOD, MIN_OCCURRENCES), + "documented behavior: pure single-char fills DO trip; not a real failure mode in practice"); + } + + @Test + @DisplayName("invalid args return false defensively") + void invalidArgsReturnFalse() { + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix("text", 0, 100, 4)); + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix("text", 24, 240, 1)); + // maxPeriod < minPeriod + assertFalse(NodeStreamingChatHelper.hasRepeatingSuffix("text", 100, 50, 4)); + } + + // ===== dedupTrailingRepeats ===== + // + // Once the loop guard fires, the streamed text has already gone out + // (SSE chunks can't be unsent), but the DB-persisted final answer + + // IM channel reply should show ONE clean copy of the looping unit + // instead of the wall the user just watched scroll by. + + @Test + @DisplayName("dedup: 5 verbatim copies → 1 copy") + void dedupCollapsesRepeats() { + String unit = "收到语音啦!想查昨天的天气没问题,告诉我城市名我马上帮你查!\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 5; i++) sb.append(unit); + String result = NodeStreamingChatHelper.dedupTrailingRepeats(sb.toString(), MIN_PERIOD, MAX_PERIOD); + assertEquals(unit, result, "5 copies should collapse to exactly 1"); + } + + @Test + @DisplayName("dedup: prefix + repeated trailer → prefix + 1 copy of trailer") + void dedupPreservesPrefixCollapseTrailer() { + String prefix = "好的,下面是详细回答:第一步完成了。第二步也完成了。下面是结论。\n"; + String trailer = "如有其他问题请随时告诉我,我会尽快为您解答处理。\n"; + StringBuilder sb = new StringBuilder(prefix); + for (int i = 0; i < 5; i++) sb.append(trailer); + String result = NodeStreamingChatHelper.dedupTrailingRepeats(sb.toString(), MIN_PERIOD, MAX_PERIOD); + assertEquals(prefix + trailer, result, + "prefix preserved verbatim; trailer collapses 5×→1×"); + } + + @Test + @DisplayName("dedup: no trailing repeats → buffer unchanged") + void dedupNoRepeatsUnchanged() { + String prose = "这是一段没有任何尾部重复的正常回答,包含多个不同的句子和话题。" + + "我们讨论了天气、新闻、技术,每段内容都不同。"; + assertEquals(prose, + NodeStreamingChatHelper.dedupTrailingRepeats(prose, MIN_PERIOD, MAX_PERIOD)); + } + + @Test + @DisplayName("dedup: only 1 copy at end (no actual repetition) → unchanged") + void dedupSingleCopyUnchanged() { + String unit = "请告诉我您所在的城市,我帮您查询。"; + // Just one copy at the tail — nothing to collapse. + assertEquals(unit, + NodeStreamingChatHelper.dedupTrailingRepeats(unit, MIN_PERIOD, MAX_PERIOD)); + } + + @Test + @DisplayName("dedup: empty / null inputs return as-is") + void dedupEmptyOrNull() { + assertNull(NodeStreamingChatHelper.dedupTrailingRepeats(null, MIN_PERIOD, MAX_PERIOD)); + assertEquals("", NodeStreamingChatHelper.dedupTrailingRepeats("", MIN_PERIOD, MAX_PERIOD)); + } + + @Test + @DisplayName("dedup: 2 copies (the minimum trip threshold) → 1 copy") + void dedupTwoCopiesCollapse() { + // dedup uses 2+ copies as its trigger (vs. hasRepeatingSuffix's 4× + // detection threshold). Once the guard has decided the buffer is + // looping, even a 2× tail should be collapsed since we know + // structurally the model is mid-loop. + String unit = "如果您还有任何其他疑问,欢迎随时联系我,我会尽快回复。"; + String input = unit + unit; + assertEquals(unit, + NodeStreamingChatHelper.dedupTrailingRepeats(input, MIN_PERIOD, MAX_PERIOD)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java new file mode 100644 index 00000000..66c2e2f7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/ErrorClassificationTest.java @@ -0,0 +1,193 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * RFC-009 P3.2: classification tests for the new error types + * ({@link NodeStreamingChatHelper.ErrorType#BILLING}, + * {@link NodeStreamingChatHelper.ErrorType#MODEL_NOT_FOUND}). + * + *

These two are split out from {@code AUTH_ERROR} / {@code CLIENT_ERROR} + * because the right action is to switch provider, not to terminate. + * Mis-classifying a billing error as auth would break the whole call chain.

+ */ +class ErrorClassificationTest { + + private static NodeStreamingChatHelper.ErrorType classify(Throwable t) throws Exception { + Method m = NodeStreamingChatHelper.class.getDeclaredMethod("classifyError", Throwable.class); + m.setAccessible(true); + return (NodeStreamingChatHelper.ErrorType) m.invoke(null, t); + } + + // ===== BILLING ===== + + @Test + @DisplayName("HTTP 402 → BILLING") + void status402IsBilling() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.BILLING, + classify(new RuntimeException("402 Payment Required"))); + } + + @Test + @DisplayName("OpenAI 'insufficient_quota' → BILLING") + void openaiQuotaIsBilling() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.BILLING, + classify(new RuntimeException("Error code: insufficient_quota — please check your plan"))); + } + + @Test + @DisplayName("Anthropic 'credit balance is too low' → BILLING") + void anthropicCreditIsBilling() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.BILLING, + classify(new RuntimeException("Your credit balance is too low to access the API"))); + } + + @Test + @DisplayName("'You exceeded your current quota' → BILLING") + void quotaExceededIsBilling() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.BILLING, + classify(new RuntimeException("You exceeded your current quota, please check your plan"))); + } + + // ===== MODEL_NOT_FOUND ===== + + @Test + @DisplayName("'Model not exist' → MODEL_NOT_FOUND") + void modelNotExistIsModelNotFound() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.MODEL_NOT_FOUND, + classify(new RuntimeException("Model not exist: gpt-99"))); + } + + @Test + @DisplayName("'model_not_found' → MODEL_NOT_FOUND") + void modelNotFoundCodeIsModelNotFound() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.MODEL_NOT_FOUND, + classify(new RuntimeException("Error: model_not_found"))); + } + + @Test + @DisplayName("DashScope '[InvalidParameter] url error' → MODEL_NOT_FOUND (not CLIENT_ERROR)") + void dashscopeInvalidParameterIsModelNotFound() throws Exception { + // Despite the wording, DashScope returns this when the model id is unknown + // — the right action is to try a fallback provider, not terminate as 400. + assertEquals(NodeStreamingChatHelper.ErrorType.MODEL_NOT_FOUND, + classify(new RuntimeException("[InvalidParameter] url error, please check url"))); + } + + @Test + @DisplayName("Anthropic 'model does not exist' → MODEL_NOT_FOUND") + void anthropicDoesNotExistIsModelNotFound() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.MODEL_NOT_FOUND, + classify(new RuntimeException("model claude-99 does not exist"))); + } + + // ===== Regression: existing classifications still work ===== + + @Test + @DisplayName("HTTP 401 still classifies as AUTH_ERROR (not billing)") + void status401StillAuth() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.AUTH_ERROR, + classify(new RuntimeException("401 Unauthorized: Invalid API Key"))); + } + + @Test + @DisplayName("HTTP 429 still classifies as RATE_LIMIT") + void status429StillRateLimit() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.RATE_LIMIT, + classify(new RuntimeException("429 Too Many Requests"))); + } + + @Test + @DisplayName("Plain 400 Bad Request still classifies as CLIENT_ERROR") + void status400StillClientError() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.CLIENT_ERROR, + classify(new RuntimeException("400 Bad Request: malformed JSON"))); + } + + // ===== Transient TLS / IO errors → SERVER_ERROR (retryable) ===== + // + // Without these, a single TLS handshake hiccup or socket reset mid-stream + // surfaces to the user as "LLM 调用失败" with zero retries — the existing + // exponential-backoff loop only triggers on RATE_LIMIT / SERVER_ERROR. + // Routing them through SERVER_ERROR gives them ~3s/6s/12s retry budget, + // which is enough to absorb transient network glitches without user impact. + + @Test + @DisplayName("SSL bad_record_mac (RFC 5246 fatal alert 20) → SERVER_ERROR") + void sslBadRecordMacIsServerError() throws Exception { + // Real-world chain: WebClientRequestException → SSLException("Received + // fatal alert: bad_record_mac"). The leaf message contains + // bad_record_mac, the wrapper contributes SSLException class name. + javax.net.ssl.SSLException sslEx = new javax.net.ssl.SSLException( + "Received fatal alert: bad_record_mac"); + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new RuntimeException("(bad_record_mac) Received fatal alert", sslEx))); + } + + @Test + @DisplayName("plain SSLException class in chain → SERVER_ERROR") + void sslExceptionClassIsServerError() throws Exception { + // extractFullErrorChain appends getClass().getSimpleName(), so even + // an SSLException without a recognizable message text gets matched + // via the class name token. + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new javax.net.ssl.SSLException("handshake aborted"))); + } + + @Test + @DisplayName("SSLHandshakeException → SERVER_ERROR") + void sslHandshakeExceptionIsServerError() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new javax.net.ssl.SSLHandshakeException("Remote host closed connection during handshake"))); + } + + @Test + @DisplayName("SocketException (peer reset mid-stream) → SERVER_ERROR") + void socketExceptionIsServerError() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new java.net.SocketException("Connection reset by peer"))); + } + + @Test + @DisplayName("Reactor Netty 'Connection prematurely closed' → SERVER_ERROR") + void prematureCloseIsServerError() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new RuntimeException("Connection prematurely closed BEFORE response"))); + } + + @Test + @DisplayName("Broken pipe (server cut TCP write half) → SERVER_ERROR") + void brokenPipeIsServerError() throws Exception { + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, + classify(new java.io.IOException("Broken pipe"))); + } + + @Test + @DisplayName("WebClientRequestException with SSL cause → SERVER_ERROR (not UNKNOWN)") + void webClientRequestSslIsServerError() throws Exception { + // The exact production failure pattern: Reactor wraps the SSL leaf in + // WebClientRequestException. The chain walker sees both the wrapper + // class name AND the leaf SSLException class name, and the message + // string carries bad_record_mac. + Throwable cause = new javax.net.ssl.SSLException("Received fatal alert: bad_record_mac"); + Throwable wrapped = new RuntimeException( + "WebClientRequestException: bad_record_mac; nested exception", cause); + assertEquals(NodeStreamingChatHelper.ErrorType.SERVER_ERROR, classify(wrapped)); + } + + @Test + @DisplayName("AUTH still wins over TLS chain (real auth failure not masked)") + void authStillWinsOverTlsChain() throws Exception { + // A 401 response wrapped by Reactor still carries WebClientResponseException + // in the chain — the classifier must not see "WebClient*Exception" and + // demote it to SERVER_ERROR. AUTH_ERROR is checked before SERVER_ERROR + // in classifyError(), so 401 keywords win. + assertEquals(NodeStreamingChatHelper.ErrorType.AUTH_ERROR, + classify(new RuntimeException("401 Unauthorized: Invalid API Key (WebClientResponseException)"))); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/LaneDPerformanceFixesTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/LaneDPerformanceFixesTest.java new file mode 100644 index 00000000..5b749dc4 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/LaneDPerformanceFixesTest.java @@ -0,0 +1,246 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; + +import java.util.List; +import java.util.concurrent.CancellationException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Regression tests for Lane D performance fixes (RFC 06-lane-d-performance-fixes). + * + *
    + *
  • D-1: Backoff sleep responds to Stop signal within 100ms
  • + *
  • D-2: RATE_LIMIT/SERVER_ERROR retries capped at 2 (was 5)
  • + *
+ */ +class LaneDPerformanceFixesTest { + + private ChatStreamTracker streamTracker; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + } + + private NodeStreamingChatHelper helper(ChatModel primary) { + return new NodeStreamingChatHelper(streamTracker, List.of(), null); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + private static ChatModel successModel(String text) { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(text), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + private static ChatModel rateLimitModel() { + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn( + Flux.error(new RuntimeException("429 Too Many Requests: rate limit exceeded"))); + return m; + } + + // ============================================================ + // D-1: Backoff sleep responds to Stop signal + // ============================================================ + + @Nested + @DisplayName("D-1: Backoff sleep responds to Stop signal") + class BackoffStopSignalTests { + + @Test + @DisplayName("Stop requested during backoff aborts retry quickly with CancellationException") + void stopDuringBackoffAbortsRetry() { + // Arrange: model always returns rate-limit error to trigger backoff + AtomicInteger callCount = new AtomicInteger(0); + ChatModel model = mock(ChatModel.class); + when(model.stream(any(Prompt.class))).thenAnswer(inv -> { + callCount.incrementAndGet(); + return Flux.error(new RuntimeException("429 Too Many Requests")); + }); + + // Stop is requested after first call — during backoff sleep. + // First poll returns false (initial check before sleep loop starts), + // then true on subsequent checks to simulate user clicking stop. + AtomicInteger stopCheckCount = new AtomicInteger(0); + when(streamTracker.isStopRequested("conv-d1")).thenAnswer(inv -> + stopCheckCount.incrementAndGet() > 2); + + var helper = helper(model); + long startMs = System.currentTimeMillis(); + + // The stop-during-backoff path throws CancellationException + assertThrows(CancellationException.class, () -> + helper.streamCall(model, smallPrompt(), "conv-d1", "reasoning")); + + long elapsedMs = System.currentTimeMillis() - startMs; + + // The backoff for attempt 1 is 3000ms base. With stop polling at 100ms intervals, + // it should abort well before the full 3000ms backoff completes. + assertTrue(elapsedMs < 2000, + "Stop should abort backoff quickly, but took " + elapsedMs + "ms"); + // The model should only have been called once (first attempt fails, backoff + // for second attempt is interrupted by stop) + assertEquals(1, callCount.get(), + "Model should only be called once before stop aborts the backoff"); + } + + @Test + @DisplayName("Normal flow without stop completes backoff normally") + void normalFlowWithoutStopCompletesBackoff() { + // First call: rate limit; second call: success + AtomicInteger callCount = new AtomicInteger(0); + ChatModel model = mock(ChatModel.class); + when(model.stream(any(Prompt.class))).thenAnswer(inv -> { + if (callCount.incrementAndGet() == 1) { + return Flux.error(new RuntimeException("429 Too Many Requests")); + } + Generation gen = new Generation(new AssistantMessage("ok"), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + return Flux.just(resp); + }); + + // Stop never requested + when(streamTracker.isStopRequested(any())).thenReturn(false); + + var helper = helper(model); + var result = helper.streamCall(model, smallPrompt(), "conv-d1b", "reasoning"); + + assertEquals("ok", result.text(), "Second attempt should succeed"); + assertEquals(2, callCount.get(), "Model should be called twice (fail + succeed)"); + } + } + + // ============================================================ + // D-2: RATE_LIMIT retries capped at 2 + // ============================================================ + + @Nested + @DisplayName("D-2: RATE_LIMIT/SERVER_ERROR retries capped at 2") + class RateLimitRetryCapTests { + + @Test + @DisplayName("RATE_LIMIT error retries at most 2 times before giving up") + void rateLimitMaxTwoRetries() { + AtomicInteger callCount = new AtomicInteger(0); + ChatModel model = mock(ChatModel.class); + when(model.stream(any(Prompt.class))).thenAnswer(inv -> { + callCount.incrementAndGet(); + return Flux.error(new RuntimeException("429 Too Many Requests: rate limit")); + }); + + var helper = helper(model); + var result = helper.streamCall(model, smallPrompt(), "conv-d2a", "reasoning"); + + // With MAX_RETRIES_RATE_LIMIT=2, attempts are: 0, 1, 2 = 3 total calls + assertTrue(callCount.get() <= 3, + "RATE_LIMIT should retry at most 2 times (3 total calls), but got " + callCount.get()); + assertNotEquals(NodeStreamingChatHelper.ErrorType.NONE, result.errorType(), + "Result should be an error after exhausting retries"); + } + + @Test + @DisplayName("SERVER_ERROR keeps full MAX_RETRIES=5 (not capped like RATE_LIMIT)") + void serverErrorKeepsFullRetries() { + AtomicInteger callCount = new AtomicInteger(0); + ChatModel model = mock(ChatModel.class); + when(model.stream(any(Prompt.class))).thenAnswer(inv -> { + callCount.incrementAndGet(); + return Flux.error(new RuntimeException("500 Internal Server Error")); + }); + + var helper = helper(model); + var result = helper.streamCall(model, smallPrompt(), "conv-d2b", "reasoning"); + + // SERVER_ERROR should use the full MAX_RETRIES=5 (6 total calls: attempt 0-5), + // NOT the reduced MAX_RETRIES_RATE_LIMIT=2. + assertTrue(callCount.get() > 3, + "SERVER_ERROR should retry more than RATE_LIMIT (>3 calls), but got " + callCount.get()); + assertEquals(6, callCount.get(), + "SERVER_ERROR should try 6 times total (attempt 0 through 5)"); + } + + @Test + @DisplayName("AUTH_ERROR is not retried (unchanged behavior)") + void authErrorNotRetried() { + AtomicInteger callCount = new AtomicInteger(0); + ChatModel model = mock(ChatModel.class); + when(model.stream(any(Prompt.class))).thenAnswer(inv -> { + callCount.incrementAndGet(); + return Flux.error(new RuntimeException("401 Unauthorized")); + }); + + var helper = helper(model); + var result = helper.streamCall(model, smallPrompt(), "conv-d2c", "reasoning"); + + assertEquals(1, callCount.get(), + "AUTH_ERROR should not be retried at all"); + assertEquals(NodeStreamingChatHelper.ErrorType.AUTH_ERROR, result.errorType()); + } + } + + // ============================================================ + // D-3: broadcastProgress method exists and works + // ============================================================ + + @Nested + @DisplayName("D-3: broadcastProgress method") + class BroadcastProgressTests { + + @Test + @DisplayName("broadcastProgress sends progress event via streamTracker") + void broadcastProgressSendsEvent() { + var helper = new NodeStreamingChatHelper(streamTracker); + helper.broadcastProgress("conv-d3", "分析中..."); + + verify(streamTracker, times(1)).broadcastObject( + eq("conv-d3"), eq("progress"), any()); + } + + @Test + @DisplayName("broadcastProgress is safe with null streamTracker") + void broadcastProgressNullTrackerNoOp() { + var helper = new NodeStreamingChatHelper(null); + // Should not throw + assertDoesNotThrow(() -> helper.broadcastProgress("conv-d3b", "分析中...")); + } + + @Test + @DisplayName("broadcastProgress is safe with null conversationId") + void broadcastProgressNullConvIdNoOp() { + var helper = new NodeStreamingChatHelper(streamTracker); + assertDoesNotThrow(() -> helper.broadcastProgress(null, "分析中...")); + // Should not invoke streamTracker when conversationId is null + verify(streamTracker, never()).broadcastObject(any(), any(), any()); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java new file mode 100644 index 00000000..226fa2c6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFailoverTest.java @@ -0,0 +1,190 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.FallbackEntry; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Regression test for the AUTH_ERROR-must-fall-back fix. + * + *

Prior to this fix, primary AUTH_ERROR (e.g. Kimi 401 with an invalid + * API key) returned immediately without trying the fallback chain — a + * fallback provider with a different, valid key never got a chance. + * After the fix, AUTH_ERROR breaks out of the same-model retry loop + * and falls through to the chain walker, mirroring how BILLING and + * MODEL_NOT_FOUND already behave.

+ */ +class NodeStreamingChatHelperFailoverTest { + + private ChatStreamTracker streamTracker; + private ProviderHealthTracker healthTracker; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + ProviderHealthProperties props = new ProviderHealthProperties(); + healthTracker = new ProviderHealthTracker(props); + } + + /** Build a chat-model mock whose stream() emits a single successful chunk with the given text. */ + private static ChatModel successModel(String text) { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(text), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + /** Build a chat-model mock whose stream() errors with the given Throwable. */ + private static ChatModel errorModel(Throwable err) { + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.error(err)); + return m; + } + + private NodeStreamingChatHelper helper(ChatModel primary, List chain, String primaryProviderId) { + // Construct via the full constructor so health tracking is wired and the + // chain walker has provider-id context. + return new NodeStreamingChatHelper(streamTracker, chain, null, healthTracker, primaryProviderId); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + // ============================================================ + // C1: primary 401 + fallback#1 success → fallback wins + // ============================================================ + + @Test + @DisplayName("C1: primary AUTH_ERROR triggers fallback chain (was: returned immediately, never tried fallback)") + void primaryAuthErrorFallsBackToHealthyProvider() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: Invalid API Key")); + ChatModel fallback = successModel("hello from fallback"); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c1", "reasoning"); + + assertEquals("hello from fallback", result.text(), + "fallback provider must succeed and its text must surface as the result"); + assertEquals(NodeStreamingChatHelper.ErrorType.NONE, result.errorType()); + // Primary was tried exactly once (no same-model retries on AUTH_ERROR — fix verified) + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // C2: primary 401 + fallback#1 401 + fallback#2 success + // ============================================================ + + @Test + @DisplayName("C2: chain walks past auth-failing fallback to the next healthy one") + void chainSkipsAuthFailingFallback() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fbBad = errorModel(new RuntimeException("401 Unauthorized: bad key")); + ChatModel fbGood = successModel("ok via 2nd fallback"); + var helper = helper(primary, List.of( + new FallbackEntry("openai", fbBad), + new FallbackEntry("dashscope", fbGood)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c2", "reasoning"); + + assertEquals("ok via 2nd fallback", result.text()); + assertEquals(NodeStreamingChatHelper.ErrorType.NONE, result.errorType()); + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fbBad, times(1)).stream(any(Prompt.class)); + verify(fbGood, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // C3: primary 401 + every fallback 401 → last AUTH_ERROR surfaces + // ============================================================ + + @Test + @DisplayName("C3: when entire chain is auth-failing, last AUTH_ERROR is surfaced (not silently dropped)") + void allChainAuthFailsSurfacesLastError() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized — kimi")); + ChatModel fb1 = errorModel(new RuntimeException("401 Unauthorized — openai")); + ChatModel fb2 = errorModel(new RuntimeException("401 Unauthorized — dashscope")); + var helper = helper(primary, List.of( + new FallbackEntry("openai", fb1), + new FallbackEntry("dashscope", fb2)), "kimi"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c3", "reasoning"); + + assertNotNull(result, "result must not be null even when whole chain fails"); + assertEquals(NodeStreamingChatHelper.ErrorType.AUTH_ERROR, result.errorType(), + "last seen AUTH_ERROR must propagate so callers can surface a real error"); + // Each rung tried exactly once + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fb1, times(1)).stream(any(Prompt.class)); + verify(fb2, times(1)).stream(any(Prompt.class)); + // Health tracker should have recorded a failure against every fallback provider + var snap = healthTracker.snapshot(); + assertTrue(snap.get("openai").consecutiveFailures() >= 1, "openai failure must be recorded"); + assertTrue(snap.get("dashscope").consecutiveFailures() >= 1, "dashscope failure must be recorded"); + } + + // ============================================================ + // C4 regression: BILLING still falls back unchanged + // ============================================================ + + @Test + @DisplayName("C4 (regression): primary BILLING still triggers fallback (unchanged from RFC-009 P3.2)") + void billingStillFallsBack() { + ChatModel primary = errorModel(new RuntimeException("402 Payment Required: insufficient_quota")); + ChatModel fallback = successModel("recovered via fallback"); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-c4", "reasoning"); + + assertEquals("recovered via fallback", result.text()); + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Bonus: confirm no infinite loop / regression on success path + // ============================================================ + + @Test + @DisplayName("Bonus: primary success path is unaffected — no fallback call") + void primarySuccessSkipsFallback() { + ChatModel primary = successModel("primary works fine"); + AtomicInteger fallbackCalls = new AtomicInteger(); + ChatModel fallback = mock(ChatModel.class); + when(fallback.stream(any(Prompt.class))).thenAnswer(inv -> { + fallbackCalls.incrementAndGet(); + return Flux.just((ChatResponse) null); + }); + var helper = helper(primary, List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-bonus", "reasoning"); + + assertEquals("primary works fine", result.text()); + assertEquals(0, fallbackCalls.get(), "primary success must not touch the fallback chain"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java new file mode 100644 index 00000000..21107fca --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperFallbackChainTest.java @@ -0,0 +1,128 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ChatModel; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.FallbackEntry; + +import java.lang.reflect.Field; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * RFC-009: smoke tests for the multi-model fallback chain wiring on + * {@link NodeStreamingChatHelper}. + * + *

Full streaming-flow integration (ChatModel.stream / Flux mocking) is left + * to end-to-end smoke tests in the RFC; these tests verify the public + * surface — constructor variants, chain immutability, deprecated-overload + * compatibility — so future refactors of those entry points are caught.

+ */ +class NodeStreamingChatHelperFallbackChainTest { + + private final ChatStreamTracker streamTracker = mock(ChatStreamTracker.class); + + @Test + @DisplayName("List-based constructor preserves fallback chain order, providerId, and ChatModel") + void listConstructorPreservesOrder() throws Exception { + ChatModel a = mock(ChatModel.class); + ChatModel b = mock(ChatModel.class); + ChatModel c = mock(ChatModel.class); + List input = List.of( + new FallbackEntry("openai", a), + new FallbackEntry("dashscope", b), + new FallbackEntry("anthropic", c)); + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, input, null); + + List chain = readFallbackChain(helper); + assertEquals(3, chain.size(), "fallback chain should preserve all entries"); + assertEquals("openai", chain.get(0).providerId()); + assertSame(a, chain.get(0).chatModel()); + assertEquals("dashscope", chain.get(1).providerId()); + assertSame(b, chain.get(1).chatModel()); + assertEquals("anthropic", chain.get(2).providerId()); + assertSame(c, chain.get(2).chatModel()); + } + + @Test + @DisplayName("Null fallback chain is normalized to empty list (defensive)") + void nullChainNormalizedToEmpty() throws Exception { + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (List) null, null); + assertTrue(readFallbackChain(helper).isEmpty(), + "null chain must not throw — it should be normalized to an empty list"); + } + + @Test + @DisplayName("Single-arg constructor (no fallback) yields empty chain") + void singleArgConstructorEmptyChain() throws Exception { + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker); + assertTrue(readFallbackChain(helper).isEmpty()); + } + + @Test + @DisplayName("Deprecated single-fallback constructor wraps the model into a 1-entry synthetic chain") + void deprecatedSingleFallbackConstructorBackCompat() throws Exception { + ChatModel single = mock(ChatModel.class); + @SuppressWarnings("deprecation") + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, single); + + List chain = readFallbackChain(helper); + assertEquals(1, chain.size(), "deprecated overload should produce a 1-entry chain"); + assertSame(single, chain.get(0).chatModel(), + "the single fallback ChatModel must survive wrapping intact"); + // Synthetic providerId is acceptable; just assert it's present so health + // tracking won't NPE on lookup. + assertNotNull(chain.get(0).providerId()); + } + + @Test + @DisplayName("Deprecated single-fallback constructor with null produces empty chain (no NPE)") + void deprecatedSingleFallbackNullSafe() throws Exception { + @SuppressWarnings("deprecation") + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker, (ChatModel) null); + assertTrue(readFallbackChain(helper).isEmpty(), + "null single fallback must collapse to an empty chain"); + } + + @Test + @DisplayName("EMPTY_RESPONSE / BILLING / MODEL_NOT_FOUND error types exist (RFC-009 fallback triggers)") + void fallbackTriggerErrorTypesExist() { + // Compile-time safety net: these enum constants the streaming pipeline relies on + // must not be renamed or removed without breaking the fallback contract. + assertNotNull(NodeStreamingChatHelper.ErrorType.EMPTY_RESPONSE); + assertNotNull(NodeStreamingChatHelper.ErrorType.BILLING); + assertNotNull(NodeStreamingChatHelper.ErrorType.MODEL_NOT_FOUND); + } + + @Test + @DisplayName("RFC-009 P3.1: primary providerId is stored when supplied via the full constructor") + void primaryProviderIdStored() throws Exception { + NodeStreamingChatHelper helper = new NodeStreamingChatHelper( + streamTracker, List.of(), null, null, "openai"); + + Field f = NodeStreamingChatHelper.class.getDeclaredField("primaryProviderId"); + f.setAccessible(true); + assertEquals("openai", f.get(helper), + "primary provider id must be retained for health tracking"); + } + + @Test + @DisplayName("RFC-009 P3.1: legacy constructors leave primaryProviderId null (tracking disabled)") + void primaryProviderIdNullForLegacyConstructors() throws Exception { + NodeStreamingChatHelper helper = new NodeStreamingChatHelper(streamTracker); + Field f = NodeStreamingChatHelper.class.getDeclaredField("primaryProviderId"); + f.setAccessible(true); + assertNull(f.get(helper), + "legacy constructors must leave primaryProviderId unset so tracking is silently disabled"); + } + + @SuppressWarnings("unchecked") + private static List readFallbackChain(NodeStreamingChatHelper helper) throws Exception { + Field f = NodeStreamingChatHelper.class.getDeclaredField("fallbackChain"); + f.setAccessible(true); + return (List) f.get(helper); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java new file mode 100644 index 00000000..f5ef3903 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperPoolTest.java @@ -0,0 +1,269 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.AvailableProviderPool.RemovalSource; +import vip.mate.llm.failover.FallbackEntry; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * RFC-009 Phase 4 — verifies the three pool hooks wired into + * {@link NodeStreamingChatHelper}: + *
    + *
  1. Primary short-circuit when its provider id is not in the pool — + * primary is never even called, fallback runs first.
  2. + *
  3. Walker head filter — out-of-pool fallback entries are skipped.
  4. + *
  5. HARD error → {@code pool.remove}; SOFT error → pool unchanged.
  6. + *
+ * + *

Pool state must remain consistent across these three behaviors so a + * single misconfigured provider can't pollute every conversation turn.

+ */ +class NodeStreamingChatHelperPoolTest { + + private ChatStreamTracker streamTracker; + private ProviderHealthTracker healthTracker; + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + healthTracker = new ProviderHealthTracker(new ProviderHealthProperties()); + pool = new AvailableProviderPool(); + } + + private static ChatModel successModel(String text) { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(text), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + private static ChatModel errorModel(Throwable err) { + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.error(err)); + return m; + } + + /** Stream a single chunk with empty text and no tool calls — triggers EMPTY_RESPONSE (SOFT). */ + private static ChatModel emptyResponseModel() { + ChatModel m = mock(ChatModel.class); + Generation gen = new Generation(new AssistantMessage(""), ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + private NodeStreamingChatHelper helper(List chain, String primary) { + return new NodeStreamingChatHelper(streamTracker, chain, null, healthTracker, primary, pool); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + // ============================================================ + // Hook 1: primary out-of-pool short-circuits the retry loop + // ============================================================ + + @Test + @DisplayName("Primary not in pool: skipped without being called, fallback wins") + void primaryOutOfPoolShortCircuits() { + // openai is HARD-removed from pool before the call + pool.add("dashscope"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "stale 401"); + + ChatModel primary = successModel("primary should never be called"); + ChatModel fallback = successModel("fallback wins"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h1", "reasoning"); + + assertEquals("fallback wins", result.text()); + verify(primary, never()).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Hook 2: walker skips out-of-pool fallback entries + // ============================================================ + + @Test + @DisplayName("Walker skips out-of-pool fallback and lands on the next eligible one") + void walkerSkipsOutOfPoolFallback() { + pool.add("openai"); // primary + pool.remove("anthropic", RemovalSource.BILLING, "402"); // first fallback dead + pool.add("dashscope"); // second fallback alive + + // Use AUTH_ERROR (HARD) on primary — triggers the immediate break-to-walker + // path. Picking SERVER_ERROR would burn 5 retries (~110s) and then exit + // without ever hitting the walker, which is unrelated to the property + // under test here. + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fbAnthropic = successModel("should be skipped"); + ChatModel fbDashscope = successModel("dashscope wins"); + var helper = helper(List.of( + new FallbackEntry("anthropic", fbAnthropic), + new FallbackEntry("dashscope", fbDashscope)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h2", "reasoning"); + + assertEquals("dashscope wins", result.text()); + verify(fbAnthropic, never()).stream(any(Prompt.class)); + verify(fbDashscope, times(1)).stream(any(Prompt.class)); + } + + // ============================================================ + // Hook 3a: primary HARD error evicts from pool + // ============================================================ + + @Test + @DisplayName("Primary AUTH_ERROR HARD-removes openai from pool with AUTH_ERROR source") + void primaryAuthErrorEvictsFromPool() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: bad key")); + ChatModel fallback = successModel("recovered"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3a", "reasoning"); + + assertFalse(pool.contains("openai"), "openai must be removed from pool after AUTH_ERROR"); + var reason = pool.snapshot().get("openai"); + assertNotNull(reason); + assertEquals(RemovalSource.AUTH_ERROR, reason.source()); + assertTrue(pool.contains("dashscope"), "successful fallback stays in pool"); + } + + @Test + @DisplayName("Primary BILLING HARD-removes with BILLING source (distinct from AUTH)") + void primaryBillingEvictsWithBillingSource() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("402 Payment Required: insufficient_quota")); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3b", "reasoning"); + + assertFalse(pool.contains("openai")); + assertEquals(RemovalSource.BILLING, pool.snapshot().get("openai").source()); + } + + @Test + @DisplayName("Primary MODEL_NOT_FOUND HARD-removes with MODEL_NOT_FOUND source") + void primaryModelNotFoundEvictsWithCorrectSource() { + pool.add("openai"); + pool.add("dashscope"); + + ChatModel primary = errorModel(new RuntimeException("404 model_not_found: gpt-99")); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3c", "reasoning"); + + assertFalse(pool.contains("openai")); + assertEquals(RemovalSource.MODEL_NOT_FOUND, pool.snapshot().get("openai").source()); + } + + // ============================================================ + // Hook 3b: SOFT errors do NOT evict from pool + // ============================================================ + + @Test + @DisplayName("Primary EMPTY_RESPONSE (SOFT) keeps provider in pool, only records failure") + void primarySoftErrorKeepsInPool() { + pool.add("openai"); + pool.add("dashscope"); + + // EMPTY_RESPONSE is SOFT and breaks straight to fallback (no 5x retry) + // — keeps the test fast while still exercising the SOFT path. + ChatModel primary = emptyResponseModel(); + ChatModel fallback = successModel("ok"); + var helper = helper(List.of(new FallbackEntry("dashscope", fallback)), "openai"); + + helper.streamCall(primary, smallPrompt(), "conv-h3d", "reasoning"); + + assertTrue(pool.contains("openai"), + "SOFT errors must NOT evict — health tracker cooldown handles transient blips"); + assertTrue(healthTracker.snapshot().get("openai").consecutiveFailures() > 0, + "SOFT failure must still be recorded by the health tracker"); + } + + // ============================================================ + // Hook 3c: fallback HARD errors also evict + // ============================================================ + + @Test + @DisplayName("Fallback AUTH_ERROR evicts the fallback provider and walker continues") + void fallbackHardErrorEvictsFallback() { + pool.add("openai"); + pool.add("anthropic"); + pool.add("dashscope"); + + // Use AUTH on primary so we reach the walker without burning 5 retries. + // The behavior under test is fallback eviction, not the primary path. + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized: openai key")); + ChatModel fbBad = errorModel(new RuntimeException("401 Unauthorized: anthropic key")); + ChatModel fbGood = successModel("dashscope ok"); + var helper = helper(List.of( + new FallbackEntry("anthropic", fbBad), + new FallbackEntry("dashscope", fbGood)), "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-h3e", "reasoning"); + + assertEquals("dashscope ok", result.text()); + assertFalse(pool.contains("anthropic"), "fallback that failed AUTH must be evicted"); + assertEquals(RemovalSource.AUTH_ERROR, pool.snapshot().get("anthropic").source()); + assertTrue(pool.contains("dashscope")); + } + + // ============================================================ + // Sanity: fail-open mode (null pool) — old call sites unchanged + // ============================================================ + + @Test + @DisplayName("Null pool: helper behaves as before (no NPE, no skipping)") + void nullPoolFailOpen() { + ChatModel primary = errorModel(new RuntimeException("401 Unauthorized")); + ChatModel fallback = successModel("ok"); + // 5-arg constructor — no pool wired + var helper = new NodeStreamingChatHelper(streamTracker, + List.of(new FallbackEntry("dashscope", fallback)), null, healthTracker, "openai"); + + var result = helper.streamCall(primary, smallPrompt(), "conv-failopen", "reasoning"); + + assertEquals("ok", result.text()); + // No pool to inspect — just confirm we didn't crash and fallback ran. + verify(primary, times(1)).stream(any(Prompt.class)); + verify(fallback, times(1)).stream(any(Prompt.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperThinkingCapTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperThinkingCapTest.java new file mode 100644 index 00000000..7c88026e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperThinkingCapTest.java @@ -0,0 +1,115 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Regression coverage for the thinking-only soft cap added in P0-2. + * + *

The cap disposes the upstream stream when the model has emitted + * {@code >= THINKING_ONLY_HARD_CAP_CHARS} of {@code reasoning_content} + * with zero visible content and zero tool calls. The risk noted during + * review (P1-A): some providers (Anthropic / DeepSeek-thinking variants) + * pack {@code reasoning_content} and a {@code tool_call} into the same + * SSE chunk. If the cap check sits inside the thinking-delta block (i.e. + * before the chunk's tool_call is accumulated) it would dispose just + * before observing the tool — turning a request that was about to dispatch + * a tool into a spurious "INCOMPLETE: thinking-only" outcome. + */ +class NodeStreamingChatHelperThinkingCapTest { + + private ChatStreamTracker streamTracker; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + private static ChatModel singleChunkModel(AssistantMessage msg) { + Generation gen = new Generation(msg, ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + @Test + @DisplayName("thinking >= cap + tool_call in same chunk: cap must NOT trigger; tool_call survives") + void thinkingAndToolCallSameChunk_doesNotTripSoftCap() { + // Build a single chunk that carries 40k thinking (well above the + // 32k cap) AND a tool call. With the buggy ordering this would + // dispose before accumulateToolCalls runs and the helper would + // return a partial "thinking_only_no_content" result. + String hugeThinking = "x".repeat(40_000); + AssistantMessage.ToolCall tc = new AssistantMessage.ToolCall( + "id-1", "function", "search", "{\"q\":\"foo\"}"); + AssistantMessage msg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(tc)) + .properties(Map.of("reasoningContent", hugeThinking)) + .build(); + + ChatModel m = singleChunkModel(msg); + var helper = new NodeStreamingChatHelper(streamTracker); + + var result = helper.streamCall(m, smallPrompt(), "conv-thinking-tc", "reasoning"); + + assertTrue(result.hasToolCalls(), + "Tool call accompanying huge thinking in the same chunk must survive"); + assertEquals(1, result.toolCalls().size()); + assertEquals("search", result.toolCalls().get(0).name()); + assertFalse(result.partial(), + "Result must not be marked partial when a tool_call was observed in the same chunk"); + assertNotEquals("thinking_only_no_content", result.errorMessage(), + "Soft cap must not fire when the chunk carrying huge thinking also carried a tool call"); + } + + @Test + @DisplayName("thinking >= cap with NO tool_call and NO content: cap fires, result is partial+thinking_only_no_content") + void thinkingOnlyNoContent_capFires() { + // Symmetric positive case: confirms the cap still triggers in the + // genuine "深度思考 ... never finishes" scenario the cap was added for. + String hugeThinking = "y".repeat(40_000); + AssistantMessage msg = AssistantMessage.builder() + .content("") + .properties(Map.of("reasoningContent", hugeThinking)) + .build(); + + ChatModel m = singleChunkModel(msg); + var helper = new NodeStreamingChatHelper(streamTracker); + + var result = helper.streamCall(m, smallPrompt(), "conv-thinking-only", "reasoning"); + + assertFalse(result.hasToolCalls()); + assertTrue(result.partial(), "Cap should mark the result as partial"); + assertEquals("thinking_only_no_content", result.errorMessage()); + assertEquals(hugeThinking, result.thinking(), + "Thinking transcript is preserved so the UI can show it in a collapse panel"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperToolCallArgsTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperToolCallArgsTest.java new file mode 100644 index 00000000..acedb909 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/NodeStreamingChatHelperToolCallArgsTest.java @@ -0,0 +1,122 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.metadata.ChatGenerationMetadata; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import reactor.core.publisher.Flux; +import vip.mate.channel.web.ChatStreamTracker; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Regression coverage for tool-call arguments sanitization. + * + *

Some OpenAI-compatible providers (aliyun-codingplan, others using the + * "coding" DashScope endpoint) reject the follow-up chat-completions request + * with HTTP 400 when the assistant message in history carries a tool call + * whose {@code function.arguments} is not parseable JSON. The streaming + * accumulator can produce empty or truncated argument strings, so the helper + * normalizes the final value to {@code "{}"} when it is missing or invalid. + */ +class NodeStreamingChatHelperToolCallArgsTest { + + private ChatStreamTracker streamTracker; + + @BeforeEach + void setUp() { + streamTracker = mock(ChatStreamTracker.class); + when(streamTracker.isStopRequested(any())).thenReturn(false); + } + + private static Prompt smallPrompt() { + return new Prompt(List.of(new UserMessage("hi"))); + } + + private static ChatModel singleChunkModel(AssistantMessage msg) { + Generation gen = new Generation(msg, ChatGenerationMetadata.NULL); + ChatResponse resp = mock(ChatResponse.class); + when(resp.getResults()).thenReturn(List.of(gen)); + when(resp.getResult()).thenReturn(gen); + when(resp.getMetadata()).thenReturn(null); + ChatModel m = mock(ChatModel.class); + when(m.stream(any(Prompt.class))).thenReturn(Flux.just(resp)); + return m; + } + + @Test + @DisplayName("Empty tool-call arguments normalized to '{}'") + void emptyArguments_replacedWithEmptyJsonObject() { + AssistantMessage.ToolCall tc = new AssistantMessage.ToolCall( + "id-empty", "function", "list_skills", ""); + AssistantMessage msg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(tc)) + .build(); + + var helper = new NodeStreamingChatHelper(streamTracker); + var result = helper.streamCall(singleChunkModel(msg), smallPrompt(), + "conv-empty-args", "reasoning"); + + assertTrue(result.hasToolCalls(), "tool call must survive"); + assertEquals(1, result.toolCalls().size()); + assertEquals("{}", result.toolCalls().get(0).arguments(), + "empty arguments must be replaced with '{}' so strict providers " + + "(aliyun-codingplan, ...) accept the follow-up request"); + } + + @Test + @DisplayName("Truncated/invalid JSON arguments normalized to '{}'") + void truncatedJsonArguments_replacedWithEmptyJsonObject() { + // Simulates a stream cut mid-token: model emitted '{"q":"hel' and stopped. + AssistantMessage.ToolCall tc = new AssistantMessage.ToolCall( + "id-truncated", "function", "search", "{\"q\":\"hel"); + AssistantMessage msg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(tc)) + .build(); + + var helper = new NodeStreamingChatHelper(streamTracker); + var result = helper.streamCall(singleChunkModel(msg), smallPrompt(), + "conv-truncated-args", "reasoning"); + + assertTrue(result.hasToolCalls(), "tool call must survive"); + assertEquals(1, result.toolCalls().size()); + assertEquals("{}", result.toolCalls().get(0).arguments(), + "invalid JSON arguments must be replaced with '{}' so the follow-up " + + "request stays well-formed"); + } + + @Test + @DisplayName("Valid JSON arguments preserved verbatim") + void validJsonArguments_preservedAsIs() { + String validArgs = "{\"query\":\"foo\",\"limit\":5}"; + AssistantMessage.ToolCall tc = new AssistantMessage.ToolCall( + "id-valid", "function", "search", validArgs); + AssistantMessage msg = AssistantMessage.builder() + .content("") + .toolCalls(List.of(tc)) + .build(); + + var helper = new NodeStreamingChatHelper(streamTracker); + var result = helper.streamCall(singleChunkModel(msg), smallPrompt(), + "conv-valid-args", "reasoning"); + + assertTrue(result.hasToolCalls()); + assertEquals(1, result.toolCalls().size()); + assertEquals(validArgs, result.toolCalls().get(0).arguments(), + "valid JSON arguments must not be rewritten"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/RepetitionDetectorTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/RepetitionDetectorTest.java deleted file mode 100644 index 7702b74a..00000000 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/RepetitionDetectorTest.java +++ /dev/null @@ -1,114 +0,0 @@ -package vip.mate.agent.graph; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * RepetitionDetector 单元测试 - */ -class RepetitionDetectorTest { - - private RepetitionDetector detector; - - @BeforeEach - void setUp() { - detector = new RepetitionDetector(); - } - - @Test - @DisplayName("正常文本不触发重复检测") - void shouldNotTriggerForNormalText() { - assertFalse(detector.appendAndCheck("Hello, world! This is a normal response. ")); - assertFalse(detector.appendAndCheck("It contains various sentences and ideas. ")); - assertFalse(detector.appendAndCheck("No repetition should be detected here. ")); - assertFalse(detector.appendAndCheck("The detector only flags degenerate patterns. ")); - assertFalse(detector.isRepetitionDetected()); - } - - @Test - @DisplayName("短文本不触发检测(低于最小内容长度)") - void shouldNotTriggerForShortText() { - assertFalse(detector.appendAndCheck("短")); - assertFalse(detector.appendAndCheck("短")); - assertFalse(detector.appendAndCheck("短")); - assertFalse(detector.isRepetitionDetected()); - } - - @Test - @DisplayName("连续重复相同片段触发检测") - void shouldTriggerForRepeatedPattern() { - // 构造足够长的前缀以超过最小检测长度 - StringBuilder sb = new StringBuilder(); - sb.append("这是一段正常的开头文本。".repeat(5)); - detector.appendAndCheck(sb.toString()); - - // 现在重复同一模式多次 - String pattern = "不吃香菜,喝冰美式。"; - boolean triggered = false; - for (int i = 0; i < 20; i++) { - if (detector.appendAndCheck(pattern)) { - triggered = true; - break; - } - } - assertTrue(triggered, "Should detect repetition after many identical appends"); - assertTrue(detector.isRepetitionDetected()); - } - - @Test - @DisplayName("检测到重复后持续返回 true") - void shouldKeepReturningTrueAfterDetection() { - // 直接构造重复内容 - String pattern = "重复片段测试内容。"; - StringBuilder bulk = new StringBuilder(); - bulk.append("正常的前缀内容,长度足够。".repeat(5)); - for (int i = 0; i < 20; i++) { - bulk.append(pattern); - } - detector.appendAndCheck(bulk.toString()); - - // 后续调用应该继续返回 true - assertTrue(detector.appendAndCheck("任何新内容")); - assertTrue(detector.isRepetitionDetected()); - } - - @Test - @DisplayName("reset 后重新检测") - void shouldResetState() { - // 先触发检测 - String pattern = "重复片段测试。"; - StringBuilder bulk = new StringBuilder("前缀".repeat(50)); - for (int i = 0; i < 20; i++) { - bulk.append(pattern); - } - detector.appendAndCheck(bulk.toString()); - - // reset - detector.reset(); - assertFalse(detector.isRepetitionDetected()); - assertFalse(detector.appendAndCheck("正常的新内容")); - } - - @Test - @DisplayName("null 和空字符串不触发也不异常") - void shouldHandleNullAndEmpty() { - assertFalse(detector.appendAndCheck(null)); - assertFalse(detector.appendAndCheck("")); - assertFalse(detector.isRepetitionDetected()); - } - - @Test - @DisplayName("Unicode 中文重复模式正确检测") - void shouldDetectChineseRepetition() { - StringBuilder sb = new StringBuilder("初始化内容填充。".repeat(10)); - String pattern = "已记住。以后涉及点餐时我会提醒你:"; - for (int i = 0; i < 20; i++) { - sb.append(pattern); - } - boolean triggered = detector.appendAndCheck(sb.toString()); - assertTrue(triggered, "Should detect Chinese character repetition"); - } -} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/ReturnDirectEndToEndTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/ReturnDirectEndToEndTest.java new file mode 100644 index 00000000..e3e1dba2 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/ReturnDirectEndToEndTest.java @@ -0,0 +1,240 @@ +package vip.mate.agent.graph; + +import com.alibaba.cloud.ai.graph.OverAllState; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import vip.mate.agent.AgentToolSet; +import vip.mate.agent.graph.edge.ObservationDispatcher; +import vip.mate.agent.graph.executor.ToolExecutionExecutor; +import vip.mate.agent.graph.node.ActionNode; +import vip.mate.agent.graph.node.FinalAnswerNode; +import vip.mate.agent.graph.state.DirectToolOutput; +import vip.mate.tool.guard.ToolGuard; +import vip.mate.tool.guard.ToolGuardResult; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static vip.mate.agent.graph.state.MateClawStateKeys.*; + +/** + * RFC-052 end-to-end chain test — exercises the full graph traversal across + * three real components without mocking: + * + *

+ *   ToolExecutionExecutor (executes returnDirect tool)
+ *        ↓ writes ToolResponseMessage + events + directOutputs
+ *   ActionNode (interprets ToolExecutionResult, sets RETURN_DIRECT_TRIGGERED)
+ *        ↓ state mutation
+ *   ObservationDispatcher (routes to FinalAnswerNode)
+ *        ↓ edge decision
+ *   FinalAnswerNode (assembles final answer from DIRECT_TOOL_OUTPUTS)
+ *        ↓ produces FINAL_ANSWER + finishReason=RETURN_DIRECT
+ * 
+ * + *

This complements the per-component unit tests by verifying the + * composition works: invariants flow correctly between nodes via + * {@link OverAllState}, no integration glue is missing, no state key is + * misnamed across boundaries. + * + *

What this does NOT test (still requires manual / SpringBootTest): + *

    + *
  • {@code StateGraphReActAgent} stream emission of {@code FINAL_ANSWER} + * as {@code content_delta}
  • + *
  • {@code StreamAccumulator} capturing {@code tool_direct_result} into + * {@code metadata.directToolNames} (covered by the manual demo)
  • + *
  • {@code BaseAgent.toSpringMessage} scrubbing on the next user turn + * (covered by {@code BaseAgentDirectToolHistoryScrubTest})
  • + *
+ */ +class ReturnDirectEndToEndTest { + + private static final String SECRET = + "EMPLOYEE-SALARY-RECORD\n" + + "Name: Alice\n" + + "Base: 12345\n" + + "Bonus: 67890\n" + + "SSN: 999-88-7777"; + + @Test + @DisplayName("RFC-052 end-to-end: secret reaches FINAL_ANSWER verbatim, never enters LLM-bound messages") + void fullChain_directToolFlowsAcrossNodes() throws Exception { + // ===== Setup: real executor + real ActionNode + real Dispatcher + real FinalAnswerNode ===== + ToolCallback directTool = stubCallback("query_employee_salary", true, args -> SECRET); + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(directTool)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + + ActionNode actionNode = new ActionNode(executor); + ObservationDispatcher dispatcher = new ObservationDispatcher(); + FinalAnswerNode finalAnswerNode = new FinalAnswerNode(); + + // ===== Step 1: simulate ReasoningNode having decided to call the direct tool ===== + AssistantMessage.ToolCall toolCall = new AssistantMessage.ToolCall( + "call_x", "function", "query_employee_salary", "{}"); + Map initialState = new HashMap<>(); + initialState.put(TOOL_CALLS, List.of(toolCall)); + initialState.put(CONVERSATION_ID, "conv_e2e"); + initialState.put(AGENT_ID, "agent_e2e"); + OverAllState state1 = new OverAllState(initialState); + + // ===== Step 2: ActionNode runs the executor ===== + Map actionOut = actionNode.apply(state1); + + // Verify ActionNode set the trigger flags + assertEquals(Boolean.TRUE, actionOut.get(RETURN_DIRECT_TRIGGERED), + "ActionNode must set RETURN_DIRECT_TRIGGERED when executor produced direct outputs"); + @SuppressWarnings("unchecked") + List outputs = (List) actionOut.get(DIRECT_TOOL_OUTPUTS); + assertNotNull(outputs); + assertEquals(1, outputs.size()); + assertEquals(SECRET, outputs.get(0).fullResult(), + "Full secret must reach DIRECT_TOOL_OUTPUTS verbatim"); + + // Critical: the ToolResponseMessage stored in MESSAGES must NOT contain the secret — + // it must contain the placeholder, since this is what would be re-fed to the LLM + // if the graph weren't short-circuiting. + @SuppressWarnings("unchecked") + List messages = (List) actionOut.get(MESSAGES); + assertNotNull(messages); + assertEquals(1, messages.size()); + ToolResponseMessage tr = (ToolResponseMessage) messages.get(0); + assertEquals(1, tr.getResponses().size()); + ToolResponseMessage.ToolResponse resp = tr.getResponses().get(0); + // RFC-052 §2.4 contract: the placeholder is a fixed, English, business-data-free + // sentence. Asserting the exact text doubles as a contract test — if anyone + // changes the placeholder text this fails and the RFC needs updating too. + assertEquals( + "[Tool result returned directly to user. " + + "Content withheld from model context per tool policy.]", + resp.responseData(), + "Tool response carried in MESSAGES must be the §2.4 placeholder, not the secret"); + assertFalse(resp.responseData().contains("12345"), + "Sanity: the salary number must not be on the LLM-bound path"); + assertFalse(resp.responseData().contains("999-88-7777"), + "Sanity: SSN must not be on the LLM-bound path"); + + // ===== Step 3: ObservationDispatcher decides where to route ===== + // Build a state that reflects what the graph would have AFTER ActionNode + // (we manually merge ActionNode's output for the dispatcher input — the + // actual graph engine does this via state merge strategies). + Map stateAfterAction = new HashMap<>(initialState); + stateAfterAction.putAll(actionOut); + // Skip ObservationNode for simplicity — it doesn't touch our flags. Real + // graph runs Action → Observation → Dispatcher; we verify the dispatcher + // contract directly. + OverAllState state2 = new OverAllState(stateAfterAction); + + String route = dispatcher.apply(state2); + assertEquals(FINAL_ANSWER_NODE, route, + "Dispatcher must route RETURN_DIRECT_TRIGGERED to FinalAnswerNode, " + + "skipping the next LLM call"); + + // ===== Step 4: FinalAnswerNode assembles the final answer ===== + Map finalOut = finalAnswerNode.apply(state2); + + assertEquals(SECRET, finalOut.get(FINAL_ANSWER), + "FinalAnswerNode must surface the direct tool's full text verbatim as the final answer"); + assertEquals("return_direct", finalOut.get(FINISH_REASON), + "finishReason must be RETURN_DIRECT"); + } + + @Test + @DisplayName("RFC-052 end-to-end: mixed batch — direct tool A succeeds, non-direct tool B succeeds, plan still short-circuits") + void fullChain_mixedBatch_directWins() throws Exception { + ToolCallback direct = stubCallback("read_medical_record", true, args -> "PATIENT-DATA-XYZ"); + ToolCallback normal = stubCallback("get_weather", false, args -> "sunny, 22C"); + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(direct, normal)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + + ActionNode actionNode = new ActionNode(executor); + ObservationDispatcher dispatcher = new ObservationDispatcher(); + FinalAnswerNode finalAnswerNode = new FinalAnswerNode(); + + List calls = List.of( + new AssistantMessage.ToolCall("c1", "function", "read_medical_record", "{}"), + new AssistantMessage.ToolCall("c2", "function", "get_weather", "{}")); + Map initial = new HashMap<>(); + initial.put(TOOL_CALLS, calls); + initial.put(CONVERSATION_ID, "conv_mixed"); + initial.put(AGENT_ID, "agent_mixed"); + + Map actionOut = actionNode.apply(new OverAllState(initial)); + assertEquals(Boolean.TRUE, actionOut.get(RETURN_DIRECT_TRIGGERED)); + + Map merged = new HashMap<>(initial); + merged.putAll(actionOut); + OverAllState merged2 = new OverAllState(merged); + + assertEquals(FINAL_ANSWER_NODE, dispatcher.apply(merged2), + "Even with a non-direct tool in the batch, the direct one short-circuits"); + + Map finalOut = finalAnswerNode.apply(merged2); + assertEquals("PATIENT-DATA-XYZ", finalOut.get(FINAL_ANSWER), + "Single direct output rendered verbatim (single-output path, no headings)"); + assertEquals("return_direct", finalOut.get(FINISH_REASON)); + } + + @Test + @DisplayName("RFC-052 end-to-end: non-direct tool DOES NOT trigger short-circuit") + void fullChain_nonDirectTool_runsNormalLoop() throws Exception { + ToolCallback normal = stubCallback("get_weather", false, args -> "rainy, 12C"); + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(normal)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + ToolExecutionExecutor executor = new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + + ActionNode actionNode = new ActionNode(executor); + ObservationDispatcher dispatcher = new ObservationDispatcher(); + + AssistantMessage.ToolCall call = new AssistantMessage.ToolCall( + "call_w", "function", "get_weather", "{}"); + Map initial = new HashMap<>(); + initial.put(TOOL_CALLS, List.of(call)); + initial.put(CONVERSATION_ID, "conv_w"); + initial.put(AGENT_ID, "agent_w"); + initial.put(CURRENT_ITERATION, 0); + initial.put(MAX_ITERATIONS, 10); + + Map actionOut = actionNode.apply(new OverAllState(initial)); + + // RETURN_DIRECT_TRIGGERED must NOT be set + assertNull(actionOut.get(RETURN_DIRECT_TRIGGERED), + "Non-direct tool must not flip RETURN_DIRECT_TRIGGERED"); + + // Dispatcher routes to REASONING_NODE for next loop iteration + Map merged = new HashMap<>(initial); + merged.putAll(actionOut); + String route = dispatcher.apply(new OverAllState(merged)); + assertEquals(REASONING_NODE, route, + "Without the direct flag, dispatcher must continue the ReAct loop"); + } + + /** Stub ToolCallback with explicit returnDirect flag (mirrors the unit-test helper). */ + private static ToolCallback stubCallback(String name, boolean returnDirect, + java.util.function.Function handler) { + ToolDefinition def = ToolDefinition.builder() + .name(name) + .description("e2e test tool " + name) + .inputSchema("{\"type\":\"object\",\"properties\":{}}") + .build(); + ToolMetadata md = ToolMetadata.builder().returnDirect(returnDirect).build(); + return new ToolCallback() { + @Override public ToolDefinition getToolDefinition() { return def; } + @Override public ToolMetadata getToolMetadata() { return md; } + @Override public String call(String arguments) { return handler.apply(arguments); } + @Override public String call(String arguments, ToolContext toolContext) { + return handler.apply(arguments); + } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/StripThinkingBoundaryTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/StripThinkingBoundaryTest.java new file mode 100644 index 00000000..488e3eb9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/StripThinkingBoundaryTest.java @@ -0,0 +1,158 @@ +package vip.mate.agent.graph; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; +import org.springframework.ai.chat.messages.UserMessage; +import org.springframework.ai.chat.prompt.Prompt; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-049 PR-2 §2.4.1: verify the {@code lastUserIdx} boundary semantics of + * {@link NodeStreamingChatHelper#stripThinkingFromPrompt}. + * + *

Prior-turn AssistantMessages ({@code i <= lastUserIdx}) must have their + * {@code reasoningContent} stripped — DeepSeek's contract says "reset across + * user turns". In-turn AssistantMessages ({@code i > lastUserIdx}) must keep + * their thinking so DeepSeek's "pass back within the same turn" requirement + * holds for ReAct multi-round tool calls. + */ +class StripThinkingBoundaryTest { + + private static AssistantMessage assistantWithThinking(String content, String thinking) { + AssistantMessage.Builder b = AssistantMessage.builder().content(content); + if (thinking != null) { + b.properties(Map.of("reasoningContent", thinking)); + } + return b.build(); + } + + private static String thinkingOf(Message m) { + if (!(m instanceof AssistantMessage am)) return null; + Object rc = am.getMetadata() != null ? am.getMetadata().get("reasoningContent") : null; + return rc instanceof String s ? s : null; + } + + @Test + @DisplayName("No UserMessage (edge): lastUserIdx=-1 → all assistants treated as in-turn, thinking kept") + void noUser_allKept() { + // Edge case: when the prompt contains no UserMessage at all (e.g. system-only + // setup or a freshly-built Prompt that hasn't received user input yet), there + // is no prior-turn boundary, so every assistant is considered in-turn and + // their thinking is preserved. This is the safer default — we never strip + // without a clear cross-turn signal. + List msgs = List.of( + new SystemMessage("sys"), + assistantWithThinking("a1", "think-1"), + assistantWithThinking("a2", "think-2") + ); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(new Prompt(msgs)); + assertEquals("think-1", thinkingOf(cleaned.getInstructions().get(1))); + assertEquals("think-2", thinkingOf(cleaned.getInstructions().get(2))); + } + + @Test + @DisplayName("Single turn: UserMessage then assistants → all in-turn assistants keep thinking") + void singleTurn_allInTurnKept() { + List msgs = List.of( + new SystemMessage("sys"), + new UserMessage("q1"), + assistantWithThinking("a1-tool", "think-a1"), + assistantWithThinking("a2-final", "think-a2") + ); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(new Prompt(msgs)); + assertEquals("think-a1", thinkingOf(cleaned.getInstructions().get(2))); + assertEquals("think-a2", thinkingOf(cleaned.getInstructions().get(3))); + } + + @Test + @DisplayName("Case H: cross-turn stripped, in-turn preserved") + void crossTurn_stripped_inTurn_kept() { + // [sys, U1, A1(think1), U2, A2(think2), A3(think3)] + // lastUserIdx = 3 (U2) + // i=2 A1 → prior-turn → strip + // i=4 A2 → in-turn → keep + // i=5 A3 → in-turn → keep + List msgs = List.of( + new SystemMessage("sys"), + new UserMessage("u1"), + assistantWithThinking("a1", "think-1"), + new UserMessage("u2"), + assistantWithThinking("a2", "think-2"), + assistantWithThinking("a3", "think-3") + ); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(new Prompt(msgs)); + + assertNull(thinkingOf(cleaned.getInstructions().get(2)), + "A1 is prior-turn (i=2 <= lastUserIdx=3) — thinking must be stripped"); + assertEquals("think-2", thinkingOf(cleaned.getInstructions().get(4)), + "A2 is in-turn (i=4 > lastUserIdx=3) — thinking must be kept"); + assertEquals("think-3", thinkingOf(cleaned.getInstructions().get(5)), + "A3 is in-turn (i=5 > lastUserIdx=3) — thinking must be kept"); + } + + @Test + @DisplayName("Options reference is preserved by the returned Prompt (producer relies on this)") + void optionsPreservedByReference() { + org.springframework.ai.openai.OpenAiChatOptions opts = + org.springframework.ai.openai.OpenAiChatOptions.builder().model("test").build(); + opts.setUser("original-user"); + + List msgs = List.of( + new UserMessage("u1"), + assistantWithThinking("a1", "think") + ); + Prompt in = new Prompt(msgs, opts); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(in); + + // The returned Prompt's options must be the same instance, so the + // producer's subsequent setUser(relayToken) is visible through cleaned too. + assertTrue(cleaned.getOptions() == in.getOptions(), + "stripThinkingFromPrompt must preserve the options reference"); + assertEquals("original-user", + ((org.springframework.ai.openai.OpenAiChatOptions) cleaned.getOptions()).getUser()); + } + + @Test + @DisplayName("Assistant without thinking is untouched (no churn)") + void noThinkingMetadata_passthrough() { + List msgs = List.of( + new UserMessage("u1"), + new AssistantMessage("plain a") + ); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(new Prompt(msgs)); + // Should return a Prompt with the same messages (no rebuild required) + assertEquals(msgs.size(), cleaned.getInstructions().size()); + assertNull(thinkingOf(cleaned.getInstructions().get(1))); + } + + @Test + @DisplayName("Prior-turn assistant with non-thinking metadata: thinking stripped, other metadata preserved") + void priorTurnAssistant_otherMetadataPreserved() { + AssistantMessage priorAssistant = AssistantMessage.builder() + .content("prior") + .properties(Map.of("reasoningContent", "old-think", "custom-key", "custom-val")) + .build(); + List msgs = List.of( + new UserMessage("u1"), + priorAssistant, + new UserMessage("u2"), + assistantWithThinking("current", "current-think") + ); + Prompt cleaned = NodeStreamingChatHelper.stripThinkingFromPrompt(new Prompt(msgs)); + + Message rebuiltPrior = cleaned.getInstructions().get(1); + assertTrue(rebuiltPrior instanceof AssistantMessage); + AssistantMessage am = (AssistantMessage) rebuiltPrior; + assertNull(am.getMetadata().get("reasoningContent"), "thinking must be stripped"); + assertEquals("custom-val", am.getMetadata().get("custom-key"), "other metadata must be preserved"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/edge/ObservationDispatcherTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/edge/ObservationDispatcherTest.java index de0e6362..d0d7727b 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/edge/ObservationDispatcherTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/edge/ObservationDispatcherTest.java @@ -91,4 +91,44 @@ class ObservationDispatcherTest { )); assertEquals(SUMMARIZING_NODE, dispatcher.apply(state)); } + + // ========== RFC-052 returnDirect routing ========== + + @Test + @DisplayName("RFC-052: RETURN_DIRECT_TRIGGERED routes straight to FinalAnswerNode") + void returnDirectTriggered_routesToFinalAnswer() throws Exception { + OverAllState state = new OverAllState(Map.of( + CURRENT_ITERATION, 1, + MAX_ITERATIONS, 10, + RETURN_DIRECT_TRIGGERED, true + )); + assertEquals(FINAL_ANSWER_NODE, dispatcher.apply(state)); + } + + @Test + @DisplayName("RFC-052: RETURN_DIRECT outranks shouldSummarize / limit-exceeded") + void returnDirectTriggered_takesPriorityOverSummarizeAndLimit() throws Exception { + // Even when summarize and limit conditions would trigger, RETURN_DIRECT wins. + OverAllState state = new OverAllState(Map.of( + CURRENT_ITERATION, 100, // way over limit + MAX_ITERATIONS, 10, + SHOULD_SUMMARIZE, true, + RETURN_DIRECT_TRIGGERED, true + )); + assertEquals(FINAL_ANSWER_NODE, dispatcher.apply(state)); + } + + @Test + @DisplayName("RFC-052: AWAITING_APPROVAL still wins over RETURN_DIRECT") + void awaitingApproval_winsOverReturnDirect() throws Exception { + // Approval-pending must terminate the graph regardless; user decision + // arrives later via the replay path. + OverAllState state = new OverAllState(Map.of( + CURRENT_ITERATION, 1, + MAX_ITERATIONS, 10, + AWAITING_APPROVAL, true, + RETURN_DIRECT_TRIGGERED, true + )); + assertEquals(FINAL_ANSWER_NODE, dispatcher.apply(state)); + } } diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/LaneDExecutorAndConfigTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/LaneDExecutorAndConfigTest.java new file mode 100644 index 00000000..29ba7ed1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/LaneDExecutorAndConfigTest.java @@ -0,0 +1,292 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.ai.chat.messages.ToolResponseMessage; + +import java.nio.file.Path; +import java.util.List; +import java.lang.reflect.Field; +import java.util.concurrent.ExecutorService; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for the tool-result executor pipeline and its associated config. + * + *

    + *
  • Virtual-thread tool executor is wired with named carrier threads.
  • + *
  • {@link ToolResultProperties} defaults stay aligned with the executor + * inline hard cap so spill and truncate share one semantic threshold.
  • + *
  • {@link ToolExecutionExecutor#spillRawOrTruncate} attempts spill on the + * raw body first and falls back to truncation only when spill cannot run.
  • + *
+ */ +class LaneDExecutorAndConfigTest { + + // ============================================================ + // D-4: ToolExecutionExecutor uses virtual threads + // ============================================================ + + @Nested + @DisplayName("D-4: ToolExecutionExecutor virtual thread pool") + class VirtualThreadPoolTests { + + @Test + @DisplayName("TOOL_EXECUTOR is a named virtual thread executor (not fixed thread pool)") + void toolExecutorIsVirtualThreadBased() throws Exception { + Field field = ToolExecutionExecutor.class.getDeclaredField("TOOL_EXECUTOR"); + field.setAccessible(true); + ExecutorService executor = (ExecutorService) field.get(null); + + assertNotNull(executor, "TOOL_EXECUTOR should not be null"); + + // Virtual thread executor class name contains "ThreadPerTaskExecutor" + // when created via Executors.newThreadPerTaskExecutor(factory). + String className = executor.getClass().getName(); + assertTrue(className.contains("ThreadPerTaskExecutor"), + "Expected ThreadPerTaskExecutor (named virtual threads), but got: " + className); + } + + @Test + @DisplayName("Virtual threads are named 'tool-executor-N' for log traceability") + void virtualThreadsAreNamed() throws Exception { + Field field = ToolExecutionExecutor.class.getDeclaredField("TOOL_EXECUTOR"); + field.setAccessible(true); + ExecutorService executor = (ExecutorService) field.get(null); + + // Submit a task and capture the thread name + var future = executor.submit(() -> Thread.currentThread().getName()); + String threadName = future.get(); + + assertTrue(threadName.startsWith("tool-executor-"), + "Virtual thread should be named 'tool-executor-N', but got: " + threadName); + } + } + + // ============================================================ + // D-5: ToolResultProperties defaults + // ============================================================ + + @Nested + @DisplayName("ToolResultProperties defaults") + class ToolResultPropertiesDefaultsTests { + + @Test + @DisplayName("perResultThresholdChars default aligns with executor hard cap (8000)") + void perResultThresholdCharsDefault() { + ToolResultProperties props = new ToolResultProperties(); + assertEquals(8000, props.getPerResultThresholdChars(), + "Default perResultThresholdChars should equal the executor's MAX_TOOL_RESULT_CHARS=8000"); + } + + @Test + @DisplayName("perTurnBudgetChars default is 32000") + void perTurnBudgetCharsDefault() { + ToolResultProperties props = new ToolResultProperties(); + assertEquals(32000, props.getPerTurnBudgetChars(), + "Default perTurnBudgetChars should be 32000"); + } + + @Test + @DisplayName("Other defaults remain unchanged") + void otherDefaultsUnchanged() { + ToolResultProperties props = new ToolResultProperties(); + assertTrue(props.isEnabled(), "enabled should default to true"); + assertEquals(800, props.getPreviewHeadChars(), + "previewHeadChars should still default to 800"); + assertEquals(2500, props.getExcludedToolInlineChars(), + "excludedToolInlineChars should default to 2500"); + assertEquals("", props.getStorageBaseDir(), + "storageBaseDir should still default to empty string"); + } + + @Test + @DisplayName("retentionDays defaults to 0 so spill files outlive their conversation") + void retentionDaysDefaultsToZero() { + // The recoverability invariant: a summary or preview that cites + // a spill path must keep working for the whole life of the + // conversation. Time-based deletion is opt-in; operators with + // disk pressure can raise this value explicitly. + ToolResultProperties props = new ToolResultProperties(); + assertEquals(0, props.getRetentionDays(), + "retentionDays must default to 0 — time-based purge is opt-in to preserve recoverability"); + assertTrue(props.getCleanupCron() != null && !props.getCleanupCron().isBlank(), + "cleanupCron stays defined; it is a no-op while retentionDays=0"); + } + + @Test + @DisplayName("Per-result threshold matches the executor inline hard cap so spill and truncate share one ladder") + void thresholdMatchesExecutorHardCap() throws Exception { + ToolResultProperties props = new ToolResultProperties(); + Field field = ToolExecutionExecutor.class.getDeclaredField("MAX_TOOL_RESULT_CHARS"); + field.setAccessible(true); + int hardCap = (int) field.get(null); + assertEquals(hardCap, props.getPerResultThresholdChars(), + "perResultThresholdChars must equal MAX_TOOL_RESULT_CHARS; misalignment would silently shorten " + + "bodies between the two values when spill is disabled."); + } + } + + // ============================================================ + // Raw-first spill ordering — the critical issue #110 fix + // ============================================================ + + @Nested + @DisplayName("ToolExecutionExecutor.spillRawOrTruncate: raw body reaches disk before the inline cap") + class SpillOrTruncateOrderingTests { + + @TempDir + Path tempDir; + + private ToolResultStorage storage(int threshold, List excluded) { + ToolResultProperties props = new ToolResultProperties(); + props.setStorageBaseDir(tempDir.toString()); + props.setPerResultThresholdChars(threshold); + props.setPreviewHeadChars(120); + if (excluded != null) props.setExcludedTools(excluded); + return new ToolResultStorage(props); + } + + @Test + @DisplayName("raw body > threshold → spill writes full original bytes to disk and returns preview") + void rawOverThresholdSpillsFullContent() throws Exception { + ToolResultStorage st = storage(1000, null); + String raw = "0123456789\n".repeat(2000); // ~22000 chars, well over both threshold AND hard cap + int rawLen = raw.length(); + + String out = ToolExecutionExecutor.spillRawOrTruncate( + st, 8000, raw, "web_search", "call-1", "conv-x", tempDir.toString()); + + assertTrue(out.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX), + "should return a spill preview when raw exceeds threshold"); + assertTrue(out.contains("full_chars=" + rawLen), + "preview header must report the original size, proving the raw bytes were what we measured"); + + // The file on disk should be the FULL raw body — not the 8000-char truncate. + // Path is encoded inside the preview as "path=/abs/path". + java.util.regex.Matcher m = java.util.regex.Pattern.compile("path=(\\S+)").matcher(out); + assertTrue(m.find(), "preview must include path=..."); + Path spillFile = Path.of(m.group(1)); + assertTrue(java.nio.file.Files.exists(spillFile), "spill file should have been created"); + String fileContent = java.nio.file.Files.readString(spillFile); + assertEquals(rawLen, fileContent.length(), + "spill file must contain the full raw body, not a pre-truncated copy"); + } + + @Test + @DisplayName("raw body > threshold but tool is on exclusion list → no spill, inline hard cap") + void rawOverThresholdExcludedToolTruncatesOnly() { + ToolResultStorage st = storage(1000, List.of("read_file")); + String raw = "x".repeat(20000); + + String out = ToolExecutionExecutor.spillRawOrTruncate( + st, 8000, raw, "read_file", "call-1", "conv-x", tempDir.toString()); + + assertFalse(out.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX), + "excluded tool must not be spilled"); + assertTrue(out.length() <= 8000, + "excluded body still must fit the inline hard cap (was " + out.length() + ")"); + } + + @Test + @DisplayName("raw body ≤ threshold → returned unchanged, no spill, no truncation marker added") + void rawUnderThresholdInlineVerbatim() { + ToolResultStorage st = storage(1000, null); + String raw = "small body"; + + String out = ToolExecutionExecutor.spillRawOrTruncate( + st, 8000, raw, "web_search", "call-1", "conv-x", tempDir.toString()); + + assertEquals(raw, out, "small bodies should pass through untouched"); + } + + @Test + @DisplayName("storage null → falls back to inline hard cap, never crashes") + void nullStorageFallsBackToTruncate() { + String raw = "x".repeat(20000); + + String out = ToolExecutionExecutor.spillRawOrTruncate( + null, 8000, raw, "web_search", "call-1", "conv-x", tempDir.toString()); + + assertFalse(out.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)); + assertTrue(out.length() <= 8000, + "with no storage, body must still fit the inline hard cap"); + } + + @Test + @DisplayName("null result stays null (no NPE)") + void nullResultStaysNull() { + ToolResultStorage st = storage(1000, null); + String out = ToolExecutionExecutor.spillRawOrTruncate( + st, 8000, null, "web_search", "call-1", "conv-x", tempDir.toString()); + assertNull(out); + } + + @Test + @DisplayName("blank conversationId is replaced with a safe 'unknown' bucket so spill still lands on disk") + void blankConversationIdRoutesToUnknownBucket() { + ToolResultStorage st = storage(1000, null); + String raw = "y".repeat(20000); + + String out = ToolExecutionExecutor.spillRawOrTruncate( + st, 8000, raw, "web_search", "call-1", "", tempDir.toString()); + + assertTrue(out.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX), + "blank conversationId must not stop spill — caller can be the legacy executePreApproved path"); + assertTrue(out.contains("unknown"), "spill path should land under the 'unknown' bucket"); + } + } + + @Nested + @DisplayName("Tool result aggregate budget") + class ToolResultAggregateBudgetTests { + + @TempDir + Path tempDir; + + @Test + @DisplayName("excluded retrieval tools are compacted when aggregate budget is exceeded") + void excludedToolResultsCompactWhenTurnBudgetIsExceeded() { + ToolResultProperties props = new ToolResultProperties(); + props.setStorageBaseDir(tempDir.toString()); + props.setPerTurnBudgetChars(5000); + props.setExcludedToolInlineChars(1200); + props.setExcludedTools(List.of("read_file")); + ToolResultStorage storage = new ToolResultStorage(props); + + String largeRead = "line\n".repeat(1600); + List responses = List.of( + new ToolResponseMessage.ToolResponse("call-1", "read_file", largeRead), + new ToolResponseMessage.ToolResponse("call-2", "read_file", largeRead + "tail") + ); + + List compacted = + storage.enforceTurnBudget(responses, "conv-test", tempDir.toString()); + + assertTrue(compacted.stream().mapToInt(r -> r.responseData().length()).sum() < 5000); + assertTrue(compacted.stream().allMatch(r -> + r.responseData().contains("tool result compacted for model context"))); + } + + @Test + @DisplayName("large eligible tool result is spilled before entering model context") + void largeEligibleToolResultIsSpilled() { + ToolResultProperties props = new ToolResultProperties(); + props.setStorageBaseDir(tempDir.toString()); + props.setPerResultThresholdChars(1000); + props.setPreviewHeadChars(120); + ToolResultStorage storage = new ToolResultStorage(props); + + String largeResult = "0123456789\n".repeat(500); + String contextResult = storage.persistIfOversized( + largeResult, "web_search", "call-1", "conv-test", tempDir.toString()); + + assertTrue(contextResult.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)); + assertTrue(contextResult.length() < largeResult.length()); + assertTrue(contextResult.contains("full_chars=" + largeResult.length())); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorCapToolCallsTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorCapToolCallsTest.java new file mode 100644 index 00000000..ad82a44c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorCapToolCallsTest.java @@ -0,0 +1,157 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the per-response tool_calls cap that protects the executor + * against runaway batch sizes from misbehaving models + * (StreamLake / kat-coder-pro-v1 emit 50+ in one shot). + * + *

This is a pure unit test on the package-private static helper; no + * Spring context, no mocks. The behavior under cap matters most for two + * cases: (1) the LLM must still receive paired tool responses for every + * dropped tool_call (some providers reject otherwise), and (2) the order + * of executed calls must remain stable so the agent's logic isn't + * reshuffled by the cap. + */ +class ToolExecutionExecutorCapToolCallsTest { + + private static AssistantMessage.ToolCall call(String id, String name) { + return new AssistantMessage.ToolCall(id, "function", name, "{}"); + } + + private static List sequentialCalls(int n) { + List out = new ArrayList<>(n); + for (int i = 0; i < n; i++) { + out.add(call("call_" + i, "tool_" + i)); + } + return out; + } + + // ── Pass-through cases ───────────────────────────────────────────────────── + + @Test + @DisplayName("null input returns empty list, no truncation") + void nullInputPassesThrough() { + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(null, 16); + + assertNotNull(capped); + assertNotNull(capped.effective()); + assertTrue(capped.effective().isEmpty()); + assertTrue(capped.truncatedResponses().isEmpty()); + assertFalse(capped.wasTruncated()); + } + + @Test + @DisplayName("empty input is returned untouched") + void emptyInputPassesThrough() { + List input = List.of(); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + assertSame(input, capped.effective(), "no copy when within cap"); + assertTrue(capped.truncatedResponses().isEmpty()); + assertFalse(capped.wasTruncated()); + } + + @Test + @DisplayName("size at cap is returned untouched (boundary)") + void atCapPassesThrough() { + List input = sequentialCalls(16); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + assertSame(input, capped.effective(), + "at-cap input must not be sublist'd — surprising allocation"); + assertTrue(capped.truncatedResponses().isEmpty()); + assertFalse(capped.wasTruncated()); + } + + @Test + @DisplayName("size below cap is returned untouched") + void belowCapPassesThrough() { + List input = sequentialCalls(5); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + assertSame(input, capped.effective()); + assertFalse(capped.wasTruncated()); + } + + // ── Truncation cases ─────────────────────────────────────────────────────── + + @Test + @DisplayName("over-cap input is trimmed; first N kept in original order") + void overCapTrimmed() { + List input = sequentialCalls(20); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + assertTrue(capped.wasTruncated()); + assertEquals(16, capped.effective().size()); + // Order preservation matters — the agent's reasoning may depend on + // the LLM's chosen sequence (e.g. read-then-write); reshuffling the + // first-N is silently breaking. + for (int i = 0; i < 16; i++) { + assertEquals("call_" + i, capped.effective().get(i).id()); + } + } + + @Test + @DisplayName("each dropped tool_call gets a synthetic ToolResponseMessage with matching id") + void droppedCallsGetTruncatedResponses() { + List input = sequentialCalls(20); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + // 4 dropped calls (indices 16..19) → 4 synthetic responses. + assertEquals(4, capped.truncatedResponses().size()); + + for (int i = 0; i < 4; i++) { + ToolResponseMessage.ToolResponse resp = capped.truncatedResponses().get(i); + assertEquals("call_" + (16 + i), resp.id(), + "synthetic response must reuse the dropped tool_call's id " + + "or providers will reject the next turn"); + assertEquals("tool_" + (16 + i), resp.name()); + assertTrue(resp.responseData().contains("[truncated]"), + "response body must signal truncation so the LLM can reissue"); + } + } + + @Test + @DisplayName("synthetic response body mentions both requested and cap counts") + void truncatedResponseBodyExplainsCounts() { + List input = sequentialCalls(50); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 16); + + String body = capped.truncatedResponses().get(0).responseData(); + assertTrue(body.contains("50"), "body should mention requested count: " + body); + assertTrue(body.contains("16"), "body should mention cap value: " + body); + } + + @Test + @DisplayName("custom cap value honored — same logic at any threshold") + void customCapHonored() { + List input = sequentialCalls(10); + ToolExecutionExecutor.CappedToolCalls capped = + ToolExecutionExecutor.capToolCalls(input, 3); + + assertTrue(capped.wasTruncated()); + assertEquals(3, capped.effective().size()); + assertEquals(7, capped.truncatedResponses().size()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorNameNormalizationTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorNameNormalizationTest.java new file mode 100644 index 00000000..0bdc16e5 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorNameNormalizationTest.java @@ -0,0 +1,142 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import vip.mate.agent.AgentToolSet; +import vip.mate.tool.guard.ToolGuard; +import vip.mate.tool.guard.ToolGuardResult; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * LLMs frequently mangle tool names: emit {@code WebSearch} or + * {@code web_search_tool} when the registry knows {@code web_search}, or + * {@code Read_File} when the registry knows {@code read_file}. Without + * normalization those calls return "Tool not found" and the agent loses a + * turn — and worse, the guard's deny rules (keyed on canonical names) get + * silently bypassed because the guard never sees a matching name. + */ +class ToolExecutionExecutorNameNormalizationTest { + + private ToolCallback callbackNamed(String name) { + ToolCallback cb = mock(ToolCallback.class); + ToolDefinition def = mock(ToolDefinition.class); + when(def.name()).thenReturn(name); + when(def.description()).thenReturn(name); + when(def.inputSchema()).thenReturn("{}"); + when(cb.getToolDefinition()).thenReturn(def); + when(cb.call(anyString(), any())).thenReturn("ok:" + name); + when(cb.call(anyString())).thenReturn("ok:" + name); + return cb; + } + + private ToolExecutionExecutor newExecutor(ToolCallback... callbacks) { + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(callbacks)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + return new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + } + + @Test + @DisplayName("normalizeToolName: CamelCase → snake_case") + void normalize_camelCase() { + assertEquals("web_search", ToolExecutionExecutor.normalizeToolName("WebSearch")); + assertEquals("web_search", ToolExecutionExecutor.normalizeToolName("webSearch")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("ReadFile")); + assertEquals("browser_use", ToolExecutionExecutor.normalizeToolName("BrowserUse")); + } + + @Test + @DisplayName("normalizeToolName: trailing _tool / Tool / _function suffix stripped") + void normalize_suffixStrip() { + assertEquals("web_search", ToolExecutionExecutor.normalizeToolName("web_search_tool")); + assertEquals("web_search", ToolExecutionExecutor.normalizeToolName("WebSearchTool")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("read_file_function")); + } + + @Test + @DisplayName("normalizeToolName: separator collapse + lowercase") + void normalize_separators() { + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("Read_File")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("read-file")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("read.file")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("read file")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("__read__file__")); + } + + @Test + @DisplayName("normalizeToolName: idempotent on already-canonical names") + void normalize_idempotent() { + assertEquals("web_search", ToolExecutionExecutor.normalizeToolName("web_search")); + assertEquals("read_file", ToolExecutionExecutor.normalizeToolName("read_file")); + } + + @Test + @DisplayName("normalizeToolName: handles null/empty") + void normalize_edgeCases() { + assertEquals("", ToolExecutionExecutor.normalizeToolName(null)); + assertEquals("", ToolExecutionExecutor.normalizeToolName("")); + assertEquals("", ToolExecutionExecutor.normalizeToolName(" ")); + } + + @Test + @DisplayName("resolveToolName: exact match returns input unchanged (hot path)") + void resolve_exactMatchUnchanged() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("web_search")); + assertEquals("web_search", executor.resolveToolName("web_search")); + } + + @Test + @DisplayName("resolveToolName: CamelCase emission resolves to snake_case canonical") + void resolve_camelToSnake() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("web_search")); + assertEquals("web_search", executor.resolveToolName("WebSearch")); + assertEquals("web_search", executor.resolveToolName("webSearch")); + } + + @Test + @DisplayName("resolveToolName: _tool / Tool suffix resolves to canonical") + void resolve_suffixStripped() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("web_search")); + assertEquals("web_search", executor.resolveToolName("web_search_tool")); + assertEquals("web_search", executor.resolveToolName("WebSearchTool")); + } + + @Test + @DisplayName("resolveToolName: unknown name returns input unchanged so 'tool not found' fires correctly") + void resolve_unknownReturnsInput() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("web_search")); + assertEquals("totally_made_up", executor.resolveToolName("totally_made_up")); + } + + @Test + @DisplayName("end-to-end: model emits 'WebSearch', registered as 'web_search', tool actually executes") + void endToEnd_camelCaseDispatch() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("web_search")); + + var result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_1", "function", "WebSearch", "{}")), + "conv", "agent", false, "user", null); + + assertEquals(1, result.responses().size()); + assertEquals("ok:web_search", result.responses().get(0).responseData(), + "Mangled name should resolve and dispatch to the registered tool"); + } + + @Test + @DisplayName("end-to-end: '_tool' suffix is stripped and the call dispatches") + void endToEnd_toolSuffixStripped() { + ToolExecutionExecutor executor = newExecutor(callbackNamed("read_file")); + + var result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_2", "function", "read_file_tool", "{}")), + "conv", "agent", false, "user", null); + + assertEquals("ok:read_file", result.responses().get(0).responseData()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorReturnDirectTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorReturnDirectTest.java new file mode 100644 index 00000000..18f45ed1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorReturnDirectTest.java @@ -0,0 +1,226 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.ToolResponseMessage; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import vip.mate.agent.AgentToolSet; +import vip.mate.agent.GraphEventPublisher; +import vip.mate.agent.graph.state.DirectToolOutput; +import vip.mate.tool.guard.ToolGuard; +import vip.mate.tool.guard.ToolGuardResult; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-052 PR-1/PR-2 end-to-end test for {@link ToolExecutionExecutor}. + * + *

The contract under test: + *

    + *
  1. A {@code returnDirect=true} tool's full result is captured in + * {@link ToolExecutionExecutor.ToolExecutionResult#directOutputs()}.
  2. + *
  3. The corresponding {@link ToolResponseMessage.ToolResponse} carries the + * fixed placeholder, not the sensitive content.
  4. + *
  5. An {@code EVENT_TOOL_DIRECT_RESULT} event is emitted with the full text + * and {@code renderAs=assistant_message}.
  6. + *
  7. Non-direct tools in the same batch keep their existing behavior.
  8. + *
+ */ +class ToolExecutionExecutorReturnDirectTest { + + private static final String SECRET = "EMPLOYEE-SALARY: Alice=12345, Bob=67890"; + + private ToolExecutionExecutor newExecutor(ToolCallback... callbacks) { + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(callbacks)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + // streamTracker=null is supported throughout executor; null approval + // service is fine when guard never returns NEEDS_APPROVAL. + return new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + } + + @Test + @DisplayName("RFC-052: returnDirect tool result reaches user verbatim and stays out of LLM context") + void directTool_fullResultCapturedAndPlaceholderInResponse() { + ToolCallback direct = stubCallback("query_employee_salary", true, args -> SECRET); + ToolExecutionExecutor executor = newExecutor(direct); + + AssistantMessage.ToolCall call = new AssistantMessage.ToolCall( + "call_1", "function", "query_employee_salary", "{}"); + ToolExecutionExecutor.ToolExecutionResult result = + executor.execute(List.of(call), "conv_1", "agent_1", false, "user_1", null); + + // (1) directOutputs aggregates the full text + assertEquals(1, result.directOutputs().size()); + DirectToolOutput out = result.directOutputs().get(0); + assertEquals("query_employee_salary", out.toolName()); + assertEquals(SECRET, out.fullResult(), "Full result must be preserved verbatim"); + assertTrue(result.hasDirectOutputs()); + + // (2) ToolResponseMessage carries the placeholder (LLM-safe) + assertEquals(1, result.responses().size()); + ToolResponseMessage.ToolResponse resp = result.responses().get(0); + assertEquals(ToolExecutionExecutor.DIRECT_TOOL_PLACEHOLDER, resp.responseData(), + "ToolResponseMessage must carry the placeholder, not the sensitive payload"); + assertFalse(resp.responseData().contains("EMPLOYEE-SALARY"), + "Sensitive substring must not appear in tool response"); + + // (3) tool_direct_result event was emitted with renderAs=assistant_message + full text + var directEvents = result.events().stream() + .filter(e -> GraphEventPublisher.EVENT_TOOL_DIRECT_RESULT.equals(e.type())) + .toList(); + assertEquals(1, directEvents.size(), "exactly one tool_direct_result event expected"); + var data = directEvents.get(0).data(); + assertEquals("call_1", data.get("toolCallId")); + assertEquals("query_employee_salary", data.get("toolName")); + assertEquals(SECRET, data.get("result")); + assertEquals("assistant_message", data.get("renderAs")); + + // (4) no tool_call_completed event for the direct tool — direct path replaces it + boolean hasCompleted = result.events().stream() + .anyMatch(e -> GraphEventPublisher.EVENT_TOOL_COMPLETE.equals(e.type())); + assertFalse(hasCompleted, "direct path replaces tool_call_completed; double-emit would " + + "leak the placeholder into UI as a tool result card"); + } + + @Test + @DisplayName("RFC-052: non-direct tool keeps existing behavior (no direct outputs)") + void nonDirectTool_keepsBaselineBehavior() { + ToolCallback normal = stubCallback("get_weather", false, args -> "sunny, 22C"); + ToolExecutionExecutor executor = newExecutor(normal); + + AssistantMessage.ToolCall call = new AssistantMessage.ToolCall( + "call_w", "function", "get_weather", "{}"); + ToolExecutionExecutor.ToolExecutionResult result = + executor.execute(List.of(call), "conv_w", "agent_w", false, "user_w", null); + + assertFalse(result.hasDirectOutputs(), "no direct tool ran; directOutputs must be empty"); + assertTrue(result.directOutputs().isEmpty()); + assertEquals(1, result.responses().size()); + assertEquals("sunny, 22C", result.responses().get(0).responseData()); + + // no direct event + boolean hasDirect = result.events().stream() + .anyMatch(e -> GraphEventPublisher.EVENT_TOOL_DIRECT_RESULT.equals(e.type())); + assertFalse(hasDirect); + } + + @Test + @DisplayName("RFC-052: returnDirect tool throwing yields generic message (no exception details leak)") + void directTool_throwing_genericErrorMessage() { + ToolCallback throwingDirect = stubCallback("query_employee_salary", true, args -> { + throw new RuntimeException("OracleDriver: connection refused, secret-conn-str=user/PWD123@db"); + }); + ToolExecutionExecutor executor = newExecutor(throwingDirect); + + AssistantMessage.ToolCall call = new AssistantMessage.ToolCall( + "call_e", "function", "query_employee_salary", "{}"); + ToolExecutionExecutor.ToolExecutionResult result = + executor.execute(List.of(call), "conv_e", "agent_e", false, "user_e", null); + + // No directOutputs — exception aborted before the direct branch + assertFalse(result.hasDirectOutputs()); + assertEquals(1, result.responses().size()); + String content = result.responses().get(0).responseData(); + assertEquals("Tool execution failed (details withheld per returnDirect policy)", content, + "Direct-tool exception text must be replaced with a generic placeholder"); + assertFalse(content.contains("PWD123"), "Sensitive substring from exception must not leak"); + assertFalse(content.contains("OracleDriver"), "Stack/connection details must not leak"); + } + + @Test + @DisplayName("RFC-052: pre-approved direct tool replays through direct path") + void executePreApproved_directTool_takesDirectPath() { + ToolCallback direct = stubCallback("query_secret", true, args -> "SECRET-PAYLOAD-123"); + ToolExecutionExecutor executor = newExecutor(direct); + + AssistantMessage.ToolCall toolCall = new AssistantMessage.ToolCall( + "call_a", "function", "query_secret", "{}"); + java.util.List events = new java.util.ArrayList<>(); + java.util.List directOutputs = new java.util.ArrayList<>(); + + ToolResponseMessage.ToolResponse response = executor.executePreApproved( + toolCall, "{}", events, "conv_a", null, directOutputs); + + // Without the directOutputs collector wired, executePreApproved would + // have leaked SECRET-PAYLOAD-123 into the response. With the fix: + assertEquals(ToolExecutionExecutor.DIRECT_TOOL_PLACEHOLDER, response.responseData(), + "Pre-approved direct tool must produce a placeholder response"); + assertEquals(1, directOutputs.size()); + assertEquals("SECRET-PAYLOAD-123", directOutputs.get(0).fullResult()); + + // tool_direct_result event present + assertTrue(events.stream() + .anyMatch(e -> GraphEventPublisher.EVENT_TOOL_DIRECT_RESULT.equals(e.type()))); + } + + @Test + @DisplayName("RFC-052: legacy executePreApproved (no collector) does NOT silently leak — placeholder still applied") + void executePreApproved_legacyOverload_stillProducesPlaceholder() { + ToolCallback direct = stubCallback("query_secret", true, args -> "SECRET-OTHER-456"); + ToolExecutionExecutor executor = newExecutor(direct); + + AssistantMessage.ToolCall toolCall = new AssistantMessage.ToolCall( + "call_b", "function", "query_secret", "{}"); + java.util.List events = new java.util.ArrayList<>(); + + // 5-arg overload with no directOutputs collector — directOutputs is + // dropped on the floor, but the placeholder still keeps the LLM safe. + ToolResponseMessage.ToolResponse response = executor.executePreApproved( + toolCall, "{}", events, "conv_b", null); + + assertEquals(ToolExecutionExecutor.DIRECT_TOOL_PLACEHOLDER, response.responseData()); + assertFalse(response.responseData().contains("SECRET-OTHER-456")); + } + + @Test + @DisplayName("RFC-052: mixed batch — any direct tool triggers direct outputs while non-direct keeps result") + void mixedBatch_directAndNonDirect() { + ToolCallback direct = stubCallback("read_medical_record", true, args -> "PATIENT-DATA-XYZ"); + ToolCallback normal = stubCallback("get_weather", false, args -> "rainy, 12C"); + ToolExecutionExecutor executor = newExecutor(direct, normal); + + List calls = List.of( + new AssistantMessage.ToolCall("c1", "function", "read_medical_record", "{}"), + new AssistantMessage.ToolCall("c2", "function", "get_weather", "{}")); + + ToolExecutionExecutor.ToolExecutionResult result = + executor.execute(calls, "conv_m", "agent_m", false, "user_m", null); + + assertTrue(result.hasDirectOutputs()); + assertEquals(1, result.directOutputs().size()); + assertEquals("read_medical_record", result.directOutputs().get(0).toolName()); + assertEquals("PATIENT-DATA-XYZ", result.directOutputs().get(0).fullResult()); + + // non-direct response still contains its own data; placeholder is only on direct + assertEquals(2, result.responses().size()); + ToolResponseMessage.ToolResponse directResp = result.responses().get(0); + ToolResponseMessage.ToolResponse normalResp = result.responses().get(1); + assertEquals(ToolExecutionExecutor.DIRECT_TOOL_PLACEHOLDER, directResp.responseData()); + assertEquals("rainy, 12C", normalResp.responseData()); + } + + /** Build a minimal ToolCallback stub with an explicit returnDirect flag. */ + private static ToolCallback stubCallback(String name, boolean returnDirect, + java.util.function.Function handler) { + ToolDefinition def = ToolDefinition.builder() + .name(name) + .description("test tool " + name) + .inputSchema("{\"type\":\"object\",\"properties\":{}}") + .build(); + ToolMetadata md = ToolMetadata.builder().returnDirect(returnDirect).build(); + return new ToolCallback() { + @Override public ToolDefinition getToolDefinition() { return def; } + @Override public ToolMetadata getToolMetadata() { return md; } + @Override public String call(String arguments) { return handler.apply(arguments); } + @Override public String call(String arguments, ToolContext toolContext) { + return handler.apply(arguments); + } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillAutoRedirectTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillAutoRedirectTest.java new file mode 100644 index 00000000..51d07bab --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillAutoRedirectTest.java @@ -0,0 +1,185 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; +import vip.mate.agent.AgentToolSet; +import vip.mate.skill.runtime.SkillRuntimeService; +import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.tool.guard.ToolGuard; +import vip.mate.tool.guard.ToolGuardResult; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Auto-redirect: when the LLM mistakenly calls a skill name as if it were + * a tool, the executor should transparently invoke {@code readSkillFile} + * on its behalf and return the SKILL.md content as the tool result. + * + *

Why: smaller models (qwen-turbo et al.) often can't act on a + * "this is a Skill, not a Tool — go read X first" textual hint. They + * generate a polite "let me get that" reply and end the turn without + * any further tool call, leaving the user stuck. With auto-redirect + * the model receives runnable instructions on its very first attempt. + * + *

The hint-only path is still preserved for the case where + * {@code readSkillFile} isn't bound to the agent (covered by + * {@link ToolExecutionExecutorSkillHintTest}). + */ +class ToolExecutionExecutorSkillAutoRedirectTest { + + private static final String SKILL_MD = + "---\nname: tencent-meeting-mcp\n---\n\n# Quick start\nrunSkillScript scripts/setup.sh\n"; + + private ToolExecutionExecutor newExecutor(ToolCallback... callbacks) { + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(callbacks)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + return new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + } + + private SkillRuntimeService skillRuntimeWith(String... activeNames) { + SkillRuntimeService svc = mock(SkillRuntimeService.class); + List skills = java.util.Arrays.stream(activeNames).map(name -> { + ResolvedSkill s = mock(ResolvedSkill.class); + when(s.getName()).thenReturn(name); + return s; + }).toList(); + when(svc.getActiveSkills()).thenReturn(skills); + return svc; + } + + /** Captures the args that the auto-redirected readSkillFile receives. */ + private static class CapturingReadSkillFile { + final AtomicReference lastArgs = new AtomicReference<>(); + final ToolCallback callback; + + CapturingReadSkillFile(String returnContent) { + ToolDefinition def = ToolDefinition.builder() + .name("readSkillFile") + .description("test stub") + .inputSchema("{\"type\":\"object\",\"properties\":{}}") + .build(); + ToolMetadata md = ToolMetadata.builder().returnDirect(false).build(); + callback = new ToolCallback() { + @Override public ToolDefinition getToolDefinition() { return def; } + @Override public ToolMetadata getToolMetadata() { return md; } + @Override public String call(String arguments) { + lastArgs.set(arguments); + return returnContent; + } + @Override public String call(String arguments, ToolContext ctx) { + return call(arguments); + } + }; + } + } + + @Test + @DisplayName("skill-as-tool call gets auto-redirected to readSkillFile when the tool is bound") + void skillCallAutoRedirects() { + CapturingReadSkillFile rsf = new CapturingReadSkillFile(SKILL_MD); + ToolExecutionExecutor executor = newExecutor(rsf.callback); + executor.setSkillRuntimeService(skillRuntimeWith("tencent-meeting-mcp")); + + String llmArgs = "{\"action\":\"create\",\"subject\":\"AI讨论会\"}"; + var result = executor.execute( + List.of(new AssistantMessage.ToolCall( + "call_1", "function", "tencent-meeting-mcp", llmArgs)), + "conv", "agent", false, "user", null); + + assertEquals(1, result.responses().size()); + String response = result.responses().get(0).responseData(); + + // (1) readSkillFile was invoked with the skill's name and SKILL.md + String forwarded = rsf.lastArgs.get(); + assertNotNull(forwarded, "readSkillFile must have been invoked transparently"); + assertTrue(forwarded.contains("\"skillName\":\"tencent-meeting-mcp\""), forwarded); + assertTrue(forwarded.contains("\"filePath\":\"SKILL.md\""), forwarded); + + // (2) Response carries the SKILL.md content + assertTrue(response.contains("# Quick start"), + "Response should embed SKILL.md content: " + response); + assertTrue(response.contains("runSkillScript scripts/setup.sh"), + "Response should embed the runnable example from SKILL.md"); + + // (3) Response carries the [auto-redirect] nudge so the LLM understands + // why it didn't get a function-call result of the shape it expected + assertTrue(response.contains("[auto-redirect]"), + "Response should declare the auto-redirect: " + response); + assertTrue(response.contains("runSkillScript"), + "Response should tell the LLM what to call next"); + + // (4) Original payload is echoed back so the LLM doesn't have to re-derive + // args before calling runSkillScript + assertTrue(response.contains("AI讨论会"), + "Original LLM args should be echoed in the redirect: " + response); + } + + @Test + @DisplayName("skill-as-tool call falls through to hint when readSkillFile is NOT bound to this agent") + void skillCallWithoutReadSkillFileFallsThroughToHint() { + // Empty tool set — readSkillFile not registered for this agent + ToolExecutionExecutor executor = newExecutor(); + executor.setSkillRuntimeService(skillRuntimeWith("tencent-meeting-mcp")); + + var result = executor.execute( + List.of(new AssistantMessage.ToolCall( + "call_2", "function", "tencent-meeting-mcp", "{}")), + "conv", "agent", false, "user", null); + + String response = result.responses().get(0).responseData(); + assertTrue(response.contains("Skill, not a Tool"), + "Without readSkillFile, executor must fall back to the textual hint: " + response); + assertFalse(response.contains("[auto-redirect]"), + "No redirect should have happened: " + response); + } + + @Test + @DisplayName("non-skill unknown tool name still produces the bare 'Tool not found' message") + void unknownToolKeepsBareError() { + CapturingReadSkillFile rsf = new CapturingReadSkillFile(SKILL_MD); + ToolExecutionExecutor executor = newExecutor(rsf.callback); + executor.setSkillRuntimeService(skillRuntimeWith("tencent-meeting-mcp")); + + var result = executor.execute( + List.of(new AssistantMessage.ToolCall( + "call_3", "function", "made_up_tool", "{}")), + "conv", "agent", false, "user", null); + + String response = result.responses().get(0).responseData(); + assertEquals("Tool not found: made_up_tool", response); + assertNull(rsf.lastArgs.get(), + "readSkillFile must NOT be invoked for non-skill names"); + } + + @Test + @DisplayName("skill name with special chars in the LLM args is JSON-escaped before forwarding") + void specialCharsInArgsAreEscaped() { + CapturingReadSkillFile rsf = new CapturingReadSkillFile(SKILL_MD); + ToolExecutionExecutor executor = newExecutor(rsf.callback); + // Skill name with double quotes / backslash to verify the inline JSON + // we build for the readSkillFile call escapes them properly. + executor.setSkillRuntimeService(skillRuntimeWith("weird\"name\\skill")); + + var result = executor.execute( + List.of(new AssistantMessage.ToolCall( + "call_4", "function", "weird\"name\\skill", "{}")), + "conv", "agent", false, "user", null); + + // If escaping were broken, readSkillFile would have rejected the + // malformed JSON and returned an error. The response carrying SKILL_MD + // proves the forwarded args parsed cleanly. + assertTrue(result.responses().get(0).responseData().contains("# Quick start")); + String forwarded = rsf.lastArgs.get(); + assertTrue(forwarded.contains("weird\\\"name\\\\skill"), + "Forwarded args should JSON-escape quotes and backslashes: " + forwarded); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillHintTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillHintTest.java new file mode 100644 index 00000000..0facac70 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolExecutionExecutorSkillHintTest.java @@ -0,0 +1,122 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.tool.ToolCallback; +import vip.mate.agent.AgentToolSet; +import vip.mate.skill.runtime.SkillRuntimeService; +import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.tool.guard.ToolGuard; +import vip.mate.tool.guard.ToolGuardResult; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Issue #46: when the LLM mis-calls a skill name as a tool, the executor + * should return a precise hint explaining that the name is a Skill (not a + * Tool) and how to invoke it via {@code readSkillFile} — instead of the + * dead-end "Tool not found" string that gave the model nothing to act on. + */ +class ToolExecutionExecutorSkillHintTest { + + private ToolExecutionExecutor newExecutor(ToolCallback... callbacks) { + AgentToolSet toolSet = AgentToolSet.fromCallbacks(List.of(), List.of(callbacks)); + ToolGuard alwaysAllow = (n, a) -> ToolGuardResult.allow(); + return new ToolExecutionExecutor(toolSet, alwaysAllow, null, null); + } + + private SkillRuntimeService skillRuntimeWith(String... activeNames) { + SkillRuntimeService svc = mock(SkillRuntimeService.class); + List skills = java.util.Arrays.stream(activeNames).map(name -> { + ResolvedSkill s = mock(ResolvedSkill.class); + when(s.getName()).thenReturn(name); + return s; + }).toList(); + when(svc.getActiveSkills()).thenReturn(skills); + return svc; + } + + @Test + @DisplayName("issue#46: tool name matching an active skill yields skill-aware hint") + void unknownToolMatchingSkill_returnsHint() { + ToolExecutionExecutor executor = newExecutor(); // empty tool set + executor.setSkillRuntimeService(skillRuntimeWith("RedisOps", "browser_cdp")); + + ToolExecutionExecutor.ToolExecutionResult result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_1", "function", "RedisOps", "{}")), + "conv", "agent", false, "user", null); + + assertEquals(1, result.responses().size()); + String response = result.responses().get(0).responseData(); + assertTrue(response.contains("Skill, not a Tool"), + "Response should declare the name is a Skill: " + response); + assertTrue(response.contains("readSkillFile(skillName=\"RedisOps\""), + "Response should suggest the concrete invocation: " + response); + assertFalse(response.equals("Tool not found: RedisOps"), + "Response should NOT fall back to the bare error string"); + } + + @Test + @DisplayName("issue#46: case-insensitive skill match — LLMs sometimes alter casing") + void unknownToolCaseInsensitiveSkillMatch_returnsHint() { + ToolExecutionExecutor executor = newExecutor(); + executor.setSkillRuntimeService(skillRuntimeWith("RedisOps")); + + ToolExecutionExecutor.ToolExecutionResult result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_2", "function", "redisops", "{}")), + "conv", "agent", false, "user", null); + + String response = result.responses().get(0).responseData(); + assertTrue(response.contains("Skill, not a Tool"), + "Lowercase 'redisops' should still match active skill 'RedisOps': " + response); + } + + @Test + @DisplayName("issue#46: tool name not matching any skill keeps the bare 'Tool not found' message") + void unknownToolWithNoSkillMatch_keepsBareError() { + ToolExecutionExecutor executor = newExecutor(); + executor.setSkillRuntimeService(skillRuntimeWith("RedisOps", "browser_cdp")); + + ToolExecutionExecutor.ToolExecutionResult result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_3", "function", "totally_made_up_tool", "{}")), + "conv", "agent", false, "user", null); + + String response = result.responses().get(0).responseData(); + assertEquals("Tool not found: totally_made_up_tool", response, + "When the name doesn't match any skill, the executor must fall back to the bare error"); + } + + @Test + @DisplayName("issue#46: when skillRuntimeService is unset (legacy/test path), behavior is unchanged") + void unknownToolWithoutSkillRuntime_keepsBareError() { + ToolExecutionExecutor executor = newExecutor(); + // intentionally do NOT call setSkillRuntimeService + + ToolExecutionExecutor.ToolExecutionResult result = executor.execute( + List.of(new AssistantMessage.ToolCall("call_4", "function", "RedisOps", "{}")), + "conv", "agent", false, "user", null); + + String response = result.responses().get(0).responseData(); + assertEquals("Tool not found: RedisOps", response, + "Without a wired SkillRuntimeService, the executor must keep the legacy bare error"); + } + + @Test + @DisplayName("issue#46: pre-approved replay path also gets the skill-aware hint") + void preApprovedReplayUnknownTool_returnsHint() { + ToolExecutionExecutor executor = newExecutor(); + executor.setSkillRuntimeService(skillRuntimeWith("RedisOps")); + + java.util.List events = new java.util.ArrayList<>(); + var response = executor.executePreApproved( + new AssistantMessage.ToolCall("call_5", "function", "RedisOps", "{}"), + "{}", events, "conv", null); + + assertTrue(response.responseData().contains("Skill, not a Tool"), + "Pre-approved replay should also produce the skill-aware hint: " + response.responseData()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolResultStorageRetentionTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolResultStorageRetentionTest.java new file mode 100644 index 00000000..d2132e97 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/executor/ToolResultStorageRetentionTest.java @@ -0,0 +1,161 @@ +package vip.mate.agent.graph.executor; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.FileTime; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Retention sweep + per-conversation purge for {@link ToolResultStorage}. + * + *

The store keeps a per-JVM "observed roots" registry — every time a + * spill resolves a directory, that directory is remembered so the + * retention sweep can reach it even after the workspace path has gone + * out of scope. These tests verify the registry behaviour, the + * mtime-based deletion contract, and the targeted per-conversation purge + * called from {@code ConversationService.deleteConversation}. + */ +class ToolResultStorageRetentionTest { + + @Test + void successfulSpillRegistersTheRoot(@TempDir Path tempDir) { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 7); + + // Trigger a spill so resolveBaseDir() is invoked. + String out = storage.persistIfOversized( + "x".repeat(500), "web_search", "call-1", "conv-A", tempDir.toString()); + assertTrue(out.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)); + + assertTrue(storage.getObservedRoots().contains(tempDir), + "successful spill must register its resolved root for later cleanup"); + } + + @Test + void cleanupDeletesFilesOlderThanRetention(@TempDir Path tempDir) throws Exception { + // retention = 1 day + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 1); + + // Drop a "fresh" spill via the public API. + String fresh = storage.persistIfOversized( + "fresh".repeat(200), "web_search", "call-fresh", "conv-A", tempDir.toString()); + assertTrue(fresh.startsWith(ToolResultStorage.SPILL_MARKER_PREFIX)); + Path freshFile = pathFromPreview(fresh); + + // Drop a "stale" spill and back-date its mtime by 8 days. + String stale = storage.persistIfOversized( + "stale".repeat(200), "web_search", "call-stale", "conv-B", tempDir.toString()); + Path staleFile = pathFromPreview(stale); + Files.setLastModifiedTime(staleFile, + FileTime.from(Instant.now().minus(8, ChronoUnit.DAYS))); + + int deleted = storage.cleanupExpired(); + + assertEquals(1, deleted, "only the stale file should be removed"); + assertTrue(Files.exists(freshFile), "fresh file must survive"); + assertFalse(Files.exists(staleFile), "stale file must be deleted"); + } + + @Test + void cleanupIsNoOpWhenRetentionDisabled(@TempDir Path tempDir) throws Exception { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 0); + + String stale = storage.persistIfOversized( + "stale".repeat(200), "web_search", "call-1", "conv-A", tempDir.toString()); + Path staleFile = pathFromPreview(stale); + Files.setLastModifiedTime(staleFile, + FileTime.from(Instant.now().minus(100, ChronoUnit.DAYS))); + + int deleted = storage.cleanupExpired(); + assertEquals(0, deleted, "retentionDays<=0 must disable the sweep entirely"); + assertTrue(Files.exists(staleFile), "stale file must remain when sweep is disabled"); + } + + @Test + void cleanupRemovesEmptiedConversationDirectories(@TempDir Path tempDir) throws Exception { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 1); + + String stale = storage.persistIfOversized( + "stale".repeat(200), "web_search", "call-stale", "conv-old", tempDir.toString()); + Path staleFile = pathFromPreview(stale); + Path staleDir = staleFile.getParent(); + Files.setLastModifiedTime(staleFile, + FileTime.from(Instant.now().minus(8, ChronoUnit.DAYS))); + + storage.cleanupExpired(); + + assertFalse(Files.exists(staleFile)); + assertFalse(Files.exists(staleDir), + "the empty conv-old/ directory should be cleaned up too"); + } + + @Test + void purgeConversationDeletesAllFilesForOneConversation(@TempDir Path tempDir) throws Exception { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 30); + + // Two spills for conv-A, one spill for conv-B. + String a1 = storage.persistIfOversized( + "a1".repeat(200), "web_search", "call-a1", "conv-A", tempDir.toString()); + String a2 = storage.persistIfOversized( + "a2".repeat(200), "web_search", "call-a2", "conv-A", tempDir.toString()); + String b1 = storage.persistIfOversized( + "b1".repeat(200), "web_search", "call-b1", "conv-B", tempDir.toString()); + Path af1 = pathFromPreview(a1); + Path af2 = pathFromPreview(a2); + Path bf1 = pathFromPreview(b1); + + int deleted = storage.purgeConversation("conv-A"); + + assertEquals(2, deleted, "both A files should be deleted"); + assertFalse(Files.exists(af1)); + assertFalse(Files.exists(af2)); + assertTrue(Files.exists(bf1), "conv-B files must not be touched by a conv-A purge"); + } + + @Test + void purgeConversationIsSilentForUnknownConversation(@TempDir Path tempDir) { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 7); + + // No spill at all → nothing to purge → 0, no exception. + int deleted = storage.purgeConversation("never-existed"); + assertEquals(0, deleted); + } + + @Test + void purgeConversationHandlesBlankIdSafely(@TempDir Path tempDir) { + ToolResultStorage storage = newStorage(tempDir, /*threshold*/ 100, /*retention*/ 7); + + assertEquals(0, storage.purgeConversation(null)); + assertEquals(0, storage.purgeConversation("")); + } + + // ------------------------------------------------------------------ helpers + + private static ToolResultStorage newStorage(Path tempDir, int threshold, int retentionDays) { + ToolResultProperties props = new ToolResultProperties(); + props.setStorageBaseDir(tempDir.toString()); + props.setPerResultThresholdChars(threshold); + props.setPreviewHeadChars(80); + props.setRetentionDays(retentionDays); + props.setExcludedTools(List.of()); + return new ToolResultStorage(props); + } + + private static Path pathFromPreview(String preview) { + java.util.regex.Matcher m = java.util.regex.Pattern.compile("path=(\\S+)").matcher(preview); + assertTrue(m.find(), "preview must include path=..."); + Path p = Path.of(m.group(1)); + assertNotNull(p); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/FinalAnswerNodeTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/FinalAnswerNodeTest.java index aa0e0cf4..83a25979 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/FinalAnswerNodeTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/FinalAnswerNodeTest.java @@ -4,7 +4,12 @@ import com.alibaba.cloud.ai.graph.OverAllState; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import vip.mate.agent.GraphEventPublisher; +import vip.mate.agent.graph.state.DirectToolOutput; +import vip.mate.agent.graph.state.SourceEvidenceLedger; +import vip.mate.tool.document.GeneratedFileCache; +import java.util.List; import java.util.Map; import static org.junit.jupiter.api.Assertions.*; @@ -90,4 +95,396 @@ class FinalAnswerNodeTest { assertEquals("Failed to generate a response, please retry.", result.get(FINAL_ANSWER)); assertEquals("error_fallback", result.get(FINISH_REASON)); } + + // ========== RFC-052 returnDirect ========== + + @Test + @DisplayName("RFC-052: single direct tool output becomes the final answer verbatim") + void directSingle_verbatim() throws Exception { + DirectToolOutput out = new DirectToolOutput( + "call_1", "query_employee_salary", + "Alice's salary is 12345.", System.currentTimeMillis()); + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + DIRECT_TOOL_OUTPUTS, List.of(out) + )); + + Map result = node.apply(state); + + assertEquals("Alice's salary is 12345.", result.get(FINAL_ANSWER), + "Direct tool result must reach the user verbatim, no LLM rewriting"); + assertEquals("return_direct", result.get(FINISH_REASON)); + } + + @Test + @DisplayName("RFC-052: multiple direct outputs are joined with tool-name headings") + void directMultiple_joinedWithHeadings() throws Exception { + DirectToolOutput a = new DirectToolOutput( + "call_1", "tool_a", "result_a", System.currentTimeMillis()); + DirectToolOutput b = new DirectToolOutput( + "call_2", "tool_b", "result_b", System.currentTimeMillis()); + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + DIRECT_TOOL_OUTPUTS, List.of(a, b) + )); + + Map result = node.apply(state); + + String answer = (String) result.get(FINAL_ANSWER); + assertNotNull(answer); + assertTrue(answer.startsWith("### tool_a\nresult_a"), + "first heading + body should appear at the top"); + assertTrue(answer.contains("### tool_b\nresult_b"), + "second heading + body should follow"); + assertEquals("return_direct", result.get(FINISH_REASON)); + } + + @Test + @DisplayName("RFC-052: trigger flag without outputs falls through to default assembly") + void directTriggerEmpty_fallsThrough() throws Exception { + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + FINAL_ANSWER, "fallback content" + )); + + Map result = node.apply(state); + + // Without DIRECT_TOOL_OUTPUTS we should NOT short-circuit to RETURN_DIRECT — + // the existing FINAL_ANSWER path handles it as NORMAL. + assertEquals("fallback content", result.get(FINAL_ANSWER)); + assertEquals("normal", result.get(FINISH_REASON)); + } + + @Test + @DisplayName("RFC-052: direct path takes precedence over draft / existing answer / approval") + void directBranch_highestPriority() throws Exception { + DirectToolOutput out = new DirectToolOutput( + "call_1", "tool_x", "direct text", System.currentTimeMillis()); + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + DIRECT_TOOL_OUTPUTS, List.of(out), + FINAL_ANSWER, "must be ignored", + FINAL_ANSWER_DRAFT, "must also be ignored" + )); + + Map result = node.apply(state); + + assertEquals("direct text", result.get(FINAL_ANSWER)); + assertEquals("return_direct", result.get(FINISH_REASON)); + } + + @Test + @DisplayName("源码证据不足时降级 finishReason 并提示未验证引用") + void unsupportedSourceReferencesBecomeEvidenceInsufficient() throws Exception { + SourceEvidenceLedger ledger = SourceEvidenceLedger.empty() + .withSourcePath("src/main/java/vip/mate/skill/SkillController.java"); + OverAllState state = new OverAllState(Map.of( + SOURCE_EVIDENCE_LEDGER, ledger, + FINAL_ANSWER, "SkillController.java 是入口,SkillServiceImpl.java 负责业务逻辑。" + )); + + Map result = node.apply(state); + + assertEquals("evidence_insufficient", result.get(FINISH_REASON)); + assertTrue(((String) result.get(FINAL_ANSWER)).contains("SkillServiceImpl.java")); + assertTrue(((String) result.get(FINAL_ANSWER)).contains("证据不足")); + } + + // ========== finish_reason GraphEvent (P1: must ride PENDING_EVENTS, not SSE bypass) ========== + + /** + * Pull the {@code finish_reason} GraphEvent attached to a node output. + * Returns null when no such event was emitted. + */ + @SuppressWarnings("unchecked") + private static GraphEventPublisher.GraphEvent pickFinishReasonEvent(Map output) { + Object raw = output.get(PENDING_EVENTS); + if (!(raw instanceof List list)) return null; + for (Object item : list) { + if (item instanceof GraphEventPublisher.GraphEvent ev + && GraphEventPublisher.EVENT_FINISH_REASON.equals(ev.type())) { + return ev; + } + } + return null; + } + + @Test + @DisplayName("normal path emits finish_reason GraphEvent on PENDING_EVENTS so the accumulator can persist it") + void normalPath_emitsFinishReasonEvent() throws Exception { + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "正常回答" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFinishReasonEvent(result); + assertNotNull(ev, "FinalAnswerNode must attach a finish_reason GraphEvent (NORMAL path)"); + assertEquals("normal", ev.data().get("reason")); + } + + @Test + @DisplayName("incomplete path also emits finish_reason GraphEvent (regression for the SSE-bypass bug)") + void incompletePath_emitsFinishReasonEvent() throws Exception { + // Simulates ReasoningNode handing INCOMPLETE through to FinalAnswerNode + // (e.g. repetition-truncated partial). The earlier fix wired this via + // streamTracker.broadcastObject which was an SSE-only bypass — the + // accumulator never saw it. Now it MUST ride PENDING_EVENTS. + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "已经流式输出的部分内容…", + FINISH_REASON, "incomplete" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFinishReasonEvent(result); + assertNotNull(ev, "INCOMPLETE finish_reason must reach the channel via PENDING_EVENTS"); + assertEquals("incomplete", ev.data().get("reason")); + } + + @Test + @DisplayName("RFC-052 RETURN_DIRECT path emits finish_reason GraphEvent") + void returnDirectPath_emitsFinishReasonEvent() throws Exception { + DirectToolOutput out = new DirectToolOutput( + "call_1", "tool_x", "direct text", System.currentTimeMillis()); + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + DIRECT_TOOL_OUTPUTS, List.of(out) + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFinishReasonEvent(result); + assertNotNull(ev, "RETURN_DIRECT path must attach a finish_reason GraphEvent"); + assertEquals("return_direct", ev.data().get("reason")); + } + + @Test + @DisplayName("AWAITING_APPROVAL path emits finish_reason GraphEvent (NORMAL while paused)") + void awaitingApprovalPath_emitsFinishReasonEvent() throws Exception { + OverAllState state = new OverAllState(Map.of( + AWAITING_APPROVAL, true, + STREAMED_CONTENT, "我现在要做 X 操作。" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFinishReasonEvent(result); + assertNotNull(ev, "AWAITING_APPROVAL path must attach a finish_reason GraphEvent"); + assertEquals("normal", ev.data().get("reason"), + "Approval pause is treated as a normal pause; the resolved decision will emit a fresh event on replay"); + } + + @Test + @DisplayName("evidence_insufficient path emits finish_reason GraphEvent with the downgraded reason") + void evidenceInsufficientPath_emitsFinishReasonEvent() throws Exception { + SourceEvidenceLedger ledger = SourceEvidenceLedger.empty() + .withSourcePath("src/main/java/vip/mate/skill/SkillController.java"); + OverAllState state = new OverAllState(Map.of( + SOURCE_EVIDENCE_LEDGER, ledger, + FINAL_ANSWER, "SkillController.java 是入口,SkillServiceImpl.java 负责业务逻辑。" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFinishReasonEvent(result); + assertNotNull(ev); + assertEquals("evidence_insufficient", ev.data().get("reason"), + "Downgraded finishReason must surface in the GraphEvent, not the original NORMAL"); + } + + // ========== fake-URL guard ========== + // + // Without the guard, a hallucinated /api/v1/files/generated/{uuid} URL + // surfaces verbatim to every channel — IM clients render a clickable + // link that 404s, and users save the 404 HTML body as a .docx which + // they then report as "corrupted file". Putting the guard at the + // FinalAnswerNode terminal means EVERY channel (Web SSE, Slack, + // DingTalk, WeCom, Telegram, …) sees the same scrubbed text. + + @Test + @DisplayName("fake-URL guard: hallucinated generated-file URL → user-visible warning") + void fakeUrl_replacedWithWarning() throws Exception { + FinalAnswerNode guarded = new FinalAnswerNode(new GeneratedFileCache()); + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "您的文档已生成: /api/v1/files/generated/a1b2c3d4-e5f6-7890-abcd-ef1234567890" + )); + + Map result = guarded.apply(state); + + String answer = (String) result.get(FINAL_ANSWER); + assertFalse(answer.contains("/api/v1/files/generated/"), + "fake URL must not survive in the persisted answer; got: " + answer); + assertTrue(answer.contains(GeneratedFileCache.MISSING_REFERENCE_NOTICE), + "user-visible warning must appear in place of the fake URL; got: " + answer); + } + + @Test + @DisplayName("fake-URL guard: real cached URL passes through so channel adapters can rewrite it") + void realUrl_leftIntact() throws Exception { + GeneratedFileCache cache = new GeneratedFileCache(); + String id = cache.put("real-bytes".getBytes(), "report.pdf", "application/pdf"); + FinalAnswerNode guarded = new FinalAnswerNode(cache); + + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "下载: /api/v1/files/generated/" + id + )); + + Map result = guarded.apply(state); + + // Cached URLs survive verbatim — downstream WeCom / Slack / etc. + // adapters can still rewrite them into native attachments. + assertTrue(((String) result.get(FINAL_ANSWER)) + .contains("/api/v1/files/generated/" + id), + "live cached URL must pass through for downstream native-attachment rewrite"); + } + + @Test + @DisplayName("fake-URL guard: also fires on RETURN_DIRECT path (tool output may also hallucinate)") + void fakeUrl_scrubbedOnDirectPath() throws Exception { + FinalAnswerNode guarded = new FinalAnswerNode(new GeneratedFileCache()); + DirectToolOutput out = new DirectToolOutput( + "call_1", "tool_x", + "see /api/v1/files/generated/never-rendered-uuid", + System.currentTimeMillis()); + OverAllState state = new OverAllState(Map.of( + RETURN_DIRECT_TRIGGERED, true, + DIRECT_TOOL_OUTPUTS, List.of(out) + )); + + Map result = guarded.apply(state); + + String answer = (String) result.get(FINAL_ANSWER); + assertFalse(answer.contains("never-rendered-uuid"), + "RETURN_DIRECT path must also scrub; got: " + answer); + } + + @Test + @DisplayName("fake-URL guard: no-cache constructor (legacy callers, narrow tests) is a no-op") + void noCache_passThrough() throws Exception { + // FinalAnswerNode without an injected cache must not throw — the + // narrow unit tests that construct the node with the no-arg ctor + // still need to work. The trade-off: tests that don't exercise + // file outputs simply skip the scrub. Production wiring always + // passes a real cache from AgentGraphBuilder. + FinalAnswerNode unguarded = new FinalAnswerNode(); + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "/api/v1/files/generated/anything" + )); + Map result = unguarded.apply(state); + assertEquals("/api/v1/files/generated/anything", result.get(FINAL_ANSWER)); + } + + // ========== feedback_event recovery affordance ========== + // + // After NodeStreamingChatHelper has exhausted its TLS/IO retry budget + // and the turn ends in ERROR_FALLBACK, the user is left staring at + // red "[错误] …" text with no recovery affordance. FinalAnswerNode + // attaches a feedback_event GraphEvent so the frontend can render + // retry/regenerate/report buttons next to the failed bubble — and + // the event is persisted into message metadata so a page reload + // doesn't make the affordance vanish. + + /** Pull the feedback_event GraphEvent attached to a node output. */ + @SuppressWarnings("unchecked") + private static GraphEventPublisher.GraphEvent pickFeedbackEvent(Map output) { + Object raw = output.get(PENDING_EVENTS); + if (!(raw instanceof List list)) return null; + for (Object item : list) { + if (item instanceof GraphEventPublisher.GraphEvent ev + && GraphEventPublisher.EVENT_FEEDBACK.equals(ev.type())) { + return ev; + } + } + return null; + } + + @Test + @DisplayName("ERROR_FALLBACK turn emits feedback_event with retry/regenerate/report actions") + void errorFallback_emitsFeedbackEvent() throws Exception { + // Mirrors the production path: ReasoningNode hands a fatal-error + // finalAnswer + ERROR_FALLBACK finishReason to FinalAnswerNode. + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "[错误] LLM 调用失败: bad_record_mac", + FINISH_REASON, "error_fallback" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent ev = pickFeedbackEvent(result); + assertNotNull(ev, "ERROR_FALLBACK turn must attach a feedback_event for the UI"); + assertEquals("ERROR_FALLBACK", ev.data().get("errorType")); + assertEquals("[错误] LLM 调用失败: bad_record_mac", ev.data().get("errorMessage")); + Object actions = ev.data().get("actions"); + assertTrue(actions instanceof List); + assertEquals(List.of("retry", "regenerate", "report"), actions); + } + + @Test + @DisplayName("ERROR_FALLBACK still emits the standard finish_reason event alongside feedback_event") + void errorFallback_alsoEmitsFinishReason() throws Exception { + // The two events ride the same PENDING_EVENTS list. Existing + // consumers (memory gate, channel accumulator, message metadata + // persistence) read finish_reason; the new feedback_event is + // additive — losing finish_reason here would silently break + // those consumers. + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "[错误] 认证失败: Invalid API Key", + FINISH_REASON, "error_fallback" + )); + + Map result = node.apply(state); + + GraphEventPublisher.GraphEvent fr = pickFinishReasonEvent(result); + assertNotNull(fr, "finish_reason event must remain on PENDING_EVENTS"); + assertEquals("error_fallback", fr.data().get("reason")); + + GraphEventPublisher.GraphEvent fb = pickFeedbackEvent(result); + assertNotNull(fb, "feedback_event must coexist with finish_reason on the same output"); + } + + @Test + @DisplayName("NORMAL turn does NOT emit feedback_event (no recovery affordance needed)") + void normalTurn_noFeedbackEvent() throws Exception { + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "正常回答" + )); + + Map result = node.apply(state); + + assertNull(pickFeedbackEvent(result), + "Successful turns must not attach feedback_event — would render misleading retry buttons"); + } + + @Test + @DisplayName("INCOMPLETE turn does NOT emit feedback_event (handled by its own card)") + void incompleteTurn_noFeedbackEvent() throws Exception { + // INCOMPLETE has its own dedicated UI card ("regenerate" button + // wired via finishReason=incomplete). Adding feedback_event there + // would duplicate the affordance and confuse users. + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "已经流式输出的部分内容…", + FINISH_REASON, "incomplete" + )); + + Map result = node.apply(state); + + assertNull(pickFeedbackEvent(result), + "INCOMPLETE has its own card; must not also surface feedback_event"); + } + + @Test + @DisplayName("STOPPED (user-initiated abort) does NOT emit feedback_event") + void stoppedTurn_noFeedbackEvent() throws Exception { + // User clicked stop. They don't need a "retry" prompt — the + // partial output is the explicit signal they asked for. + OverAllState state = new OverAllState(Map.of( + FINAL_ANSWER, "我刚在生成…", + FINISH_REASON, "stopped" + )); + + Map result = node.apply(state); + + assertNull(pickFeedbackEvent(result)); + } } diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/LimitExceededNodeFallbackTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/LimitExceededNodeFallbackTest.java new file mode 100644 index 00000000..58f79cfd --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/LimitExceededNodeFallbackTest.java @@ -0,0 +1,100 @@ +package vip.mate.agent.graph.node; + +import com.alibaba.cloud.ai.graph.OverAllState; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatModel; +import vip.mate.agent.graph.NodeStreamingChatHelper; +import vip.mate.agent.graph.observation.ObservationProcessor; +import vip.mate.i18n.I18nService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; +import static vip.mate.agent.graph.state.MateClawStateKeys.*; + +/** + * Verifies that {@link LimitExceededNode} surfaces fallback strings via + * {@link I18nService} (RFC: prompt-cleanup E2) instead of literal Chinese + * hardcodes. Two paths are covered: + * + *

    + *
  • Empty observation history — the inline {@code contextForLLM} + * defaults to {@code i18n.msg("agent.limit_exceeded.empty_context")}
  • + *
  • LLM returns empty text — the {@code finalAnswerDraft} fallback + * comes from {@code i18n.msg("agent.limit_exceeded.fallback")}
  • + *
+ */ +class LimitExceededNodeFallbackTest { + + private ChatModel chatModel; + private ObservationProcessor observationProcessor; + private NodeStreamingChatHelper streamingHelper; + private I18nService i18n; + + @BeforeEach + void setUp() { + chatModel = mock(ChatModel.class); + observationProcessor = mock(ObservationProcessor.class); + when(observationProcessor.getMaxTotalObservationChars()).thenReturn(24000); + when(observationProcessor.truncate(anyString(), anyInt())).thenAnswer(inv -> inv.getArgument(0)); + + streamingHelper = mock(NodeStreamingChatHelper.class); + + i18n = mock(I18nService.class); + when(i18n.msg("agent.limit_exceeded.empty_context")).thenReturn("CANNED_EMPTY_CTX"); + when(i18n.msg("agent.limit_exceeded.fallback")).thenReturn("CANNED_FALLBACK"); + } + + private LimitExceededNode createNode() { + return new LimitExceededNode(chatModel, observationProcessor, streamingHelper, i18n); + } + + @Test + @DisplayName("Empty LLM response → finalAnswerDraft uses i18n fallback (not Chinese literal)") + void emptyLlmResponse_usesI18nFallback() throws Exception { + // LLM returns null text → triggers the i18n fallback branch. + NodeStreamingChatHelper.StreamResult result = new NodeStreamingChatHelper.StreamResult( + null, "", new AssistantMessage(""), List.of(), false, 0, 0); + when(streamingHelper.streamCall(any(), any(), anyString(), anyString())).thenReturn(result); + + Map output = createNode().apply(buildStateWithObservations()); + + assertEquals("CANNED_FALLBACK", output.get(FINAL_ANSWER_DRAFT), + "finalAnswerDraft must come from i18n.msg(\"agent.limit_exceeded.fallback\") when the LLM returns nothing"); + } + + @Test + @DisplayName("Non-empty LLM response → finalAnswerDraft uses LLM text (i18n untouched)") + void nonEmptyLlmResponse_usesLlmText() throws Exception { + NodeStreamingChatHelper.StreamResult result = new NodeStreamingChatHelper.StreamResult( + "real answer", "", new AssistantMessage("real answer"), List.of(), false, 10, 5); + when(streamingHelper.streamCall(any(), any(), anyString(), anyString())).thenReturn(result); + + Map output = createNode().apply(buildStateWithObservations()); + + assertEquals("real answer", output.get(FINAL_ANSWER_DRAFT)); + } + + private OverAllState buildStateWithObservations() { + Map map = new HashMap<>(); + map.put(CONVERSATION_ID, "test-conv"); + map.put(USER_MESSAGE, "hello"); + map.put(MAX_ITERATIONS, 5); + map.put(CURRENT_ITERATION, 5); + // Non-empty observations so contextForLLM doesn't take the empty-context branch + // (that branch is exercised separately by an integration test, hard to mock here + // because OverAllState.value() may return immutable empty list defaults). + map.put(OBSERVATION_HISTORY, List.of("obs1", "obs2")); + return new OverAllState(map); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeOutputTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeOutputTest.java index e5961f58..0ef58218 100644 --- a/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeOutputTest.java +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/node/ReasoningNodeOutputTest.java @@ -5,10 +5,14 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.tool.ToolCallback; import vip.mate.agent.AgentToolSet; import vip.mate.agent.graph.NodeStreamingChatHelper; +import vip.mate.agent.graph.state.SourceEvidenceLedger; import vip.mate.channel.web.ChatStreamTracker; import java.util.HashMap; @@ -17,6 +21,7 @@ import java.util.Map; import java.util.concurrent.CancellationException; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentCaptor.forClass; import static org.mockito.ArgumentMatchers.*; import static org.mockito.Mockito.*; import static vip.mate.agent.graph.state.MateClawStateKeys.*; @@ -98,6 +103,38 @@ class ReasoningNodeOutputTest { assertEquals("回答内容", output.get(FINAL_ANSWER)); } + @Test + @DisplayName("源码证据不足的 final answer:原文进 streamedContent,警告作为 finalAnswer 追加") + void evidenceInsufficientFinalAnswer_splitsPersistedContentAndWarning() throws Exception { + NodeStreamingChatHelper.StreamResult result = new NodeStreamingChatHelper.StreamResult( + "SkillController.java 是入口,SkillServiceImpl.java 负责业务。", "", + new AssistantMessage("SkillController.java 是入口,SkillServiceImpl.java 负责业务。"), + List.of(), false, 100, 50); + when(streamingHelper.streamCall(any(), any(), anyString(), anyString())).thenReturn(result); + Map stateMap = new HashMap<>(); + stateMap.put(CONVERSATION_ID, "test-conv"); + stateMap.put(SYSTEM_PROMPT, "you are a helper"); + stateMap.put(USER_MESSAGE, "分析源码"); + stateMap.put(MESSAGES, List.of()); + stateMap.put(CURRENT_ITERATION, 3); + stateMap.put(MAX_ITERATIONS, 10); + stateMap.put(LLM_CALL_COUNT, 5); + stateMap.put(FORCED_TOOL_CALL, ""); + stateMap.put(SOURCE_EVIDENCE_LEDGER, SourceEvidenceLedger.empty() + .withSourcePath("src/main/java/vip/mate/skill/controller/SkillController.java")); + + Map output = createNode().apply(new OverAllState(stateMap)); + + assertControlFlagsCleared(output, "evidenceInsufficientFinalAnswer"); + assertEquals("evidence_insufficient", output.get(FINISH_REASON)); + assertEquals("SkillController.java 是入口,SkillServiceImpl.java 负责业务。", + output.get(STREAMED_CONTENT)); + assertTrue(((String) output.get(FINAL_ANSWER)).contains("证据不足")); + assertTrue(((String) output.get(FINAL_ANSWER)).contains("SkillServiceImpl.java")); + assertEquals(false, output.get(CONTENT_STREAMED), + "warning suffix should be broadcast and persisted as a visible final delta"); + } + // ===== 工具调用 ===== @Test @@ -138,6 +175,33 @@ class ReasoningNodeOutputTest { assertEquals("error_fallback", output.get(FINISH_REASON)); } + @Test + @DisplayName("thinking-only no-content 路径:标 INCOMPLETE 并附带可重试提示") + void thinkingOnlyCap_preservedAsIncomplete() throws Exception { + // Simulates the "深度思考 ... 5.4k chars never finishes" symptom: + // helper disposes the stream after THINKING_ONLY_HARD_CAP_CHARS of + // reasoning_content with zero visible content/tools. ReasoningNode + // surfaces a short fallback line and preserves the thinking transcript. + String thinkingTranscript = "我先读 X,再读 Y,再读 Z…".repeat(64); + NodeStreamingChatHelper.StreamResult result = new NodeStreamingChatHelper.StreamResult( + "", thinkingTranscript, new AssistantMessage(""), + List.of(), false, 0, 600, true, "thinking_only_no_content", + NodeStreamingChatHelper.ErrorType.UNKNOWN); + when(streamingHelper.streamCall(any(), any(), anyString(), anyString())).thenReturn(result); + + Map output = createNode().apply(buildStaleState()); + + assertControlFlagsCleared(output, "thinkingOnlyCap"); + assertLlmCallCountWritten(output, "thinkingOnlyCap"); + assertEquals("incomplete", output.get(FINISH_REASON)); + String answer = (String) output.get(FINAL_ANSWER); + assertNotNull(answer); + assertTrue(answer.contains("思考阶段"), + "Fallback line should explain the thinking-only loop to the user"); + assertEquals(thinkingTranscript, output.get(FINAL_THINKING), + "Thinking transcript must be preserved for the UI's collapse panel"); + } + // ===== CancellationException (no content stop) ===== @Test diff --git a/mateclaw-server/src/test/java/vip/mate/agent/graph/state/SourceEvidenceLedgerTest.java b/mateclaw-server/src/test/java/vip/mate/agent/graph/state/SourceEvidenceLedgerTest.java new file mode 100644 index 00000000..fa154e9c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/graph/state/SourceEvidenceLedgerTest.java @@ -0,0 +1,152 @@ +package vip.mate.agent.graph.state; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.ToolResponseMessage; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class SourceEvidenceLedgerTest { + + @Test + @DisplayName("records successful read_file paths and symbols") + void recordsReadFileEvidence() { + String response = """ + { + "filePath": "/repo/src/main/java/vip/mate/skill/SkillController.java", + "totalLines": 120, + "startLine": 1, + "endLine": 80, + "content": " 1\\tpackage vip.mate.skill;\\n 2\\tpublic class SkillController { }\\n" + } + """; + + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "read_file", response))); + + assertTrue(ledger.hasPath("/repo/src/main/java/vip/mate/skill/SkillController.java")); + assertTrue(ledger.hasSymbol("SkillController")); + assertFalse(ledger.hasSymbol("SkillServiceImpl")); + } + + @Test + @DisplayName("ignores failed read_file responses") + void ignoresFailedReads() { + String response = """ + {"filePath": "/repo/Missing.java", "error": true, "message": "not found"} + """; + + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "read_file", response))); + + assertFalse(ledger.hasPath("/repo/Missing.java")); + assertTrue(ledger.failedPaths().contains("/repo/Missing.java")); + } + + @Test + @DisplayName("validates Java references in final answers against evidence") + void detectsUnsupportedAnswerReferences() { + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "execute_shell_command", + "src/main/java/vip/mate/skill/SkillController.java\n"))); + + SourceEvidenceLedger.Validation validation = ledger.validateAnswer(""" + 已确认 SkillController.java 负责接口,但 SkillServiceImpl.java 负责业务。 + """); + + assertFalse(validation.valid()); + assertTrue(validation.unsupportedReferences().contains("SkillServiceImpl.java")); + assertFalse(validation.unsupportedReferences().contains("SkillController.java")); + } + + // ====== Regression coverage for the "grep output → ledger" path ====== + // Reviewer point: JAVA_PATH already accepts bare file names, so a P2 + // "add JAVA_FILE_REF to plain text scan" would be redundant. These tests + // pin that contract so the next person doesn't try the same wrong fix. + + @Test + @DisplayName("bare .java filename in shell stdout is recorded as both path and symbol") + void recordsBareFilenameFromShellStdout() { + // Some greps / find -printf outputs emit just the filename — no path + // prefix, no `:` line marker. JAVA_PATH still matches because [+] + // demands ≥1 word/dot/slash chars, which "ObservationNode" satisfies. + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "execute_shell_command", + "ObservationNode.java\n"))); + + assertTrue(ledger.hasPath("ObservationNode.java"), + "bare filename must register under sourcePaths"); + assertTrue(ledger.hasSymbol("ObservationNode"), + "the .java stem must be auto-promoted into sourceSymbols"); + } + + @Test + @DisplayName("grep -rn output (`path:line:body`) is parsed and the file goes into ledger") + void recordsGrepDashRnOutput() { + // Real-world grep -rn output: `relative/path:lineno:matching line`. + // JAVA_PATH greedy match consumes through the .java suffix and stops + // at the colon (\\b boundary), so the path portion lands in sourcePaths. + String grepStdout = """ + src/main/java/vip/mate/agent/graph/node/ObservationNode.java:42: public class ObservationNode implements NodeAction { + src/main/java/vip/mate/agent/graph/node/ObservationNode.java:88: log.info("[Observation]"); + """; + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "execute_shell_command", grepStdout))); + + assertTrue(ledger.hasPath("src/main/java/vip/mate/agent/graph/node/ObservationNode.java")); + assertTrue(ledger.hasSymbol("ObservationNode")); + // Critical: an answer citing ObservationNode (no .java suffix) must NOT be + // flagged as evidence-insufficient on the strength of the grep alone. + // Use only this one symbol in the answer so the test isolates exactly + // what we're verifying (other *Node names in the sentence would be + // counted as separate symbol citations). + SourceEvidenceLedger.Validation validation = ledger.validateAnswer( + "ObservationNode 写回观察历史。"); + assertTrue(validation.valid(), + "Symbol named ObservationNode is supported by the grep evidence; should not be flagged"); + } + + @Test + @DisplayName("real-task regression: ObservationNode + ToolGuardAuditLogEntity grep evidence supports their citations") + void regressionForRealTraceUnsupportedRefs() { + // The exact two unsupported refs from production trace 4b38f04f: + // unsupportedReferences=[ObservationNode, ToolGuardAuditLogEntity] + // If the model had genuinely seen these names in shell results, ledger + // should have accepted them. This test simulates the grep output that + // would have appeared in a real run — if it passes, the production + // miss is NOT a JAVA_PATH parsing bug; root cause must be elsewhere + // (spill / compact dropping the matching lines before ActionNode + // builds the ledger). + String evidence = """ + src/main/java/vip/mate/agent/graph/node/ObservationNode.java + src/main/java/vip/mate/tool/guard/entity/ToolGuardAuditLogEntity.java + """; + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "execute_shell_command", evidence))); + + SourceEvidenceLedger.Validation validation = ledger.validateAnswer( + "工具结果由 ObservationNode 写回,并落库到 ToolGuardAuditLogEntity。"); + assertTrue(validation.valid(), + "Both citations must be considered supported when their .java files appear in shell output. " + + "If this fails, fix JAVA_PATH; if it passes, the production miss is in spill/compact, " + + "not in ledger parsing."); + } + + @Test + @DisplayName("citing a class with NO matching .java in any tool output is correctly flagged unsupported") + void unrelatedSymbolInAnswerIsStillFlagged() { + // Negative control for the regression test above: make sure the + // 'support' check isn't trivially over-broad — symbols that have no + // backing evidence at all must still trip evidence_insufficient. + SourceEvidenceLedger ledger = SourceEvidenceLedger.fromToolResponses(List.of( + new ToolResponseMessage.ToolResponse("c1", "execute_shell_command", + "ObservationNode.java\n"))); + + SourceEvidenceLedger.Validation validation = ledger.validateAnswer( + "ObservationNode 协作 RandomMadeUpService 完成处理。"); + assertFalse(validation.valid()); + assertTrue(validation.unsupportedReferences().contains("RandomMadeUpService")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/agent/service/TemplateServiceBindingTest.java b/mateclaw-server/src/test/java/vip/mate/agent/service/TemplateServiceBindingTest.java new file mode 100644 index 00000000..a806c45c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/agent/service/TemplateServiceBindingTest.java @@ -0,0 +1,346 @@ +package vip.mate.agent.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.agent.AgentService; +import vip.mate.agent.binding.service.AgentBindingService; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.model.TemplateDTO; +import vip.mate.exception.MateClawException; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.repository.SkillMapper; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.service.AvailableToolService; +import vip.mate.workspace.document.WorkspaceFileService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Hire-time pre-binding behavior for {@link TemplateService#applyTemplate}. + * + *

The contract being pinned: a template that declares + * {@code defaultSkillSlugs} / {@code defaultToolNames} produces an agent that + * already has those capabilities wired, and references that can't be + * resolved (slug not in this workspace, tool not in the picker) are dropped + * silently — the hire MUST still succeed so a partially-installed + * environment doesn't break onboarding. + */ +class TemplateServiceBindingTest { + + private static final long WORKSPACE = 1L; + private static final long CREATOR = 7L; + private static final long CREATED_AGENT_ID = 999L; + + private AgentService agentService; + private WorkspaceFileService workspaceFileService; + private AgentBindingService agentBindingService; + private SkillMapper skillMapper; + private AvailableToolService availableToolService; + private TemplateService service; + private TemplateService spyService; + + @BeforeEach + void setUp() { + agentService = mock(AgentService.class); + workspaceFileService = mock(WorkspaceFileService.class); + agentBindingService = mock(AgentBindingService.class); + skillMapper = mock(SkillMapper.class); + availableToolService = mock(AvailableToolService.class); + + // createAgent stamps an id and echoes the entity back, matching the + // real DAO contract the production code relies on. + when(agentService.createAgent(any(AgentEntity.class))).thenAnswer(inv -> { + AgentEntity a = inv.getArgument(0); + a.setId(CREATED_AGENT_ID); + return a; + }); + + service = new TemplateService( + agentService, + workspaceFileService, + new ObjectMapper(), + agentBindingService, + skillMapper, + availableToolService); + spyService = spy(service); + } + + /** Build a minimal template; tests append bind lists. */ + private TemplateDTO baseTemplate(String id) { + TemplateDTO t = new TemplateDTO(); + t.setId(id); + t.setName(id); + t.setDescription("test"); + t.setAgentType("react"); + t.setMaxIterations(10); + t.setSystemPrompt("## Role\ntest"); + return t; + } + + /** Stub the in-memory template registry so the test owns the data. */ + private void registerTemplate(TemplateDTO template) { + doReturn(List.of(template)).when(spyService).listTemplates(); + } + + private SkillEntity skillRow(long id, String slug) { + SkillEntity s = new SkillEntity(); + s.setId(id); + s.setName(slug); + s.setWorkspaceId(WORKSPACE); + return s; + } + + private AvailableToolDTO availableTool(String name) { + AvailableToolDTO dto = new AvailableToolDTO(); + dto.setName(name); + dto.setAvailable(true); + return dto; + } + + @Test + @DisplayName("declared skill slugs resolve to ids and pre-bind on the new agent") + void preBindsDeclaredSkillSlugs() { + TemplateDTO t = baseTemplate("data-analyst-stub"); + t.setDefaultSkillSlugs(List.of("sql_query", "xlsx")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(skillRow(101L, "sql_query")) + .thenReturn(skillRow(202L, "xlsx")); + + AgentEntity created = spyService.applyTemplate("data-analyst-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + verify(agentBindingService, times(1)) + .setSkillBindings(eq(CREATED_AGENT_ID), argThat(ids -> + ids.size() == 2 && ids.contains(101L) && ids.contains(202L))); + // No tool bindings declared → no tool side-effects. + verify(agentBindingService, never()).setToolBindings(anyLong(), anyList()); + } + + @Test + @DisplayName("missing slugs are skipped without aborting the hire") + void skipsMissingSlugsAndStillHires() { + TemplateDTO t = baseTemplate("partial-stub"); + t.setDefaultSkillSlugs(List.of("ghost-skill", "sql_query")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(null) // ghost-skill not in workspace + .thenReturn(skillRow(101L, "sql_query")); + + AgentEntity created = spyService.applyTemplate("partial-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + // Only the resolvable slug makes it into the binding call. + verify(agentBindingService, times(1)) + .setSkillBindings(eq(CREATED_AGENT_ID), argThat(ids -> + ids.size() == 1 && ids.contains(101L))); + } + + @Test + @DisplayName("when every slug is unknown, setSkillBindings is never called and the agent still exists") + void noSlugsResolveSoNoBindCall() { + TemplateDTO t = baseTemplate("all-ghost-stub"); + t.setDefaultSkillSlugs(List.of("ghost-a", "ghost-b")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + + AgentEntity created = spyService.applyTemplate("all-ghost-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + // Empty resolved list → caller must NOT issue an empty + // setSkillBindings (which would otherwise wipe out future bindings + // post-create if any race wrote them in between). + verify(agentBindingService, never()).setSkillBindings(anyLong(), anyList()); + } + + @Test + @DisplayName("legacy templates with no binding fields behave as before") + void noBindingFieldsLeavesAgentUntouched() { + TemplateDTO t = baseTemplate("legacy-stub"); + // Neither defaultSkillSlugs nor defaultToolNames set. + registerTemplate(t); + + AgentEntity created = spyService.applyTemplate("legacy-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + verify(agentBindingService, never()).setSkillBindings(anyLong(), anyList()); + verify(agentBindingService, never()).setToolBindings(anyLong(), anyList()); + } + + @Test + @DisplayName("tool names are pre-filtered through the picker before binding") + void toolBindingsFilterAgainstPicker() { + TemplateDTO t = baseTemplate("tool-stub"); + t.setDefaultToolNames(List.of("search", "ghost_tool", "browser_use")); + registerTemplate(t); + + when(availableToolService.listAvailable()).thenReturn(List.of( + availableTool("search"), + availableTool("browser_use"))); + + AgentEntity created = spyService.applyTemplate("tool-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + // ghost_tool is not in the picker → filtered. The remaining two + // pass through; setToolBindings's own validator would otherwise + // throw on the unknown name and abort the entire bind call. + verify(agentBindingService, times(1)) + .setToolBindings(eq(CREATED_AGENT_ID), argThat(names -> + names.size() == 2 + && names.contains("search") + && names.contains("browser_use") + && !names.contains("ghost_tool"))); + } + + @Test + @DisplayName("picker failure during apply skips tool binding instead of breaking the hire") + void pickerFailureDoesNotBreakHire() { + TemplateDTO t = baseTemplate("picker-down-stub"); + t.setDefaultToolNames(List.of("search")); + registerTemplate(t); + + when(availableToolService.listAvailable()) + .thenThrow(new RuntimeException("MCP discovery upstream timeout")); + + AgentEntity created = spyService.applyTemplate("picker-down-stub", WORKSPACE, CREATOR, null); + + // Hire still completes; tool bind silently skipped (conservative + // stance documented on applyDefaultToolBindings). + assertEquals(CREATED_AGENT_ID, created.getId()); + verify(agentBindingService, never()).setToolBindings(anyLong(), anyList()); + } + + @Test + @DisplayName("setSkillBindings exception propagates so @Transactional rolls back the hire") + void bindServiceExceptionPropagates() { + // Pins the documented split: resolution failures are graceful, but + // service-layer exceptions (a race deleting the skill row between + // resolve and bind, a workspace-mismatch we couldn't predict) are + // fail-stop. If someone later wraps the bind call in try/catch to + // "make it more robust", this test forces them to also revisit + // applyDefaultSkillBindings's Javadoc and the @Transactional + // rollback contract instead of silently changing behavior. + TemplateDTO t = baseTemplate("racey-stub"); + t.setDefaultSkillSlugs(List.of("sql_query")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(skillRow(101L, "sql_query")); + doThrow(new MateClawException("err.skill.cross_workspace_binding", 403, "simulated race")) + .when(agentBindingService).setSkillBindings(anyLong(), anyList()); + + MateClawException thrown = assertThrows(MateClawException.class, + () -> spyService.applyTemplate("racey-stub", WORKSPACE, CREATOR, null)); + assertEquals(403, thrown.getCode()); + assertEquals("err.skill.cross_workspace_binding", thrown.getMsgKey()); + } + + @Test + @DisplayName("workspace lookup reads from the persisted agent — survives a service-side workspace override") + void workspaceLookupUsesPersistedAgent() { + // Defends against a future where AgentService.createAgent normalises + // workspaceId (auto-assign default, project-onto-user-default, etc.) + // — the slug resolver MUST query the same workspace that the bind + // validator will check. Here we mutate the persisted agent's + // workspace to a value different from the input parameter; if the + // helper still queried the parameter, the lookup would target the + // wrong workspace and (in production) miss the seeded skill. We + // can't introspect the LambdaQueryWrapper's parameter map from a + // Mockito-only test (MyBatis-Plus lambda cache isn't bootstrapped), + // so this test pins the flow against crashes; the workspace-source + // correctness is enforced by code review on the helper itself. + when(agentService.createAgent(any(AgentEntity.class))).thenAnswer(inv -> { + AgentEntity a = inv.getArgument(0); + a.setId(CREATED_AGENT_ID); + a.setWorkspaceId(42L); + return a; + }); + + TemplateDTO t = baseTemplate("ws-override-stub"); + t.setDefaultSkillSlugs(List.of("sql_query")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(skillRow(101L, "sql_query")); + + spyService.applyTemplate("ws-override-stub", WORKSPACE /* = 1 */, CREATOR, null); + + verify(agentBindingService, times(1)) + .setSkillBindings(eq(CREATED_AGENT_ID), argThat(ids -> + ids.size() == 1 && ids.contains(101L))); + } + + @Test + @DisplayName("null workspace on the persisted agent does not crash the helper") + void nullWorkspaceFallsBackToOne() { + // Mirrors the AgentBindingService.requireSameWorkspace fallback — + // a row with workspace_id = null must not produce an `IS NULL` + // lookup that silently matches nothing. The helper falls back to + // workspace 1; without that, the LambdaQueryWrapper would still + // build but every seeded skill would miss. Smoke-tested here for + // crash-freeness; the value of the fallback (1L) is asserted by + // code review of the helper. + when(agentService.createAgent(any(AgentEntity.class))).thenAnswer(inv -> { + AgentEntity a = inv.getArgument(0); + a.setId(CREATED_AGENT_ID); + a.setWorkspaceId(null); + return a; + }); + + TemplateDTO t = baseTemplate("null-ws-stub"); + t.setDefaultSkillSlugs(List.of("sql_query")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(skillRow(101L, "sql_query")); + + spyService.applyTemplate("null-ws-stub", WORKSPACE, CREATOR, null); + + verify(agentBindingService, times(1)) + .setSkillBindings(eq(CREATED_AGENT_ID), argThat(ids -> ids.contains(101L))); + } + + @Test + @DisplayName("blank slug entries are skipped before they reach the mapper") + void blankSlugsSkipped() { + TemplateDTO t = baseTemplate("blanks-stub"); + t.setDefaultSkillSlugs(java.util.Arrays.asList("sql_query", "", null, " ")); + registerTemplate(t); + + when(skillMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(skillRow(101L, "sql_query")); + + AgentEntity created = spyService.applyTemplate("blanks-stub", WORKSPACE, CREATOR, null); + + assertNotNull(created.getId()); + // Only the one real slug triggers a mapper lookup → only one bind. + verify(skillMapper, times(1)).selectOne(any(LambdaQueryWrapper.class)); + verify(agentBindingService, times(1)) + .setSkillBindings(eq(CREATED_AGENT_ID), argThat(ids -> + ids.size() == 1 && ids.contains(101L))); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/approval/ApprovalReplayContinuityTest.java b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalReplayContinuityTest.java new file mode 100644 index 00000000..19d99525 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalReplayContinuityTest.java @@ -0,0 +1,91 @@ +package vip.mate.approval; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; +import vip.mate.agent.context.ChannelTarget; +import vip.mate.agent.context.ChatOrigin; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-063r §2.12: Memento round-trip for cross-restart approval replay. + * + *

Exercises {@link ApprovalWorkflowService#restoreChatOrigin(String)} + * directly — independent of the DB layer — to pin the serialization + * contract: full round-trip preserves every field, and a corrupt or null + * payload falls back to {@link ChatOrigin#EMPTY} rather than throwing. + */ +class ApprovalReplayContinuityTest { + + private ApprovalWorkflowService workflow; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() { + objectMapper = new ObjectMapper(); + // Don't run the @PostConstruct GC scheduler — only need restoreChatOrigin. + workflow = new ApprovalWorkflowService(null, null, objectMapper, null); + // Inject objectMapper via reflection so the helper does not NPE. + ReflectionTestUtils.setField(workflow, "objectMapper", objectMapper); + } + + @Test + void chatOrigin_persistedAndRestored_preservesAllFields() throws Exception { + ChatOrigin original = new ChatOrigin( + /* agentId */ 7L, + /* conversationId */ "wechat:chat-42", + /* requesterId */ "u-123", + /* workspaceId */ 5L, + /* workspaceBasePath */ "/data/ws/5", + /* channelId */ 9L, + /* channelTarget */ new ChannelTarget("group-a", "thread-1", "bot-001")); + + String json = objectMapper.writeValueAsString(original); + ChatOrigin restored = workflow.restoreChatOrigin(json); + + assertEquals(original, restored, + "Memento round-trip must preserve every field — RFC-063r §2.12"); + } + + @Test + void chatOrigin_corruptJson_fallsBackToEmpty() { + String corrupt = "{\"this is not\":valid JSON"; + ChatOrigin restored = workflow.restoreChatOrigin(corrupt); + assertSame(ChatOrigin.EMPTY, restored, + "Corrupt payload must fall back to EMPTY without throwing"); + } + + @Test + void chatOrigin_nullPayload_returnsEmpty() { + assertSame(ChatOrigin.EMPTY, workflow.restoreChatOrigin(null)); + } + + @Test + void chatOrigin_blankPayload_returnsEmpty() { + assertSame(ChatOrigin.EMPTY, workflow.restoreChatOrigin(" ")); + } + + @Test + void chatOrigin_unknownFieldsInJson_areTolerated() throws Exception { + // Forward-compat: a payload written by a future build with extra + // fields must still restore the known fields. + String json = """ + { + "agentId": 7, + "conversationId": "wechat:chat-42", + "requesterId": "u-123", + "workspaceId": 5, + "workspaceBasePath": "/data/ws/5", + "channelId": 9, + "channelTarget": {"targetId":"group-a","threadId":null,"accountId":null,"newField":"x"}, + "futureTopLevelField": "y" + } + """; + ChatOrigin restored = workflow.restoreChatOrigin(json); + assertEquals(7L, restored.agentId()); + assertEquals("wechat:chat-42", restored.conversationId()); + assertEquals("group-a", restored.channelTarget().targetId()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceGcTest.java b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceGcTest.java new file mode 100644 index 00000000..d9cf8090 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceGcTest.java @@ -0,0 +1,202 @@ +package vip.mate.approval; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.approval.model.ToolApprovalEntity; +import vip.mate.approval.repository.ToolApprovalMapper; +import vip.mate.workspace.conversation.ConversationService; + +import java.time.Duration; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * GC contract for {@link ApprovalWorkflowService} (RFC-067 §4.4). + *

+ * The pre-RFC GC lived on {@link ApprovalService} and only mutated the in-memory + * map; mate_tool_approval rows stayed PENDING forever (recover-from-DB on next + * restart resurrected them) and message metadata kept showing a ghost approval + * banner. These tests pin the migrated behavior: + *

    + *
  • Phase A (TTL): expired pending → DB TIMEOUT + metadata DENIED + map removal
  • + *
  • Phase B (overflow): pending count over MAX → oldest evicted via the same + * full-sync path
  • + *
  • Phase C (resolved cleanup): non-pending entries past RESOLVED_TTL drop + * from the map only — DB / metadata are not touched
  • + *
  • Idempotent on idle ticks: nothing to GC means zero DB / metadata interactions
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +class ApprovalWorkflowServiceGcTest { + + @Mock private ToolApprovalMapper approvalMapper; + @Mock private ConversationService conversationService; + + private ApprovalService approvalService; + private ApprovalWorkflowService workflow; + + @BeforeAll + static void initMyBatisPlusCache() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new Configuration(), ""), + ToolApprovalEntity.class); + } + + @BeforeEach + void setUp() { + approvalService = new ApprovalService(); + workflow = new ApprovalWorkflowService( + approvalService, approvalMapper, new ObjectMapper(), conversationService); + } + + @Test + @DisplayName("Phase A: pending past PENDING_TTL goes through full DB+metadata+memory sync") + void expiredPendingGoesThroughMarkTimeout() { + // Pre-RFC: this row would silently be removed from the in-memory map but + // mate_tool_approval would stay PENDING and the next recoverFromDb would + // resurrect it. New contract: full two-phase markTimeout. + Instant created = Instant.now().minus(Duration.ofMinutes(31)); + seedPending("pid-expired", "conv-1", "write_file", created); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-1"), any(), eq(MetadataDecision.DENIED))).thenReturn(1); + + workflow.garbageCollect(); + + // Map cleared + assertThat(approvalService.size()).isZero(); + // DB UPDATE happened exactly once (markTimeout's conditional update). + verify(approvalMapper, times(1)).update(isNull(), any(Wrapper.class)); + // Metadata reconciled with DENIED. + verify(conversationService, times(1)).markPendingApprovalsResolved( + eq("conv-1"), any(), eq(MetadataDecision.DENIED)); + } + + @Test + @DisplayName("Phase A: pending within TTL is not touched") + void freshPendingIsKept() { + Instant created = Instant.now().minus(Duration.ofMinutes(5)); + PendingApproval p = seedPending("pid-fresh", "conv-2", "search", created); + + workflow.garbageCollect(); + + assertThat(approvalService.getPending("pid-fresh")).isPresent(); + assertThat(p.getStatus()).isEqualTo("pending"); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Phase C: resolved entry past RESOLVED_TTL drops from map without DB / metadata touch") + void resolvedTtlExpiredIsMemoryOnlyDrop() { + // DB row already terminal — workflow correctly decides this is memory-only cleanup. + Instant created = Instant.now().minus(Duration.ofHours(2)); + PendingApproval p = seedPending("pid-old-approved", "conv-3", "shell", created); + p.setStatus("approved"); + p.setResolvedAt(Instant.now().minus(Duration.ofHours(2))); + + workflow.garbageCollect(); + + assertThat(approvalService.getPending("pid-old-approved")).isEmpty(); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Phase C: resolved entry within TTL is kept") + void freshResolvedIsKept() { + PendingApproval p = seedPending("pid-recent-approved", "conv-4", "search", Instant.now()); + p.setStatus("approved"); + p.setResolvedAt(Instant.now().minus(Duration.ofMinutes(10))); + + workflow.garbageCollect(); + + assertThat(approvalService.getPending("pid-recent-approved")).isPresent(); + assertThat(p.getStatus()).isEqualTo("approved"); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Idle GC tick (no entries): zero DB / metadata interactions") + void idleGcIsNoop() { + workflow.garbageCollect(); + + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + assertThat(approvalService.size()).isZero(); + } + + @Test + @DisplayName("markTimeout idempotent: pendingId already off PENDING -> alreadyResolved, no metadata change") + void markTimeoutAlreadyConsumed() { + PendingApproval p = seedPending("pid-already", "conv-5", "search", Instant.now()); + p.setStatus("consumed"); + + ResolveOutcome outcome = workflow.markTimeout("pid-already"); + + assertThat(outcome.isAlreadyResolved()).isTrue(); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Phase A: per-row failure doesn't abort the sweep — other expired entries still process") + void perRowFailureContinuesSweep() { + // Two expired pendings; first one's DB UPDATE throws, second one succeeds. + // Pre-RFC's "all-or-nothing" loop would lose progress on the second; new GC + // catches per-row exceptions and continues. + Instant created = Instant.now().minus(Duration.ofMinutes(40)); + seedPending("pid-fail", "conv-fail", "write_file", created); + seedPending("pid-ok", "conv-ok", "shell", created); + + // First call throws, second returns 1. + when(approvalMapper.update(isNull(), any(Wrapper.class))) + .thenThrow(new RuntimeException("simulated outage")) + .thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-ok"), any(), eq(MetadataDecision.DENIED))).thenReturn(1); + + workflow.garbageCollect(); + + // pid-fail is still in memory (markTimeout's @Transactional roll-back leaves it untouched + // and the GC catch-block logs but continues). + assertThat(approvalService.getPending("pid-fail")).isPresent(); + // pid-ok was successfully timed out. + assertThat(approvalService.getPending("pid-ok")).isEmpty(); + + verify(approvalMapper, times(2)).update(isNull(), any(Wrapper.class)); + // Metadata reconciliation only fired for the successful row. + verify(conversationService, times(1)).markPendingApprovalsResolved( + eq("conv-ok"), any(), eq(MetadataDecision.DENIED)); + } + + // ---------- helpers ---------- + + private PendingApproval seedPending(String pendingId, String conversationId, + String toolName, Instant createdAt) { + PendingApproval p = new PendingApproval( + pendingId, conversationId, "system", toolName, "{}", "test", + createdAt, "pending"); + approvalService.registerRecovered(p); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceRecoveryTest.java b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceRecoveryTest.java new file mode 100644 index 00000000..e2dc922e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceRecoveryTest.java @@ -0,0 +1,243 @@ +package vip.mate.approval; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.approval.model.ToolApprovalEntity; +import vip.mate.approval.repository.ToolApprovalMapper; +import vip.mate.workspace.conversation.ConversationService; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * Recovery contract for ApprovalWorkflowService.recoverFromDb (RFC-067 §4.1). + *

+ * The pre-RFC implementation generated a fresh random pendingId on recovery, + * which silently desynchronized the in-memory map from mate_tool_approval and + * left every later resolve()/updateDbStatus() call hitting zero rows. These + * tests pin the new contract: + *

    + *
  • Live row → pendingMap entry preserves the DB pendingId AND createdAt + * (so PENDING_TTL math still works after restart)
  • + *
  • Expired row (expireAt past) → DB → TIMEOUT, metadata reconciled DENIED, + * no pendingMap entry
  • + *
  • Legacy row with expireAt NULL falls back to createdAt + PENDING_TTL — + * this is the regression-prevention case for §4.1's effectiveExpireAt + * fallback. A naive "if expireAt != null && now > expireAt" check would + * silently revive ancient PENDING rows after every restart.
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +class ApprovalWorkflowServiceRecoveryTest { + + @Mock private ToolApprovalMapper approvalMapper; + @Mock private ConversationService conversationService; + + private ApprovalService approvalService; // real, so registerRecovered is exercised + private ApprovalWorkflowService workflow; + + @BeforeAll + static void initMyBatisPlusCache() { + // PR-2 resolveAndConsume builds a LambdaUpdateWrapper.set(...) which needs + // ToolApprovalEntity's TableInfo to be registered in MyBatis-Plus's static + // cache (a Spring context normally does this during mapper scan). + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new Configuration(), ""), + ToolApprovalEntity.class); + } + + private void initWorkflow(List dbRows) { + approvalService = new ApprovalService(); + // Skip the GC scheduler — initGc() spins up a daemon thread we don't need here. + // Tests interact with the registry via registerRecovered + getPending only. + workflow = new ApprovalWorkflowService( + approvalService, + approvalMapper, + new ObjectMapper(), + conversationService); + when(approvalMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(dbRows); + } + + @Test + @DisplayName("Live PENDING row recovers with DB pendingId + createdAt preserved") + void recoversLiveRowPreservingIdAndCreatedAt() { + LocalDateTime created = LocalDateTime.now().minusMinutes(5); + ToolApprovalEntity row = newPendingRow("pid-live-1", "conv-1", created, + created.plusMinutes(30)); + initWorkflow(List.of(row)); + + workflow.recoverFromDb(); + + PendingApproval recovered = approvalService.getPending("pid-live-1").orElse(null); + assertThat(recovered).isNotNull(); + assertThat(recovered.getPendingId()).isEqualTo("pid-live-1"); + assertThat(recovered.getConversationId()).isEqualTo("conv-1"); + assertThat(recovered.getStatus()).isEqualTo("pending"); + // createdAt round-trips with second precision (LocalDateTime → Instant via system zone) + assertThat(recovered.getCreatedAt().getEpochSecond()) + .isEqualTo(created.atZone(java.time.ZoneId.systemDefault()).toEpochSecond()); + + // Did not silently expire the live row. + verify(approvalMapper, never()).updateById(any(ToolApprovalEntity.class)); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Expired row with explicit past expireAt: DB -> TIMEOUT, metadata DENIED, not in map") + void expiredRowWithExplicitExpireAt() { + LocalDateTime created = LocalDateTime.now().minusMinutes(31); + ToolApprovalEntity row = newPendingRow("pid-exp-1", "conv-2", created, + created.plusMinutes(30)); + initWorkflow(List.of(row)); + when(approvalMapper.updateById(any(ToolApprovalEntity.class))).thenReturn(1); + + workflow.recoverFromDb(); + + assertThat(approvalService.getPending("pid-exp-1")).isEmpty(); + ArgumentCaptor updated = ArgumentCaptor.forClass(ToolApprovalEntity.class); + verify(approvalMapper).updateById(updated.capture()); + assertThat(updated.getValue().getStatus()).isEqualTo("TIMEOUT"); + assertThat(updated.getValue().getResolvedAt()).isNotNull(); + verify(conversationService).markPendingApprovalsResolved( + eq("conv-2"), eq(Set.of("pid-exp-1")), eq(MetadataDecision.DENIED)); + } + + @Test + @DisplayName("Legacy row with expireAt=NULL still expires via createdAt + PENDING_TTL fallback") + void legacyRowFallsBackToCreatedAtPlusTtl() { + // Mirrors the §4.1 regression case: pre-RFC rows persisted by an older build + // never got an expireAt column populated. Without the fallback, recoverFromDb + // would resurrect them as live PENDING after every restart — a permanent ghost + // approval source. + LocalDateTime created = LocalDateTime.now().minusMinutes(31); + ToolApprovalEntity row = newPendingRow("pid-legacy-1", "conv-3", created, null); + initWorkflow(List.of(row)); + when(approvalMapper.updateById(any(ToolApprovalEntity.class))).thenReturn(1); + + workflow.recoverFromDb(); + + assertThat(approvalService.getPending("pid-legacy-1")).isEmpty(); + verify(approvalMapper).updateById(any(ToolApprovalEntity.class)); + verify(conversationService).markPendingApprovalsResolved( + eq("conv-3"), eq(Set.of("pid-legacy-1")), eq(MetadataDecision.DENIED)); + } + + @Test + @DisplayName("DB updateById returning 0 rows: metadata is NOT touched (no drift)") + void expireSkipsMetadataWhenDbAffectsZeroRows() { + // Concurrent resolve case: another path already moved the row off PENDING + // between selectList and updateById. Metadata flip MUST be gated on DB + // success, otherwise message metadata = denied while DB is e.g. CONSUMED, + // and the next recoverFromDb would resurrect it — exactly the drift we + // came here to fix. + LocalDateTime created = LocalDateTime.now().minusMinutes(31); + ToolApprovalEntity row = newPendingRow("pid-race-1", "conv-race", created, + created.plusMinutes(30)); + initWorkflow(List.of(row)); + when(approvalMapper.updateById(any(ToolApprovalEntity.class))).thenReturn(0); + + workflow.recoverFromDb(); + + assertThat(approvalService.getPending("pid-race-1")).isEmpty(); + verify(approvalMapper).updateById(any(ToolApprovalEntity.class)); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("DB updateById throwing: metadata is NOT touched") + void expireSkipsMetadataWhenDbThrows() { + LocalDateTime created = LocalDateTime.now().minusMinutes(31); + ToolApprovalEntity row = newPendingRow("pid-throw-1", "conv-throw", created, + created.plusMinutes(30)); + initWorkflow(List.of(row)); + when(approvalMapper.updateById(any(ToolApprovalEntity.class))) + .thenThrow(new RuntimeException("simulated DB outage")); + + workflow.recoverFromDb(); + + assertThat(approvalService.getPending("pid-throw-1")).isEmpty(); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Legacy row with expireAt=NULL but createdAt within TTL: still recovers as live") + void legacyRowWithinTtlStillRecovers() { + LocalDateTime created = LocalDateTime.now().minusMinutes(5); + ToolApprovalEntity row = newPendingRow("pid-legacy-live", "conv-4", created, null); + initWorkflow(List.of(row)); + + workflow.recoverFromDb(); + + assertThat(approvalService.getPending("pid-legacy-live")).isPresent(); + verify(approvalMapper, never()).updateById(any(ToolApprovalEntity.class)); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("Recovered pending carries its replay payload through resolveAndConsume") + void resolveAfterRecoveryYieldsRecoveredPayload() { + // Pre-RFC, recovery generated a fresh random id; the next resolveAndConsume + // either pulled the wrong record or the in-memory map was empty altogether. + // This pins that the original DB pendingId AND the replay payload (toolCallPayload) + // round-trip through recovery and still drive consume successfully through the + // PR-2 ResolveOutcome contract. + LocalDateTime created = LocalDateTime.now().minusMinutes(2); + ToolApprovalEntity row = newPendingRow("pid-resolve-1", "conv-5", created, + created.plusMinutes(30)); + row.setToolCallPayload("{\"name\":\"write_file\"}"); + initWorkflow(List.of(row)); + // Stub the DB UPDATE that the new two-phase resolve runs; metadata mock is + // already injected and returns 0 by default which matches "no message rewrites". + when(approvalMapper.update(any(), any())).thenReturn(1); + + workflow.recoverFromDb(); + PendingApproval recovered = approvalService.getPending("pid-resolve-1").orElseThrow(); + assertThat(recovered.getToolCallPayload()).isEqualTo("{\"name\":\"write_file\"}"); + + ResolveOutcome outcome = workflow.resolveAndConsume("pid-resolve-1", "alice"); + assertThat(outcome.isConsumed()).isTrue(); + assertThat(outcome.dbSynced()).isTrue(); + assertThat(outcome.consumedSnapshot()).isNotNull(); + assertThat(outcome.consumedSnapshot().getPendingId()).isEqualTo("pid-resolve-1"); + assertThat(outcome.consumedSnapshot().getToolCallPayload()).isEqualTo("{\"name\":\"write_file\"}"); + assertThat(outcome.consumedSnapshot().getStatus()).isEqualTo("consumed"); + assertThat(outcome.consumedSnapshot().getResolvedBy()).isEqualTo("alice"); + // pendingMap entry has been removed; a second consume is idempotent already_resolved. + ResolveOutcome second = workflow.resolveAndConsume("pid-resolve-1", "alice"); + assertThat(second.isAlreadyResolved()).isTrue(); + } + + private ToolApprovalEntity newPendingRow(String pendingId, String conversationId, + LocalDateTime createdAt, LocalDateTime expireAt) { + ToolApprovalEntity e = new ToolApprovalEntity(); + e.setPendingId(pendingId); + e.setConversationId(conversationId); + e.setUserId("u"); + e.setToolName("write_file"); + e.setToolArguments("{}"); + e.setSummary("test"); + e.setStatus("PENDING"); + e.setCreatedAt(createdAt); + e.setExpireAt(expireAt); + return e; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceResolveTest.java b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceResolveTest.java new file mode 100644 index 00000000..b19a6469 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/approval/ApprovalWorkflowServiceResolveTest.java @@ -0,0 +1,351 @@ +package vip.mate.approval; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.apache.ibatis.session.Configuration; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.approval.model.ToolApprovalEntity; +import vip.mate.approval.repository.ToolApprovalMapper; +import vip.mate.workspace.conversation.ConversationService; + +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * Two-phase resolve contract for {@link ApprovalWorkflowService} (RFC-067 §4.2 / §4.3). + *

+ * The pre-RFC implementation removed the in-memory entry FIRST then attempted DB + * UPDATE on a best-effort try / catch — payload could be lost while DB stayed + * PENDING. These tests pin the new ordering: + *

    + *
  1. snapshot (no map mutation)
  2. + *
  3. DB UPDATE conditional on {@code status='PENDING'} (idempotent against concurrent resolve)
  4. + *
  5. metadata reconciliation (same tx)
  6. + *
  7. memory mutation only on commit (afterCommit hook; immediate when no tx active)
  8. + *
+ *

+ * Tests run outside Spring's tx manager, so the {@code afterCommit} hook executes + * immediately — that exercises the same observable end-state as a committed tx. + */ +@ExtendWith(MockitoExtension.class) +class ApprovalWorkflowServiceResolveTest { + + @Mock private ToolApprovalMapper approvalMapper; + @Mock private ConversationService conversationService; + + private ApprovalService approvalService; + private ApprovalWorkflowService workflow; + + @BeforeAll + static void initMyBatisPlusCache() { + // LambdaUpdateWrapper.set / .eq need the entity's TableInfo to be registered in + // MyBatis-Plus's static cache. In a Spring context this happens during mapper + // scan; in a plain MockitoExtension test we trigger it manually. + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new Configuration(), ""), + ToolApprovalEntity.class); + } + + @BeforeEach + void setUp() { + approvalService = new ApprovalService(); + workflow = new ApprovalWorkflowService( + approvalService, approvalMapper, new ObjectMapper(), conversationService); + } + + @Test + @DisplayName("resolve(approved) updates DB, metadata, and snapshot status; entry stays in map") + void resolveApprovedHappyPath() { + PendingApproval pending = seedPending("pid-1", "conv-1", "write_file"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-1"), eq(Set.of("pid-1")), eq(MetadataDecision.APPROVED))).thenReturn(1); + + ResolveOutcome outcome = workflow.resolve("pid-1", "alice", "approved"); + + assertThat(outcome.decision()).isEqualTo("approved"); + assertThat(outcome.dbSynced()).isTrue(); + assertThat(outcome.messagesRewritten()).isEqualTo(1); + assertThat(outcome.consumedSnapshot()).isNull(); + + // Memory: status flipped to "approved", entry stays in map (resolve does NOT remove) + assertThat(pending.getStatus()).isEqualTo("approved"); + assertThat(pending.getResolvedBy()).isEqualTo("alice"); + assertThat(approvalService.getPending("pid-1")).isPresent(); + + verify(approvalMapper, times(1)).update(isNull(), any(Wrapper.class)); + verify(conversationService).markPendingApprovalsResolved( + "conv-1", Set.of("pid-1"), MetadataDecision.APPROVED); + } + + @Test + @DisplayName("resolve(denied) flips metadata + snapshot to denied") + void resolveDeniedHappyPath() { + PendingApproval pending = seedPending("pid-2", "conv-2", "shell"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-2"), eq(Set.of("pid-2")), eq(MetadataDecision.DENIED))).thenReturn(1); + + ResolveOutcome outcome = workflow.resolve("pid-2", "bob", "denied"); + + assertThat(outcome.decision()).isEqualTo("denied"); + assertThat(outcome.dbSynced()).isTrue(); + assertThat(pending.getStatus()).isEqualTo("denied"); + verify(conversationService).markPendingApprovalsResolved( + "conv-2", Set.of("pid-2"), MetadataDecision.DENIED); + } + + @Test + @DisplayName("resolve no-op when pendingId not in map: no DB / metadata interaction") + void resolveUnknownPendingId() { + ResolveOutcome outcome = workflow.resolve("ghost-id", "alice", "approved"); + + assertThat(outcome.isAlreadyResolved()).isTrue(); + assertThat(outcome.dbSynced()).isFalse(); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("resolve idempotent against concurrent resolve: DB rows=0 -> no metadata, no memory mutation") + void resolveIdempotentOnConcurrentResolve() { + PendingApproval pending = seedPending("pid-race", "conv-race", "write_file"); + // Another path already moved the row off PENDING between snapshot and DB UPDATE. + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(0); + + ResolveOutcome outcome = workflow.resolve("pid-race", "alice", "approved"); + + assertThat(outcome.isAlreadyResolved()).isTrue(); + assertThat(outcome.dbSynced()).isFalse(); + assertThat(outcome.messagesRewritten()).isZero(); + // Snapshot status was NOT flipped to approved — memory stays consistent with DB. + assertThat(pending.getStatus()).isEqualTo("pending"); + assertThat(approvalService.getPending("pid-race")).isPresent(); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("resolveAndConsume happy path: DB CONSUMED, metadata APPROVED, snapshot removed from map") + void resolveAndConsumeHappyPath() { + PendingApproval pending = seedPending("pid-c-1", "conv-c", "write_file"); + pending.setToolCallPayload("{\"name\":\"write_file\"}"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-c"), eq(Set.of("pid-c-1")), eq(MetadataDecision.APPROVED))).thenReturn(2); + + ResolveOutcome outcome = workflow.resolveAndConsume("pid-c-1", "carol"); + + assertThat(outcome.isConsumed()).isTrue(); + assertThat(outcome.dbSynced()).isTrue(); + assertThat(outcome.messagesRewritten()).isEqualTo(2); + assertThat(outcome.consumedSnapshot()).isNotNull(); + assertThat(outcome.consumedSnapshot().getToolCallPayload()).isEqualTo("{\"name\":\"write_file\"}"); + + // Memory: status flipped to consumed, entry REMOVED (single-shot consume). + assertThat(pending.getStatus()).isEqualTo("consumed"); + assertThat(approvalService.getPending("pid-c-1")).isEmpty(); + + // Second consume returns idempotent already_resolved (entry is gone). + ResolveOutcome second = workflow.resolveAndConsume("pid-c-1", "carol"); + assertThat(second.isAlreadyResolved()).isTrue(); + } + + @Test + @DisplayName("resolveAndConsume DB rows=0: no metadata, snapshot stays in map") + void resolveAndConsumeRaceLeavesMapAlone() { + PendingApproval pending = seedPending("pid-c-race", "conv-cr", "write_file"); + pending.setToolCallPayload("{}"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(0); + + ResolveOutcome outcome = workflow.resolveAndConsume("pid-c-race", "alice"); + + assertThat(outcome.isAlreadyResolved()).isTrue(); + // Critical: payload is NOT lost. Replay can still find the entry next loop. + assertThat(approvalService.getPending("pid-c-race")).isPresent(); + assertThat(pending.getStatus()).isEqualTo("pending"); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("consumeApproved redeems the earliest approved record; missing match -> alreadyResolved") + void consumeApprovedHappyAndMiss() { + PendingApproval pending = seedPending("pid-app-1", "conv-app", "search"); + // Caller previously approved but did not consume — common in /approve text flow. + pending.setStatus("approved"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-app"), eq(Set.of("pid-app-1")), eq(MetadataDecision.APPROVED))).thenReturn(1); + + ResolveOutcome consumed = workflow.consumeApproved("conv-app", "search"); + + assertThat(consumed.isConsumed()).isTrue(); + assertThat(consumed.consumedSnapshot()).isNotNull(); + assertThat(approvalService.getPending("pid-app-1")).isEmpty(); + + // Second call: nothing approved left → no additional DB / metadata interaction. + org.mockito.Mockito.clearInvocations(approvalMapper, conversationService); + ResolveOutcome miss = workflow.consumeApproved("conv-app", "search"); + assertThat(miss.isAlreadyResolved()).isTrue(); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("cancelStalePending issues a SUPERSEDED outcome per pending in the conversation") + void cancelStalePendingMultipleEntries() { + PendingApproval a = seedPending("pid-stale-A", "conv-stale", "write_file"); + PendingApproval b = seedPending("pid-stale-B", "conv-stale", "shell"); + PendingApproval keep = seedPending("pid-keep", "conv-stale", "memory_recall"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-stale"), any(), eq(MetadataDecision.DENIED))).thenReturn(1); + + List outcomes = workflow.cancelStalePending("conv-stale", "pid-keep"); + + assertThat(outcomes).hasSize(2); + assertThat(outcomes).extracting(ResolveOutcome::pendingId) + .containsExactlyInAnyOrder("pid-stale-A", "pid-stale-B"); + assertThat(outcomes).allMatch(o -> "superseded".equals(o.decision())); + // Excluded entry untouched. + assertThat(approvalService.getPending("pid-keep")).isPresent(); + assertThat(keep.getStatus()).isEqualTo("pending"); + // Cancelled entries removed from map. + assertThat(approvalService.getPending("pid-stale-A")).isEmpty(); + assertThat(approvalService.getPending("pid-stale-B")).isEmpty(); + assertThat(a.getStatus()).isEqualTo("superseded"); + assertThat(b.getStatus()).isEqualTo("superseded"); + + // Two DB updates fired (one per cancellation). + verify(approvalMapper, times(2)).update(isNull(), any(Wrapper.class)); + } + + @Test + @DisplayName("denyAllByConversation: every pending becomes denied; metadata reconciled per row") + void denyAllConversationSweep() { + // Stop endpoint scenario: user halts a turn while two pendings sit in the map. + PendingApproval a = seedPending("pid-stop-A", "conv-stop", "write_file"); + PendingApproval b = seedPending("pid-stop-B", "conv-stop", "shell"); + seedPending("pid-other-conv", "conv-other", "search"); // not in target conversation + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-stop"), any(), eq(MetadataDecision.DENIED))).thenReturn(1); + + List outcomes = workflow.denyAllByConversation("conv-stop", "alice"); + + assertThat(outcomes).hasSize(2); + assertThat(outcomes).extracting(ResolveOutcome::pendingId) + .containsExactlyInAnyOrder("pid-stop-A", "pid-stop-B"); + assertThat(outcomes).allMatch(o -> "denied".equals(o.decision())); + assertThat(a.getStatus()).isEqualTo("denied"); + assertThat(b.getStatus()).isEqualTo("denied"); + // Targets removed from map. + assertThat(approvalService.getPending("pid-stop-A")).isEmpty(); + assertThat(approvalService.getPending("pid-stop-B")).isEmpty(); + // Other conversation untouched. + assertThat(approvalService.getPending("pid-other-conv")).isPresent(); + // Two metadata reconciliations fired (one per pending). + verify(conversationService, times(2)).markPendingApprovalsResolved( + eq("conv-stop"), any(), eq(MetadataDecision.DENIED)); + } + + @Test + @DisplayName("denyAllByConversation: empty conversation -> empty outcomes, no DB / metadata interaction") + void denyAllNoPendingsIsNoop() { + seedPending("pid-other", "conv-other", "search"); + + List outcomes = workflow.denyAllByConversation("conv-empty", "alice"); + + assertThat(outcomes).isEmpty(); + verifyNoInteractions(approvalMapper); + verifyNoInteractions(conversationService); + } + + @Test + @DisplayName("denyAllByConversation: per-row failure doesn't abort the sweep") + void denyAllPerRowFailureContinues() { + seedPending("pid-fail", "conv-mix", "write_file"); + seedPending("pid-ok", "conv-mix", "shell"); + // First UPDATE throws, second succeeds. + when(approvalMapper.update(isNull(), any(Wrapper.class))) + .thenThrow(new RuntimeException("simulated outage")) + .thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-mix"), any(), eq(MetadataDecision.DENIED))).thenReturn(1); + + List outcomes = workflow.denyAllByConversation("conv-mix", "alice"); + + // Only the successful row makes it into the outcomes list. + assertThat(outcomes).hasSize(1); + assertThat(outcomes.get(0).pendingId()).isEqualTo("pid-ok"); + // Failed row is still in memory (transactional rollback would leave it untouched). + assertThat(approvalService.getPending("pid-fail")).isPresent(); + assertThat(approvalService.getPending("pid-ok")).isEmpty(); + } + + @Test + @DisplayName("DB UPDATE throwing propagates so @Transactional can roll back; memory untouched") + void dbThrowsPropagatesForRollback() { + PendingApproval pending = seedPending("pid-throw", "conv-throw", "write_file"); + when(approvalMapper.update(isNull(), any(Wrapper.class))) + .thenThrow(new RuntimeException("simulated outage")); + + try { + workflow.resolve("pid-throw", "alice", "approved"); + org.junit.jupiter.api.Assertions.fail("expected RuntimeException"); + } catch (RuntimeException expected) { + assertThat(expected.getMessage()).contains("simulated outage"); + } + // Memory snapshot must not have flipped. + assertThat(pending.getStatus()).isEqualTo("pending"); + verifyNoInteractions(conversationService); + // approvalService is a real instance in these tests, not a Mockito mock — + // its untouched state is asserted via the snapshot status above. + } + + @Test + @DisplayName("ResolveOutcome carries conversationId + toolName for SSE broadcast use") + void outcomeShape() { + PendingApproval pending = seedPending("pid-shape", "conv-shape", "search_web"); + when(approvalMapper.update(isNull(), any(Wrapper.class))).thenReturn(1); + when(conversationService.markPendingApprovalsResolved( + eq("conv-shape"), eq(Set.of("pid-shape")), eq(MetadataDecision.DENIED))).thenReturn(0); + + ResolveOutcome outcome = workflow.resolve("pid-shape", "alice", "denied"); + + assertThat(outcome.pendingId()).isEqualTo("pid-shape"); + assertThat(outcome.conversationId()).isEqualTo("conv-shape"); + assertThat(outcome.toolName()).isEqualTo("search_web"); + assertThat(outcome.messagesRewritten()).isZero(); + } + + // ---------- helpers ---------- + + private PendingApproval seedPending(String pendingId, String conversationId, String toolName) { + // Use the public createPending overload, then re-key the map under the + // requested pendingId so the test asserts work against a stable id. + // The recovery constructor is package-visible from this same package. + PendingApproval p = new PendingApproval( + pendingId, conversationId, "system", toolName, "{}", "test", + java.time.Instant.now(), "pending"); + approvalService.registerRecovered(p); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/architecture/StateKeyRegistrationCoverageTest.java b/mateclaw-server/src/test/java/vip/mate/architecture/StateKeyRegistrationCoverageTest.java new file mode 100644 index 00000000..5e818618 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/architecture/StateKeyRegistrationCoverageTest.java @@ -0,0 +1,89 @@ +package vip.mate.architecture; + +import org.junit.jupiter.api.Test; +import vip.mate.agent.graph.state.MateClawStateKeys; + +import java.lang.reflect.Modifier; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Architecture guard — every state key declared on + * {@link MateClawStateKeys} that participates in graph state (i.e. is not a + * node-name constant) MUST be registered in + * {@link vip.mate.agent.AgentGraphBuilder}'s {@code KeyStrategyFactory} + * for at least one of the two graphs (ReAct + Plan-Execute). + * + *

Regression rationale: the post-deploy bug where {@code CHAT_ORIGIN} was + * declared on {@link MateClawStateKeys} but missing from both + * {@code KeyStrategyFactory} blocks shipped silently, and {@code spring-ai-alibaba-graph} + * dropped the key on multi-node merges, causing the channel-binding flakiness + * reported by the user. This test parses the source of + * {@code AgentGraphBuilder.java} for all + * {@code .addStrategy(MateClawStateKeys.X, ...)} mentions and asserts the + * coverage so the same kind of "forgot to register" can never ship again. + * + *

Excluded by suffix: any constant whose name ends with {@code _NODE} — + * those are graph-node identifiers used by {@code addNode(...)}, not state + * keys. + */ +class StateKeyRegistrationCoverageTest { + + private static final Pattern ADD_STRATEGY = Pattern.compile( + "\\.addStrategy\\(\\s*MateClawStateKeys\\.([A-Z_]+)"); + + @Test + void everyStateKeyMustBeRegisteredInKeyStrategyFactory() throws Exception { + // Read the AgentGraphBuilder source — relative to mateclaw-server module root. + Path source = Paths.get("src/main/java/vip/mate/agent/AgentGraphBuilder.java") + .toAbsolutePath(); + if (!Files.exists(source)) { + fail("Cannot find AgentGraphBuilder.java at " + source + + " — has the file moved? Update this test's path."); + } + String content = Files.readString(source); + + Set registered = new TreeSet<>(); + Matcher m = ADD_STRATEGY.matcher(content); + while (m.find()) { + registered.add(m.group(1)); + } + assertTrue(registered.size() > 10, + "Suspiciously few addStrategy hits — regex broken? Found: " + registered); + + Set declared = new TreeSet<>(); + for (var f : MateClawStateKeys.class.getDeclaredFields()) { + int mods = f.getModifiers(); + if (!Modifier.isPublic(mods) || !Modifier.isStatic(mods) + || !Modifier.isFinal(mods) || f.getType() != String.class) { + continue; + } + // Node-name constants are NOT state keys — they're graph-node + // identifiers used by addNode(...). Exclude them by suffix. + if (f.getName().endsWith("_NODE")) continue; + declared.add(f.getName()); + } + + Set missing = new TreeSet<>(declared); + missing.removeAll(registered); + + if (!missing.isEmpty()) { + fail("State keys declared on MateClawStateKeys but NOT registered in any " + + "KeyStrategyFactory in AgentGraphBuilder.java:\n" + + " " + missing + "\n\n" + + "Without registration, spring-ai-alibaba-graph may drop these keys on " + + "multi-node state merges (silently, intermittently). Add an " + + ".addStrategy(MateClawStateKeys.X, KeyStrategy.REPLACE) line for each " + + "missing key in BOTH the ReAct and Plan-Execute KeyStrategyFactory blocks " + + "(or document why the key is intentionally Plan-only / ReAct-only)."); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/architecture/ToolCallbackToolContextForwardArchTest.java b/mateclaw-server/src/test/java/vip/mate/architecture/ToolCallbackToolContextForwardArchTest.java new file mode 100644 index 00000000..8794f0f9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/architecture/ToolCallbackToolContextForwardArchTest.java @@ -0,0 +1,119 @@ +package vip.mate.architecture; + +import com.tngtech.archunit.core.domain.JavaClass; +import com.tngtech.archunit.core.domain.JavaClasses; +import com.tngtech.archunit.core.domain.JavaMethod; +import com.tngtech.archunit.core.importer.ClassFileImporter; +import com.tngtech.archunit.core.importer.ImportOption; +import com.tngtech.archunit.lang.ArchCondition; +import com.tngtech.archunit.lang.ConditionEvents; +import com.tngtech.archunit.lang.SimpleConditionEvent; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.transaction.annotation.Transactional; + +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.classes; +import static com.tngtech.archunit.lang.syntax.ArchRuleDefinition.noClasses; + +/** + * RFC-063r §2.3: every concrete {@link ToolCallback} implementation must + * override {@code call(String, ToolContext)} so it cannot silently drop the + * Spring AI {@link ToolContext} (which carries the {@code ChatOrigin}). + * + *

Background: the previous {@code LocaleAwareToolCallback} only overrode + * {@code call(String)}; the framework default routed + * {@code call(String, ToolContext)} back to {@code call(String)}, dropping the + * context. This test pins the rule so a future regression fails CI. + */ +class ToolCallbackToolContextForwardArchTest { + + private static final JavaClasses MATECLAW_CLASSES = new ClassFileImporter() + .withImportOption(ImportOption.Predefined.DO_NOT_INCLUDE_TESTS) + .importPackages("vip.mate"); + + @Test + void everyToolCallbackImplementationMustOverrideCallWithToolContext() { + classes() + .that().implement(ToolCallback.class) + .and().areNotInterfaces() + .and().areNotAnnotations() + .and(haveSimpleNameNot("ToolCallback")) + .should(overrideCallWithToolContext()) + .check(MATECLAW_CLASSES); + } + + /** + * RFC-063r §5.2 hard rule: {@code CronJobRunner} must NEVER carry + * {@code @Transactional} (class-level or method-level). The class is + * the entry point for cron-tick execution; an inline transaction would + * either swallow self-invocation calls or — worse — hold a DB connection + * across the multi-minute LLM call inside {@code runAgent}, exhausting + * the HikariCP pool under concurrent cron load. + * + *

The three transactional segments live on + * {@code CronJobLifecycleService}; cross-bean invocation routes through + * the Spring AOP proxy and works as designed. This test pins the rule. + */ + @Test + void cronJobRunnerMustNotCarryTransactional() { + noClasses() + .that().haveSimpleName("CronJobRunner") + .and().resideInAPackage("vip.mate.cron..") + .should(beAnnotatedOrHaveAnyMethodAnnotatedWith(Transactional.class)) + .because("RFC-063r §5.2: CronJobRunner.runAgent runs an LLM HTTP call (seconds-to-minutes); " + + "@Transactional would hold a DB connection during that call and exhaust HikariCP under " + + "concurrent cron load. Transactions must live on CronJobLifecycleService instead.") + .check(MATECLAW_CLASSES); + } + + private static com.tngtech.archunit.base.DescribedPredicate haveSimpleNameNot(String simpleName) { + return new com.tngtech.archunit.base.DescribedPredicate<>("simple name is not " + simpleName) { + @Override + public boolean test(JavaClass javaClass) { + return !javaClass.getSimpleName().equals(simpleName); + } + }; + } + + private static ArchCondition beAnnotatedOrHaveAnyMethodAnnotatedWith( + Class annotation) { + String desc = annotation.getName(); + return new ArchCondition<>("be annotated or have any method annotated with " + desc) { + @Override + public void check(JavaClass clazz, ConditionEvents events) { + if (clazz.isAnnotatedWith(annotation)) { + events.add(SimpleConditionEvent.satisfied(clazz, + clazz.getFullName() + " is annotated with " + desc)); + return; + } + for (JavaMethod m : clazz.getMethods()) { + if (m.isAnnotatedWith(annotation)) { + events.add(SimpleConditionEvent.satisfied(clazz, + clazz.getFullName() + "#" + m.getName() + " is annotated with " + desc)); + return; + } + } + } + }; + } + + private static ArchCondition overrideCallWithToolContext() { + return new ArchCondition<>("override call(String, ToolContext)") { + @Override + public void check(JavaClass clazz, ConditionEvents events) { + boolean overrides = clazz.getMethods().stream().anyMatch(m -> + m.getName().equals("call") + && m.getRawParameterTypes().size() == 2 + && m.getRawParameterTypes().get(0).getFullName().equals(String.class.getName()) + && m.getRawParameterTypes().get(1).getFullName().equals(ToolContext.class.getName())); + if (!overrides) { + events.add(SimpleConditionEvent.violated(clazz, + clazz.getFullName() + " does not override call(String, ToolContext); " + + "the framework default would silently drop the ChatOrigin " + + "(see RFC-063r §2.3).")); + } + } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/auth/pat/PersonalAccessTokenServiceTest.java b/mateclaw-server/src/test/java/vip/mate/auth/pat/PersonalAccessTokenServiceTest.java new file mode 100644 index 00000000..8101385f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/auth/pat/PersonalAccessTokenServiceTest.java @@ -0,0 +1,309 @@ +package vip.mate.auth.pat; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.auth.pat.repository.PersonalAccessTokenMapper; +import vip.mate.exception.MateClawException; + +import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.Optional; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * RFC-03 Lane I1 — covers {@link PersonalAccessTokenService} core contracts: + * + *

    + *
  • Plaintext format ({@code mc_*}) and uniqueness across mints.
  • + *
  • SHA-256 hashing is deterministic and matches a known vector — a + * silent change to the hash function would invalidate every existing + * row in production, so this is enforced in test.
  • + *
  • {@link PersonalAccessTokenService#findActiveByPlaintext} rejects + * null, blank, wrong-prefix, hash-miss, disabled, and expired + * tokens with no observable difference (don't leak which one).
  • + *
  • {@link PersonalAccessTokenService#recordUse} debounces writes so + * a CI loop doesn't hammer the row.
  • + *
  • {@link PersonalAccessTokenService#revoke} requires owner match — + * a token id alone is insufficient to revoke someone else's token.
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +class PersonalAccessTokenServiceTest { + + @Mock + private PersonalAccessTokenMapper mapper; + + @InjectMocks + private PersonalAccessTokenService service; + + private PersonalAccessTokenEntity entity; + + @BeforeEach + void setUp() { + entity = new PersonalAccessTokenEntity(); + entity.setId(42L); + entity.setUserId(7L); + entity.setName("ci-key"); + entity.setEnabled(true); + } + + // ── Plaintext format ────────────────────────────────────────────────── + + @Test + @DisplayName("generated plaintext starts with mc_ and is sufficiently long for 256-bit entropy") + void plaintextFormat() { + String tok = service.generatePlaintext(); + assertTrue(tok.startsWith("mc_"), "PAT must start with the observable mc_ prefix"); + // 32 bytes base64 url-encoded without padding = 43 chars; total = 3 + 43 = 46. + assertEquals(46, tok.length(), + "32 bytes of entropy → 43 base64 chars + 3-char prefix; got " + tok); + } + + @Test + @DisplayName("each generation yields a unique plaintext (entropy actually random)") + void plaintextUniqueness() { + Set seen = new HashSet<>(); + for (int i = 0; i < 100; i++) { + assertTrue(seen.add(service.generatePlaintext()), + "duplicate within 100 mints — RNG is not actually random"); + } + } + + // ── SHA-256 hashing ────────────────────────────────────────────────── + + @Test + @DisplayName("sha256Hex matches the canonical reference vector for 'abc'") + void sha256ReferenceVector() { + // From FIPS 180-4 — locking in the algorithm; if this assertion ever + // fires, every PAT in the database is invalidated by the same change. + assertEquals( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + PersonalAccessTokenService.sha256Hex("abc")); + } + + @Test + @DisplayName("sha256Hex output is always 64 lowercase hex chars") + void sha256OutputShape() { + String h = PersonalAccessTokenService.sha256Hex("any plaintext"); + assertEquals(64, h.length()); + assertTrue(h.matches("[0-9a-f]+")); + } + + // ── findActiveByPlaintext rejection paths ───────────────────────────── + + @Test + @DisplayName("null / blank input returns empty without DB roundtrip") + void nullBlankReturnsEmpty() { + assertTrue(service.findActiveByPlaintext(null).isEmpty()); + assertTrue(service.findActiveByPlaintext("").isEmpty()); + assertTrue(service.findActiveByPlaintext(" ").isEmpty()); + verify(mapper, never()).selectOne(any()); + } + + @Test + @DisplayName("token without mc_ prefix returns empty without DB roundtrip") + void wrongPrefixReturnsEmpty() { + // JWT-shaped value should not even hit the DB — keeps the auth filter + // dispatch cheap when callers send either token type by mistake. + assertTrue(service.findActiveByPlaintext("eyJhbGciOiJIUzI1NiJ9...").isEmpty()); + verify(mapper, never()).selectOne(any()); + } + + @Test + @DisplayName("hash miss returns empty") + void hashMissReturnsEmpty() { + when(mapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + assertTrue(service.findActiveByPlaintext("mc_unknown_token").isEmpty()); + } + + @Test + @DisplayName("expired token returns empty even when the row matches") + void expiredTokenReturnsEmpty() { + entity.setExpiresAt(LocalDateTime.now().minusMinutes(1)); + when(mapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(entity); + assertTrue(service.findActiveByPlaintext("mc_some_plaintext").isEmpty(), + "expired tokens must reject — past-expiry is the same as no-such-token from auth's PoV"); + } + + @Test + @DisplayName("active, unexpired token returns the entity") + void activeTokenReturned() { + entity.setExpiresAt(LocalDateTime.now().plusDays(7)); + when(mapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(entity); + + Optional result = service.findActiveByPlaintext("mc_valid_plaintext"); + + assertTrue(result.isPresent()); + assertEquals(42L, result.get().getId()); + } + + // ── recordUse debounce predicate (pure logic) ───────────────────────── + + @Test + @DisplayName("shouldRecordUse — null lastUsedAt returns true (first write always proceeds)") + void shouldRecordUseFirstCall() { + assertTrue(PersonalAccessTokenService.shouldRecordUse(null, LocalDateTime.now())); + } + + @Test + @DisplayName("shouldRecordUse — within 60s of last write returns false (debounced)") + void shouldRecordUseDebounced() { + LocalDateTime now = LocalDateTime.now(); + // 30s ago — well within the 60s window. + assertFalse(PersonalAccessTokenService.shouldRecordUse(now.minusSeconds(30), now)); + } + + @Test + @DisplayName("shouldRecordUse — after 60s window returns true (writes again)") + void shouldRecordUseAfterWindow() { + LocalDateTime now = LocalDateTime.now(); + // 2 min ago — beyond the 60s debounce. + assertTrue(PersonalAccessTokenService.shouldRecordUse(now.minusMinutes(2), now)); + } + + @Test + @DisplayName("shouldRecordUse — exactly at 60s boundary returns true") + void shouldRecordUseAtBoundary() { + LocalDateTime now = LocalDateTime.now(); + // 61s ago — just past the boundary. + assertTrue(PersonalAccessTokenService.shouldRecordUse(now.minusSeconds(61), now)); + } + + @Test + @DisplayName("recordUse — first write hits the mapper") + void recordUseFirstCallWrites() { + service.recordUse(entity); + verify(mapper, times(1)).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("recordUse — second call within debounce skips the mapper") + void recordUseDebouncedSkipsMapper() { + entity.setLastUsedAt(LocalDateTime.now()); + service.recordUse(entity); + verify(mapper, never()).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("recordUse swallows DB errors — never fails an authenticated request") + void recordUseSwallowsErrors() { + when(mapper.updateById(any(PersonalAccessTokenEntity.class))) + .thenThrow(new RuntimeException("simulated DB outage")); + // Must not throw — last-used is observability, not a correctness gate. + service.recordUse(entity); + } + + // ── revoke ownership ────────────────────────────────────────────────── + + @Test + @DisplayName("revoke with matching owner soft-deletes") + void revokeOwnedToken() { + when(mapper.selectById(42L)).thenReturn(entity); + when(mapper.updateById(any(PersonalAccessTokenEntity.class))).thenReturn(1); + + service.revoke(42L, 7L); + + verify(mapper, times(1)).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("revoke with wrong owner throws not-found — no info leak about token ownership") + void revokeWrongOwner() { + // Token exists but belongs to user 7, not 999. + when(mapper.selectById(42L)).thenReturn(entity); + + var ex = assertThrows(MateClawException.class, + () -> service.revoke(42L, 999L)); + assertTrue(ex.getMessage().contains("not found") || ex.getMessage().contains("not owned"), + "error message must indicate not-found, not 'unauthorized' — to avoid leaking which token ids exist"); + // Critically: must NOT have called updateById — owner check happens before any write. + verify(mapper, never()).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("revoke of missing token throws not-found") + void revokeMissingToken() { + when(mapper.selectById(99L)).thenReturn(null); + assertThrows(MateClawException.class, + () -> service.revoke(99L, 7L)); + verify(mapper, never()).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("revoke of already-deleted token throws not-found (no double-delete confusion)") + void revokeAlreadyDeletedToken() { + entity.setDeleted(1); + when(mapper.selectById(42L)).thenReturn(entity); + assertThrows(MateClawException.class, + () -> service.revoke(42L, 7L)); + verify(mapper, never()).updateById(any(PersonalAccessTokenEntity.class)); + } + + @Test + @DisplayName("create requires non-null userId") + void createRequiresUserId() { + assertThrows(MateClawException.class, + () -> service.create(null, "name", null, null)); + } + + @Test + @DisplayName("tokenHash never leaks via Jackson serialization (privacy regression guard)") + void tokenHashDoesNotLeakInJson() throws Exception { + // The list endpoint returns PersonalAccessTokenEntity directly to + // the client. Jackson must skip tokenHash even when other fields + // serialize normally — otherwise admin UI / log middleware leaks + // the per-token digest. Smoke test on 2026-05-02 caught this. + PersonalAccessTokenEntity e = new PersonalAccessTokenEntity(); + e.setId(123L); + e.setUserId(7L); + e.setName("ci-key"); + e.setTokenHash("8020f458548f7b433f872da4d6828933e4f3ba421823f3e7010c9ffd3c505f20"); + e.setScopes("*"); + e.setEnabled(true); + + String json = new ObjectMapper().writeValueAsString(e); + + assertFalse(json.contains("tokenHash"), + "tokenHash field name leaked to JSON: " + json); + assertFalse(json.contains("8020f458"), + "tokenHash value leaked to JSON: " + json); + // Sanity: other fields still serialize so we didn't accidentally + // @JsonIgnore the wrong field. + assertTrue(json.contains("\"name\":\"ci-key\"")); + assertTrue(json.contains("\"id\":123")); + } + + @Test + @DisplayName("create returns plaintext exactly once and inserts the row") + void createReturnsPlaintext() { + when(mapper.insert(any(PersonalAccessTokenEntity.class))).thenReturn(1); + PersonalAccessTokenService.CreatedToken result = service.create( + 7L, "ci-key", "*", LocalDateTime.now().plusDays(30)); + + assertNotNull(result); + assertNotNull(result.plaintext()); + assertTrue(result.plaintext().startsWith("mc_")); + assertNotNull(result.entity()); + // The row inserted into DB must NOT carry plaintext — only the hash. + assertFalse(result.plaintext().equals(result.entity().getTokenHash()), + "DB must store the hash, not the plaintext"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/ChannelErrorClassifierTest.java b/mateclaw-server/src/test/java/vip/mate/channel/ChannelErrorClassifierTest.java new file mode 100644 index 00000000..3de96592 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/ChannelErrorClassifierTest.java @@ -0,0 +1,58 @@ +package vip.mate.channel; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pin the error-detection heuristic so a future tweak in + * {@code NodeStreamingChatHelper} that renames an error template doesn't + * silently regress IM channels back into the self-replicating 400 loop. + */ +class ChannelErrorClassifierTest { + + private final ChannelErrorClassifier classifier = new ChannelErrorClassifier(); + + @Test + void normal_reply_is_not_error() { + assertFalse(classifier.isErrorReply("好的,我已经为您完成了任务。")); + assertFalse(classifier.isErrorReply("")); + assertFalse(classifier.isErrorReply(null)); + assertFalse(classifier.isErrorReply("⏰ 定时任务已就绪:每天 00:18")); + } + + @Test + void error_prefix_is_detected() { + assertTrue(classifier.isErrorReply("[错误] Bad request: Bad request, please check input")); + assertTrue(classifier.isErrorReply("[错误] 工具调用失败")); + } + + @Test + void error_substrings_emitted_by_NodeStreamingChatHelper_are_detected() { + // Mirrors templates in NodeStreamingChatHelper.buildErrorResultWithType + assertTrue(classifier.isErrorReply("Bad request: invalid_request_error")); + assertTrue(classifier.isErrorReply("LLM 调用失败: connection reset")); + assertTrue(classifier.isErrorReply("LLM 调用超时")); + assertTrue(classifier.isErrorReply("LLM 调用被中断")); + assertTrue(classifier.isErrorReply("Prompt 过长: token limit exceeded")); + assertTrue(classifier.isErrorReply("认证失败: 401 Unauthorized")); + assertTrue(classifier.isErrorReply("LLM 返回空响应")); + } + + @Test + void status_for_maps_correctly() { + assertEquals("error", classifier.statusFor("[错误] Bad request")); + assertEquals("completed", classifier.statusFor("Hello world")); + assertEquals("completed", classifier.statusFor("")); + } + + @Test + void aicard_partial_with_error_prefix_is_detected() { + // The DingTalk AICard catch path now wraps partial output with a + // [错误] prefix; verify the classifier catches that compound shape. + String reply = "[错误] AI Card streaming failed: timeout\n\n(已生成的部分内容,已忽略)\n部分回答 ..."; + assertTrue(classifier.isErrorReply(reply)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java b/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java new file mode 100644 index 00000000..3ede4388 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java @@ -0,0 +1,440 @@ +package vip.mate.channel; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.test.util.ReflectionTestUtils; +import vip.mate.channel.leader.ChannelLeaderElection; +import vip.mate.channel.leader.LeaderLease; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.service.ChannelService; +import vip.mate.exception.MateClawException; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.lang.reflect.Field; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Behavioural tests for the multi-instance reconciliation paths: + * heartbeat-driven detection of disabled / deleted / config-changed + * channels, and follower-retry cancellation on channel deletion. + * + *

These exercise the fixes that prevent a leader node from running + * stale config (or a deleted channel) just because the admin API call + * happened to land on a different node. + */ +class ChannelManagerReconcileTest { + + private ChannelService channelService; + private ChannelLeaderElection election; + private ChannelManager manager; + private TrackingAdapter adapter; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + channelService = mock(ChannelService.class); + election = mock(ChannelLeaderElection.class); + manager = new ChannelManager( + channelService, + mock(ChannelMessageRouter.class), + mock(ChannelSessionStore.class), + new ObjectMapper(), + mock(vip.mate.tool.document.GeneratedFileCache.class), + mock(vip.mate.channel.notification.ApprovalNotificationService.class), + mock(vip.mate.channel.wecom.cards.WeComCardDispatcher.class), + mock(vip.mate.channel.wecom.WeComKeepaliveScheduler.class), + election); + adapter = new TrackingAdapter(); + } + + @AfterEach + void tearDown() { + // Shut down the leaderScheduler so test threads don't leak. + manager.destroy(); + } + + @Test + @DisplayName("heartbeat reconcile: disabled channel triggers local stop") + void reconcileStopsOnDisabled() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(42L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + ChannelEntity disabled = entity(42L, "feishu"); + disabled.setEnabled(false); + disabled.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 0)); + when(channelService.getChannel(42L)).thenReturn(disabled); + + manager.reconcileChannel(42L, "test-channel"); + + assertFalse(manager.getAdapter(42L).isPresent(), + "Disabled channel detected via reconciliation must stop local adapter"); + assertTrue(adapter.stopped.get(), "Adapter stop() must be invoked"); + verify(lease, times(1)).release(); + } + + @Test + @DisplayName("heartbeat reconcile: not-found exception triggers local stop and lease release") + void reconcileStopsOnNotFound() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(43L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + when(channelService.getChannel(43L)) + .thenThrow(new MateClawException("err.channel.not_found", "渠道不存在: 43")); + + manager.reconcileChannel(43L, "test-channel"); + + assertFalse(manager.getAdapter(43L).isPresent(), + "Deleted channel detected via reconciliation must stop local adapter"); + assertTrue(adapter.stopped.get()); + verify(lease, times(1)).release(); + } + + @Test + @DisplayName("heartbeat reconcile: transient DB error keeps adapter running (no false-positive stop)") + void reconcileKeepsRunningOnTransientFailure() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(44L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + when(channelService.getChannel(44L)).thenThrow(new RuntimeException("connection refused")); + + manager.reconcileChannel(44L, "test-channel"); + + assertTrue(manager.getAdapter(44L).isPresent(), + "Transient lookup errors must not stop the local adapter"); + assertFalse(adapter.stopped.get()); + verify(lease, never()).release(); + } + + @Test + @DisplayName("config change to non-leader-required mode releases the lease (e.g. Feishu WS → webhook)") + void modeFlipOutOfLeaderRequiredReleasesLease() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(50L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + // Stub createAdapter so that the swap doesn't need a real + // network-backed Feishu/Telegram start(). The fresh adapter + // reports requiresSingleLeader=false, simulating a mode flip. + TrackingAdapter newAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + ChannelManager spied = spy(manager); + doReturn(newAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + ChannelEntity updated = entity(50L, "feishu"); + updated.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + + spied.applyConfigChange(50L, updated); + + verify(lease, times(1)).release(); + // The new (non-leader) adapter is started locally on this node. + assertTrue(spied.getAdapter(50L).isPresent(), + "After mode flip, the local node continues running the channel as a non-leader"); + assertTrue(newAdapter.started.get(), "New adapter must be started after flip"); + assertTrue(adapter.stopped.get(), "Old adapter must be stopped before swap"); + } + + @Test + @DisplayName("config change within leader-required mode preserves the lease (in-place swap)") + void inPlaceSwapPreservesLease() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(51L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + TrackingAdapter newAdapter = new TrackingAdapter(); // requiresSingleLeader=true + ChannelManager spied = spy(manager); + doReturn(newAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + ChannelEntity updated = entity(51L, "feishu"); + updated.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + + spied.applyConfigChange(51L, updated); + + verify(lease, never()).release(); + assertTrue(spied.getAdapter(51L).isPresent()); + assertTrue(newAdapter.started.get()); + assertTrue(adapter.stopped.get()); + } + + @Test + @DisplayName("follower retry: not-found cancels the scheduled retry (no leak)") + void followerRetryCancelsOnNotFound() { + // Seed a follower retry future so we can verify cancellation. + ScheduledFuture future = mock(ScheduledFuture.class); + @SuppressWarnings("unchecked") + Map> followerRetries = + (Map>) ReflectionTestUtils.getField(manager, "followerRetryFutures"); + assertNotNull(followerRetries); + followerRetries.put(99L, future); + + when(channelService.getChannel(99L)) + .thenThrow(new MateClawException("err.channel.not_found", "渠道不存在: 99")); + + manager.followerRetry(99L); + + assertFalse(manager.hasFollowerRetry(99L), + "Deleted channel must cancel the follower retry future"); + verify(future, times(1)).cancel(false); + } + + @Test + @DisplayName("follower retry: transient DB error keeps retry scheduled (no false-positive cancel)") + void followerRetryKeepsOnTransientFailure() { + ScheduledFuture future = mock(ScheduledFuture.class); + @SuppressWarnings("unchecked") + Map> followerRetries = + (Map>) ReflectionTestUtils.getField(manager, "followerRetryFutures"); + followerRetries.put(100L, future); + + when(channelService.getChannel(100L)).thenThrow(new RuntimeException("connection refused")); + + manager.followerRetry(100L); + + assertTrue(manager.hasFollowerRetry(100L), + "Transient lookup errors must not cancel the follower retry"); + verify(future, never()).cancel(any(Boolean.class)); + } + + @Test + @DisplayName("non-leader reconcile: disabled channel on another node triggers local stop") + void nonLeaderReconcileStopsOnDisabled() { + // Seed a non-leader active adapter (no lease, no heartbeat) — this is + // the state a Feishu-webhook or Telegram-webhook node lives in. + TrackingAdapter webhookAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + seedNonLeaderState(60L, webhookAdapter, LocalDateTime.of(2026, 1, 1, 0, 0)); + + ChannelEntity disabled = entity(60L, "feishu"); + disabled.setEnabled(false); + disabled.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 0)); + when(channelService.getChannel(60L)).thenReturn(disabled); + + manager.reconcileChannel(60L, "webhook-channel"); + + assertFalse(manager.getAdapter(60L).isPresent(), + "Non-leader reconcile must stop the local adapter when admin disables on another node"); + assertTrue(webhookAdapter.stopped.get()); + } + + @Test + @DisplayName("non-leader → leader-required flip: winner becomes leader (no direct start without election)") + void nonLeaderToLeaderFlipWinsElection() { + // Seed a non-leader webhook adapter (no lease). + TrackingAdapter webhookAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + seedNonLeaderState(80L, webhookAdapter, LocalDateTime.of(2026, 1, 1, 0, 0)); + + // New config flips into leader-required mode. + TrackingAdapter wsAdapter = new TrackingAdapter(); // requiresSingleLeader=true + ChannelManager spied = spy(manager); + doReturn(wsAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + // This node wins the election. + LeaderLease lease = mock(LeaderLease.class); + when(election.tryAcquire(anyString())).thenReturn(Optional.of(lease)); + + ChannelEntity flipped = entity(80L, "feishu"); + flipped.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + + spied.applyConfigChange(80L, flipped); + + assertTrue(webhookAdapter.stopped.get(), "Old non-leader adapter must be stopped"); + assertTrue(wsAdapter.started.get(), "New leader-required adapter starts only after we won the election"); + // Lease is recorded so the heartbeat can extend it. + @SuppressWarnings("unchecked") + Map leases = + (Map) ReflectionTestUtils.getField(spied, "activeLeases"); + assertSame(lease, leases.get(80L), "Won lease must be tracked under the channel id"); + verify(election, times(1)).tryAcquire("feishu:80"); + } + + @Test + @DisplayName("non-leader → leader-required flip: loser does NOT start the new adapter and enters follower retry") + void nonLeaderToLeaderFlipLosesElection() { + TrackingAdapter webhookAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + seedNonLeaderState(81L, webhookAdapter, LocalDateTime.of(2026, 1, 1, 0, 0)); + + TrackingAdapter wsAdapter = new TrackingAdapter(); + ChannelManager spied = spy(manager); + doReturn(wsAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + // Another node already holds the lease. + when(election.tryAcquire(anyString())).thenReturn(Optional.empty()); + + ChannelEntity flipped = entity(81L, "feishu"); + flipped.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + + spied.applyConfigChange(81L, flipped); + + assertTrue(webhookAdapter.stopped.get(), "Old non-leader adapter must be stopped"); + assertFalse(wsAdapter.started.get(), + "Loser must NOT call newAdapter.start() — that would open a duplicate WS bypassing the leader gate"); + assertFalse(spied.getAdapter(81L).isPresent()); + assertTrue(spied.hasFollowerRetry(81L), "Loser must enter follower retry to take over if the current leader dies"); + verify(election, times(1)).tryAcquire("feishu:81"); + } + + @Test + @DisplayName("non-leader reconcile: config change on another node propagates via stop+start") + void nonLeaderReconcileAppliesConfigUpdate() { + TrackingAdapter oldAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + seedNonLeaderState(61L, oldAdapter, LocalDateTime.of(2026, 1, 1, 0, 0)); + + TrackingAdapter newAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + ChannelManager spied = spy(manager); + doReturn(newAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + ChannelEntity updated = entity(61L, "feishu"); + updated.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + when(channelService.getChannel(61L)).thenReturn(updated); + + spied.reconcileChannel(61L, "webhook-channel"); + + assertTrue(oldAdapter.stopped.get(), "Old non-leader adapter must be stopped"); + assertTrue(newAdapter.started.get(), "New non-leader adapter must be started"); + assertTrue(spied.getAdapter(61L).isPresent()); + } + + @Test + @DisplayName("restartChannel: when we hold a lease and new mode is non-leader, release the lease via stop+start") + void restartChannelDetectsLeaseFlip() { + LeaderLease lease = mock(LeaderLease.class); + seedLeaderState(70L, adapter, lease, LocalDateTime.of(2026, 1, 1, 0, 0)); + + // Make createAdapter produce a non-leader adapter for the restart. + // Without the lease-aware check, restartChannel would take the + // hot-swap path and leave the lease, heartbeat, and + // lastSeenChannelUpdateTime around to be cleaned up only by the + // next heartbeat tick — causing an additional restart. + TrackingAdapter newAdapter = new TrackingAdapter() { + @Override public boolean requiresSingleLeader() { return false; } + }; + ChannelManager spied = spy(manager); + doReturn(newAdapter).when(spied).createAdapter(any(ChannelEntity.class)); + + ChannelEntity updated = entity(70L, "feishu"); + updated.setUpdateTime(LocalDateTime.of(2026, 1, 1, 0, 5)); + when(channelService.getChannel(70L)).thenReturn(updated); + + spied.restartChannel(70L); + + verify(lease, times(1)).release(); + @SuppressWarnings("unchecked") + Map leases = + (Map) ReflectionTestUtils.getField(spied, "activeLeases"); + assertFalse(leases.containsKey(70L), "Lease must be removed from activeLeases after the mode flip"); + assertTrue(adapter.stopped.get()); + assertTrue(newAdapter.started.get()); + } + + @Test + @DisplayName("stopAll releases plugin leases and cancels plugin heartbeats (no leak on shutdown)") + @SuppressWarnings("unchecked") + void stopAllReleasesPluginLeases() { + ChannelAdapter pluginAdapter = mock(ChannelAdapter.class); + when(pluginAdapter.getChannelType()).thenReturn("custom-im"); + when(pluginAdapter.getDisplayName()).thenReturn("custom-im"); + LeaderLease pluginLease = mock(LeaderLease.class); + ScheduledFuture pluginHeartbeat = mock(ScheduledFuture.class); + + Map pluginChannels = + (Map) ReflectionTestUtils.getField(manager, "pluginChannels"); + Map pluginLeases = + (Map) ReflectionTestUtils.getField(manager, "pluginLeases"); + Map> pluginHeartbeats = + (Map>) ReflectionTestUtils.getField(manager, "pluginHeartbeatFutures"); + pluginChannels.put("my-plugin", pluginAdapter); + pluginLeases.put("my-plugin", pluginLease); + pluginHeartbeats.put("my-plugin", pluginHeartbeat); + + manager.stopAll(); + + verify(pluginAdapter, times(1)).stop(); + verify(pluginLease, times(1)).release(); + verify(pluginHeartbeat, times(1)).cancel(false); + assertTrue(pluginChannels.isEmpty()); + assertTrue(pluginLeases.isEmpty()); + assertTrue(pluginHeartbeats.isEmpty()); + } + + // ==================== helpers ==================== + + private ChannelEntity entity(Long id, String type) { + ChannelEntity e = new ChannelEntity(); + e.setId(id); + e.setName("test-" + id); + e.setChannelType(type); + e.setEnabled(true); + return e; + } + + private void seedLeaderState(Long id, ChannelAdapter adapter, LeaderLease lease, + LocalDateTime updateTime) { + @SuppressWarnings("unchecked") + Map active = + (Map) ReflectionTestUtils.getField(manager, "activeAdapters"); + @SuppressWarnings("unchecked") + Map leases = + (Map) ReflectionTestUtils.getField(manager, "activeLeases"); + active.put(id, adapter); + leases.put(id, lease); + lastSeenMap().put(id, updateTime); + } + + /** + * Seed a non-leader-required active adapter: appears in + * {@code activeAdapters} and {@code lastSeenChannelUpdateTime}, but + * no entry in {@code activeLeases} / {@code heartbeatFutures} (those + * only exist for leader-required modes). + */ + private void seedNonLeaderState(Long id, ChannelAdapter adapter, LocalDateTime updateTime) { + @SuppressWarnings("unchecked") + Map active = + (Map) ReflectionTestUtils.getField(manager, "activeAdapters"); + active.put(id, adapter); + lastSeenMap().put(id, updateTime); + } + + @SuppressWarnings("unchecked") + private Map lastSeenMap() { + return (Map) ReflectionTestUtils.getField(manager, "lastSeenChannelUpdateTime"); + } + + /** + * Minimal adapter that records start/stop without opening any + * upstream connection — the tests need observable state, not real + * IM behavior. + */ + private static class TrackingAdapter implements ChannelAdapter { + final AtomicBoolean started = new AtomicBoolean(false); + final AtomicBoolean stopped = new AtomicBoolean(false); + + @Override public void start() { started.set(true); } + @Override public void stop() { stopped.set(true); } + @Override public boolean isRunning() { return started.get() && !stopped.get(); } + @Override public void onMessage(ChannelMessage message) {} + @Override public void sendMessage(String targetId, String content) {} + @Override public void sendContentParts(String targetId, List parts) {} + @Override public String getChannelType() { return "feishu"; } + @Override public boolean requiresSingleLeader() { return true; } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterDebounceTest.java b/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterDebounceTest.java new file mode 100644 index 00000000..6e9528ca --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterDebounceTest.java @@ -0,0 +1,70 @@ +package vip.mate.channel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pin the adaptive-debounce thresholds in + * {@link ChannelMessageRouter#pickDebounceMs(int)}. + * + *

The router merges same-conversation messages within a debounce window + * before forwarding to the agent. The default {@link + * ChannelMessageRouter#DEBOUNCE_MS} (500ms) is right for normal chatting + * but too short for IM clients that silently split long pasted prompts + * across multiple frames — the second fragment can arrive 1-2 seconds + * after the first, missing the window. When merged content crosses + * {@link ChannelMessageRouter#LONG_TEXT_THRESHOLD} the merger switches to + * {@link ChannelMessageRouter#LONG_DEBOUNCE_MS} so it has time to absorb + * the rest. These tests document that boundary behavior so future tuning + * is intentional rather than incidental. + */ +class ChannelMessageRouterDebounceTest { + + @Test + @DisplayName("short messages keep the 500ms default debounce") + void shortMessagesKeepDefaultDebounce() { + assertEquals(ChannelMessageRouter.DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(0)); + assertEquals(ChannelMessageRouter.DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(50)); + assertEquals(ChannelMessageRouter.DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(500)); + assertEquals(ChannelMessageRouter.DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(ChannelMessageRouter.LONG_TEXT_THRESHOLD)); + } + + @Test + @DisplayName("crossing the threshold flips to the extended 2.5s window") + void longContentTriggersLongDebounce() { + assertEquals(ChannelMessageRouter.LONG_DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(ChannelMessageRouter.LONG_TEXT_THRESHOLD + 1)); + assertEquals(ChannelMessageRouter.LONG_DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(2000)); + assertEquals(ChannelMessageRouter.LONG_DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(6000)); + assertEquals(ChannelMessageRouter.LONG_DEBOUNCE_MS, + ChannelMessageRouter.pickDebounceMs(Integer.MAX_VALUE)); + } + + @Test + @DisplayName("threshold + windows are sane: long > default, threshold below typical IM split") + void thresholdsAreSane() { + // The whole point — the extended window must actually be larger, + // otherwise the adaptive branch is a no-op. + assertTrue(ChannelMessageRouter.LONG_DEBOUNCE_MS > ChannelMessageRouter.DEBOUNCE_MS, + "extended debounce must exceed default"); + // Threshold sits below the typical ~2000-char WeCom client split + // point; if it ever crept above 2000 the merger would never + // engage on a real paste-split. + assertTrue(ChannelMessageRouter.LONG_TEXT_THRESHOLD < 2000, + "threshold must stay under the IM client's split point"); + // And well above any normally typed message — typing 1500+ chars + // in one bubble is extremely rare. Guards against accidentally + // applying the long-debounce penalty to ordinary chats. + assertTrue(ChannelMessageRouter.LONG_TEXT_THRESHOLD >= 1000, + "threshold must be high enough that typing doesn't trip it"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterGroupAttributionTest.java b/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterGroupAttributionTest.java new file mode 100644 index 00000000..feb047ee --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/ChannelMessageRouterGroupAttributionTest.java @@ -0,0 +1,153 @@ +package vip.mate.channel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the group-chat sender-attribution contract. + * + *

In groups, three users sharing one conversation send overlapping + * questions. Without {@code [@sender]} tags, the persisted history + * collapses into an unattributed wall of "user:" turns and the LLM can + * no longer tell who asked what. Without per-sender debounce boundaries, + * a paste-split fragment from user A can also accidentally absorb user + * B's text and mis-attribute it. These tests pin both contracts so a + * future refactor can't silently regress group multi-user usability. + */ +class ChannelMessageRouterGroupAttributionTest { + + private static ChannelMessage groupMessage(String senderId, String senderName, String content) { + return ChannelMessage.builder() + .channelType("wecom") + .senderId(senderId) + .senderName(senderName) + .chatId("group-abc") // chatId set ⇒ group context + .content(content) + .build(); + } + + private static ChannelMessage singleMessage(String senderId, String content) { + return ChannelMessage.builder() + .channelType("wecom") + .senderId(senderId) + .senderName(senderId) + .chatId(null) // chatId null ⇒ 1:1 chat + .content(content) + .build(); + } + + // ===== buildGroupTag ===== + + @Test + @DisplayName("single chat (chatId null) → no tag, no behavior change") + void singleChatNoTag() { + assertNull(ChannelMessageRouter.buildGroupTag(singleMessage("alice", "hi"))); + } + + @Test + @DisplayName("group chat with senderName → [@senderName] tag") + void groupWithSenderName() { + ChannelMessage m = groupMessage("alice-id", "Alice Wang", "hi"); + assertEquals("[@Alice Wang]", ChannelMessageRouter.buildGroupTag(m)); + } + + @Test + @DisplayName("group chat falls back to senderId when senderName is blank") + void groupFallsBackToSenderId() { + ChannelMessage m = groupMessage("alice-id", "", "hi"); + assertEquals("[@alice-id]", ChannelMessageRouter.buildGroupTag(m)); + } + + @Test + @DisplayName("group chat with no resolvable identity returns null (don't fabricate a tag)") + void groupNoIdentity() { + ChannelMessage m = ChannelMessage.builder() + .channelType("wecom") + .chatId("group-abc") + .content("hi") + .build(); + // Both senderId and senderName are null. Better to skip attribution + // than to invent "[@null]" which would corrupt the prompt. + assertNull(ChannelMessageRouter.buildGroupTag(m)); + } + + @Test + @DisplayName("blank chatId is treated as not-a-group") + void blankChatIdNotAGroup() { + ChannelMessage m = ChannelMessage.builder() + .channelType("wecom") + .senderId("alice") + .senderName("Alice") + .chatId(" ") + .content("hi") + .build(); + assertNull(ChannelMessageRouter.buildGroupTag(m)); + } + + // ===== applyGroupTag ===== + + @Test + @DisplayName("applyGroupTag: single chat content passes through verbatim") + void applyTagSingleChatPassesThrough() { + ChannelMessage m = singleMessage("alice", "hello world"); + assertEquals("hello world", + ChannelMessageRouter.applyGroupTag(m, "hello world")); + } + + @Test + @DisplayName("applyGroupTag: group content gets [@sender] prefix") + void applyTagGroupPrefixes() { + ChannelMessage m = groupMessage("alice-id", "Alice", "hello world"); + assertEquals("[@Alice] hello world", + ChannelMessageRouter.applyGroupTag(m, "hello world")); + } + + @Test + @DisplayName("applyGroupTag: idempotent — already-prefixed content is not double-tagged") + void applyTagIdempotent() { + ChannelMessage m = groupMessage("alice-id", "Alice", "ignored"); + // Simulates a code path that has already attributed the content + // (e.g. a future channel adapter that pre-tags inbound text). + assertEquals("[@Alice] hello", + ChannelMessageRouter.applyGroupTag(m, "[@Alice] hello")); + } + + @Test + @DisplayName("applyGroupTag: empty content stays empty (no bare-tag artifact)") + void applyTagEmptyStaysEmpty() { + ChannelMessage m = groupMessage("alice-id", "Alice", ""); + // A truly empty message (no text, no parts producing text) shouldn't + // surface as a useless "[@Alice]" turn — the agent has nothing to + // act on. Skip the tag to keep persisted history clean. + assertEquals("", ChannelMessageRouter.applyGroupTag(m, "")); + assertNull(ChannelMessageRouter.applyGroupTag(m, null)); + } + + // ===== isSameSender (the merge boundary helper) ===== + + @Test + @DisplayName("isSameSender: same sender → merge allowed (paste-split / rapid follow-up)") + void sameSenderMergeAllowed() { + assertTrue(ChannelMessageRouter.isSameSender("alice", "alice")); + } + + @Test + @DisplayName("isSameSender: different sender → no merge (group sender boundary)") + void differentSenderNoMerge() { + // The whole point of the group fix: A's pending must NOT absorb B's + // text, otherwise the merged buffer attributes both to A. + assertFalse(ChannelMessageRouter.isSameSender("alice", "bob")); + } + + @Test + @DisplayName("isSameSender: null on either side → no merge (defensive)") + void nullSendersNoMerge() { + // Pending fixtures occasionally have null senderIds; better to start + // a fresh pending than to silently merge into an unidentified buffer. + assertFalse(ChannelMessageRouter.isSameSender(null, "alice")); + assertFalse(ChannelMessageRouter.isSameSender("alice", null)); + assertFalse(ChannelMessageRouter.isSameSender(null, null)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/MediaPathGuardTest.java b/mateclaw-server/src/test/java/vip/mate/channel/MediaPathGuardTest.java new file mode 100644 index 00000000..e8e63893 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/MediaPathGuardTest.java @@ -0,0 +1,212 @@ +package vip.mate.channel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * RFC-03 Lane K3 — covers {@link MediaPathGuard} validation rules. + * + *

Each test reads as a property of "what every channel adapter must + * agree on": no path traversal, no implicit directory writes, no + * surprise extensions, no DoS via giant files. Failures bind to the + * stable {@link MediaPathGuard.Reason} codes so audit / metrics + * downstream can group violations without parsing message text. + */ +class MediaPathGuardTest { + + private static MediaPathGuard.Policy policy(Path workspace) { + return new MediaPathGuard.Policy( + workspace, + Set.of("png", "jpg", "pdf", "txt"), + 10 * 1024 * 1024 // 10 MiB + ); + } + + // ── Containment ─────────────────────────────────────────────────────── + + @Test + @DisplayName("file under workspace root → returns canonical path") + void fileInWorkspaceAccepted(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("hello.txt"); + Files.writeString(file, "hi"); + + Path resolved = MediaPathGuard.validate(file, policy(workspace)); + + assertNotNull(resolved); + assertEquals(workspace.toRealPath(), resolved.getParent()); + } + + @Test + @DisplayName("file outside workspace via ../ → PATH_OUTSIDE_WORKSPACE") + void traversalRejected(@TempDir Path workspace, @TempDir Path elsewhere) throws Exception { + Path victim = elsewhere.resolve("secret.txt"); + Files.writeString(victim, "top secret"); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(victim, policy(workspace))); + assertEquals(MediaPathGuard.Reason.PATH_OUTSIDE_WORKSPACE, ex.reason()); + } + + @Test + @DisplayName("workspace prefix-name overlap not exploitable — /ws-foo doesn't match /ws-foobar") + void prefixOverlapIsNotContainment(@TempDir Path tmp) throws Exception { + // /tmp/ws/ ← workspace + // /tmp/wsbig/file.txt ← attacker file with a similar prefix + Path workspace = tmp.resolve("ws"); + Path neighbor = tmp.resolve("wsbig"); + Files.createDirectory(workspace); + Files.createDirectory(neighbor); + Path attackerFile = neighbor.resolve("file.txt"); + Files.writeString(attackerFile, "x"); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(attackerFile, policy(workspace))); + assertEquals(MediaPathGuard.Reason.PATH_OUTSIDE_WORKSPACE, ex.reason(), + "Path.startsWith must compare elements, not strings — otherwise wsbig looks like a child of ws"); + } + + @Test + @DisplayName("missing file → FILE_MISSING (not opaque IO_ERROR)") + void missingFile(@TempDir Path workspace) { + Path nope = workspace.resolve("does-not-exist.txt"); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(nope, policy(workspace))); + assertEquals(MediaPathGuard.Reason.FILE_MISSING, ex.reason(), + "missing files have a dedicated reason so audit output isn't misleading"); + } + + // ── Type ────────────────────────────────────────────────────────────── + + @Test + @DisplayName("directory passed in place of file → NOT_A_REGULAR_FILE") + void directoryRejected(@TempDir Path workspace) throws Exception { + Path subdir = workspace.resolve("subdir"); + Files.createDirectory(subdir); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(subdir, policy(workspace))); + assertEquals(MediaPathGuard.Reason.NOT_A_REGULAR_FILE, ex.reason()); + } + + // ── Extension ───────────────────────────────────────────────────────── + + @Test + @DisplayName("extension allowlist case-insensitive") + void extensionCaseInsensitive(@TempDir Path workspace) throws Exception { + Path uppercaseExt = workspace.resolve("HelloWorld.PNG"); + Files.write(uppercaseExt, new byte[]{0x1a}); + + // Should pass — policy allows "png" lowercase, file has "PNG". + Path ok = MediaPathGuard.validate(uppercaseExt, policy(workspace)); + assertNotNull(ok); + } + + @Test + @DisplayName("policy allowlist normalizes leading-dot extensions") + void allowlistAcceptsDotPrefix(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("a.txt"); + Files.writeString(file, "x"); + + // Same policy, but the dev specified ".txt" instead of "txt" — both work. + var p = new MediaPathGuard.Policy(workspace, Set.of(".txt"), 1024L); + assertNotNull(MediaPathGuard.validate(file, p)); + } + + @Test + @DisplayName("extension not in allowlist → EXTENSION_NOT_ALLOWED") + void extensionNotAllowed(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("malware.exe"); + Files.write(file, new byte[]{0x4d, 0x5a}); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(file, policy(workspace))); + assertEquals(MediaPathGuard.Reason.EXTENSION_NOT_ALLOWED, ex.reason()); + } + + @Test + @DisplayName("file with no extension → EXTENSION_NOT_ALLOWED (defensive)") + void noExtension(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("README"); + Files.writeString(file, "doc"); + + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(file, policy(workspace))); + assertEquals(MediaPathGuard.Reason.EXTENSION_NOT_ALLOWED, ex.reason()); + } + + // ── Size ────────────────────────────────────────────────────────────── + + @Test + @DisplayName("file at policy size cap is accepted (boundary)") + void atCapAccepted(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("at-cap.txt"); + Files.write(file, new byte[100]); + + var p = new MediaPathGuard.Policy(workspace, Set.of("txt"), 100L); + Path ok = MediaPathGuard.validate(file, p); + assertNotNull(ok); + } + + @Test + @DisplayName("file 1 byte over cap → FILE_TOO_LARGE") + void overCapRejected(@TempDir Path workspace) throws Exception { + Path file = workspace.resolve("big.txt"); + Files.write(file, new byte[101]); + + var p = new MediaPathGuard.Policy(workspace, Set.of("txt"), 100L); + var ex = assertThrows(MediaPathGuard.MediaValidationException.class, + () -> MediaPathGuard.validate(file, p)); + assertEquals(MediaPathGuard.Reason.FILE_TOO_LARGE, ex.reason()); + } + + // ── Returned canonical path ─────────────────────────────────────────── + + @Test + @DisplayName("returned path is canonical (toRealPath) — TOCTOU-safe for downstream callers") + void canonicalPathReturned(@TempDir Path workspace) throws Exception { + // Use a relative path that resolves to a real file via . segment — + // the returned value must drop the redundant segment. + Path realFile = workspace.resolve("real.png"); + Files.write(realFile, new byte[]{1}); + Path withDotSegment = workspace.resolve(".").resolve("real.png"); + + Path canonical = MediaPathGuard.validate(withDotSegment, policy(workspace)); + + assertEquals(realFile.toRealPath(), canonical); + assertNotEquals(withDotSegment, canonical, + "validate must return the canonical form, not echo the user-supplied path verbatim"); + } + + // ── Policy guards ───────────────────────────────────────────────────── + + @Test + @DisplayName("non-positive maxBytes → IllegalArgumentException at policy construction") + void zeroMaxBytesRejected(@TempDir Path workspace) { + assertThrows(IllegalArgumentException.class, + () -> new MediaPathGuard.Policy(workspace, Set.of("txt"), 0L)); + assertThrows(IllegalArgumentException.class, + () -> new MediaPathGuard.Policy(workspace, Set.of("txt"), -100L)); + } + + @Test + @DisplayName("extensionOf — public-internals helper coverage") + void extensionOfHelper() { + assertEquals("png", MediaPathGuard.extensionOf(Path.of("a.png"))); + assertEquals("png", MediaPathGuard.extensionOf(Path.of("PATH/a.PNG"))); + assertEquals("", MediaPathGuard.extensionOf(Path.of("README"))); + assertEquals("", MediaPathGuard.extensionOf(Path.of("trailing."))); + assertEquals("gz", MediaPathGuard.extensionOf(Path.of("archive.tar.gz")), + "double-dot filenames should report the rightmost segment"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuTextSplitTest.java b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuTextSplitTest.java new file mode 100644 index 00000000..236fd870 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuTextSplitTest.java @@ -0,0 +1,100 @@ +package vip.mate.channel.feishu; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class FeishuTextSplitTest { + + @Test + @DisplayName("short content returns single-element list unchanged") + void shortContent() { + List chunks = FeishuChannelAdapter.splitTextForFeishu("hello world", 100); + assertEquals(List.of("hello world"), chunks); + } + + @Test + @DisplayName("null/empty returns empty list") + void nullEmpty() { + assertTrue(FeishuChannelAdapter.splitTextForFeishu(null, 100).isEmpty()); + assertTrue(FeishuChannelAdapter.splitTextForFeishu("", 100).isEmpty()); + } + + @Test + @DisplayName("split prefers paragraph (\\n\\n) boundary over hard cut") + void prefersParagraphBoundary() { + String content = "first paragraph here\n\nsecond paragraph here"; + // maxChars chosen so the paragraph boundary lands in the second half + List chunks = FeishuChannelAdapter.splitTextForFeishu(content, 30); + assertEquals(2, chunks.size()); + assertEquals("first paragraph here\n\n", chunks.get(0)); + assertEquals("second paragraph here", chunks.get(1)); + } + + @Test + @DisplayName("split falls through to line boundary when no paragraph break") + void fallsThroughToLine() { + String content = "line1 with content\nline2 with content\nline3 with content"; + List chunks = FeishuChannelAdapter.splitTextForFeishu(content, 25); + assertTrue(chunks.size() >= 2); + // each chunk should end at \n boundary or be the final chunk + for (int i = 0; i < chunks.size() - 1; i++) { + assertTrue(chunks.get(i).endsWith("\n"), + "Non-final chunk should end at line boundary: '" + chunks.get(i) + "'"); + } + assertEquals(content, String.join("", chunks), + "Concatenation must reconstruct the original"); + } + + @Test + @DisplayName("oversized single line falls through to whitespace boundary") + void fallsThroughToWhitespace() { + String content = "word ".repeat(200); // 1000 chars, no \n + List chunks = FeishuChannelAdapter.splitTextForFeishu(content, 100); + assertTrue(chunks.size() >= 10); + for (String chunk : chunks) { + assertTrue(chunk.length() <= 100, + "Each chunk must be within limit, got " + chunk.length()); + } + assertEquals(content, String.join("", chunks)); + } + + @Test + @DisplayName("zero-boundary content (no spaces, no newlines) hard-cuts") + void hardCutWhenNoBoundary() { + String content = "x".repeat(1000); + List chunks = FeishuChannelAdapter.splitTextForFeishu(content, 250); + assertEquals(4, chunks.size()); + for (String chunk : chunks) { + assertEquals(250, chunk.length()); + } + assertEquals(content, String.join("", chunks)); + } + + @Test + @DisplayName("default 4000-char limit holds for very long markdown answer") + void realisticLongAnswer() { + // Simulate a 12K-char LLM answer with paragraph breaks every ~400 chars + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 30; i++) { + sb.append("段落 ").append(i).append(":") + .append("这里是一些内容,模拟一个真实的长回答。".repeat(10)) + .append("\n\n"); + } + String content = sb.toString(); + + List chunks = FeishuChannelAdapter.splitTextForFeishu( + content, FeishuChannelAdapter.MAX_TEXT_MESSAGE_CHARS); + assertTrue(chunks.size() >= 2, + "12K-char answer should split into multiple chunks, got " + chunks.size()); + for (String chunk : chunks) { + assertTrue(chunk.length() <= FeishuChannelAdapter.MAX_TEXT_MESSAGE_CHARS, + "Chunk exceeds limit: " + chunk.length()); + } + assertEquals(content, String.join("", chunks), + "Reconstruction lossless"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/leader/ChannelLeaderElectionTest.java b/mateclaw-server/src/test/java/vip/mate/channel/leader/ChannelLeaderElectionTest.java new file mode 100644 index 00000000..55c00e99 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/leader/ChannelLeaderElectionTest.java @@ -0,0 +1,124 @@ +package vip.mate.channel.leader; + +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockProvider; +import net.javacrumbs.shedlock.core.SimpleLock; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.time.Duration; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +class ChannelLeaderElectionTest { + + @Test + @DisplayName("tryAcquire returns empty when LockProvider rejects (another node owns the lock)") + void tryAcquireWhenLockHeldElsewhere() { + LockProvider provider = mock(LockProvider.class); + when(provider.lock(any(LockConfiguration.class))).thenReturn(Optional.empty()); + + ChannelLeaderElection election = new ChannelLeaderElection(provider); + Optional lease = election.tryAcquire("feishu:42"); + + assertTrue(lease.isEmpty(), "Expected empty optional when lock is held elsewhere"); + } + + @Test + @DisplayName("tryAcquire returns a lease when LockProvider grants the lock") + void tryAcquireWhenLockGranted() { + LockProvider provider = mock(LockProvider.class); + SimpleLock simpleLock = mock(SimpleLock.class); + when(provider.lock(any(LockConfiguration.class))).thenReturn(Optional.of(simpleLock)); + + ChannelLeaderElection election = new ChannelLeaderElection(provider); + Optional lease = election.tryAcquire("qq:7"); + + assertTrue(lease.isPresent()); + assertEquals("channel-leader:qq:7", lease.get().getName()); + } + + @Test + @DisplayName("Lock name is prefixed so leases don't collide with other ShedLock users (e.g. cron)") + void lockNameIsPrefixed() { + LockProvider provider = mock(LockProvider.class); + SimpleLock simpleLock = mock(SimpleLock.class); + when(provider.lock(any(LockConfiguration.class))).thenReturn(Optional.of(simpleLock)); + + ChannelLeaderElection election = new ChannelLeaderElection(provider); + election.tryAcquire("feishu:42"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(LockConfiguration.class); + verify(provider).lock(captor.capture()); + assertEquals("channel-leader:feishu:42", captor.getValue().getName()); + } + + @Test + @DisplayName("Lease extend() reports success when ShedLock returns a new SimpleLock") + void leaseExtendSuccess() { + SimpleLock current = mock(SimpleLock.class); + SimpleLock next = mock(SimpleLock.class); + when(current.extend(any(Duration.class), any(Duration.class))).thenReturn(Optional.of(next)); + + LeaderLease lease = new LeaderLease("test", current); + assertTrue(lease.extend(Duration.ofSeconds(60))); + } + + @Test + @DisplayName("Lease extend() reports failure when ShedLock returns empty (lock lost)") + void leaseExtendLost() { + SimpleLock current = mock(SimpleLock.class); + when(current.extend(any(Duration.class), any(Duration.class))).thenReturn(Optional.empty()); + + LeaderLease lease = new LeaderLease("test", current); + assertFalse(lease.extend(Duration.ofSeconds(60))); + } + + @Test + @DisplayName("Lease extend() swallows exceptions and reports failure — heartbeats must not crash the scheduler") + void leaseExtendCatchesException() { + SimpleLock current = mock(SimpleLock.class); + when(current.extend(any(Duration.class), any(Duration.class))) + .thenThrow(new RuntimeException("db connection lost")); + + LeaderLease lease = new LeaderLease("test", current); + assertFalse(lease.extend(Duration.ofSeconds(60))); + } + + @Test + @DisplayName("Lease release() unlocks the underlying SimpleLock exactly once even if called twice") + void leaseReleaseIdempotent() { + SimpleLock simpleLock = mock(SimpleLock.class); + LeaderLease lease = new LeaderLease("test", simpleLock); + + lease.release(); + lease.release(); + + verify(simpleLock, times(1)).unlock(); + } + + @Test + @DisplayName("Lease release() swallows unlock exceptions so shutdown can't be blocked by them") + void leaseReleaseCatchesException() { + SimpleLock simpleLock = mock(SimpleLock.class); + doThrow(new RuntimeException("db gone")).when(simpleLock).unlock(); + + LeaderLease lease = new LeaderLease("test", simpleLock); + assertDoesNotThrow(lease::release); + } + + @Test + @DisplayName("Once released, extend() always returns false without touching the underlying lock") + void extendAfterReleaseIsFalse() { + SimpleLock simpleLock = mock(SimpleLock.class); + LeaderLease lease = new LeaderLease("test", simpleLock); + + lease.release(); + assertFalse(lease.extend(Duration.ofSeconds(60))); + verify(simpleLock, never()).extend(any(Duration.class), any(Duration.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/leader/SingleLeaderHookTest.java b/mateclaw-server/src/test/java/vip/mate/channel/leader/SingleLeaderHookTest.java new file mode 100644 index 00000000..95a63def --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/leader/SingleLeaderHookTest.java @@ -0,0 +1,151 @@ +package vip.mate.channel.leader; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.discord.DiscordChannelAdapter; +import vip.mate.channel.feishu.FeishuChannelAdapter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.qq.QQChannelAdapter; +import vip.mate.channel.telegram.TelegramChannelAdapter; +import vip.mate.channel.wecom.WeComChannelAdapter; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * Behavioural test for the {@code requiresSingleLeader()} hook on the + * Feishu and QQ adapters. This is what gates leader election in + * {@code ChannelManager}, so a regression that flips the answer would + * silently re-introduce the multi-instance connection-limit bug. + */ +class SingleLeaderHookTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + private final ChannelMessageRouter router = mock(ChannelMessageRouter.class); + + private ChannelEntity channel(String type, String configJson) { + ChannelEntity e = new ChannelEntity(); + e.setId(1L); + e.setName("test"); + e.setChannelType(type); + e.setConfigJson(configJson); + e.setEnabled(true); + return e; + } + + @Test + @DisplayName("Feishu in WebSocket mode requires single leader (default mode)") + void feishuWebsocketRequiresLeader() { + FeishuChannelAdapter adapter = new FeishuChannelAdapter( + channel("feishu", "{\"app_id\":\"x\",\"app_secret\":\"y\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader(), + "Default Feishu mode is websocket and must require single leader"); + } + + @Test + @DisplayName("Feishu explicitly set to websocket mode requires single leader") + void feishuExplicitWebsocketRequiresLeader() { + FeishuChannelAdapter adapter = new FeishuChannelAdapter( + channel("feishu", + "{\"app_id\":\"x\",\"app_secret\":\"y\",\"connection_mode\":\"websocket\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader()); + } + + @Test + @DisplayName("Feishu in webhook mode does NOT require single leader (load-balanced HTTP)") + void feishuWebhookDoesNotRequireLeader() { + FeishuChannelAdapter adapter = new FeishuChannelAdapter( + channel("feishu", + "{\"app_id\":\"x\",\"app_secret\":\"y\",\"connection_mode\":\"webhook\"}"), + router, objectMapper); + assertFalse(adapter.requiresSingleLeader(), + "Webhook callbacks are HTTP-fanned by the LB, so all nodes may subscribe"); + } + + @Test + @DisplayName("QQ always requires single leader (gateway rejects duplicate IDENTIFY)") + void qqAlwaysRequiresLeader() { + QQChannelAdapter adapter = new QQChannelAdapter( + channel("qq", "{\"app_id\":\"x\",\"client_secret\":\"y\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader()); + } + + @Test + @DisplayName("WeCom always requires single leader (WS-only aibot transport)") + void wecomAlwaysRequiresLeader() { + WeComChannelAdapter adapter = new WeComChannelAdapter( + channel("wecom", "{\"bot_id\":\"x\",\"secret\":\"y\"}"), + router, objectMapper, + mock(vip.mate.channel.notification.ApprovalNotificationService.class), + mock(vip.mate.channel.wecom.cards.WeComCardDispatcher.class), + mock(vip.mate.channel.wecom.WeComKeepaliveScheduler.class)); + assertTrue(adapter.requiresSingleLeader()); + } + + @Test + @DisplayName("Discord always requires single leader (Gateway WS, 1 session per token)") + void discordAlwaysRequiresLeader() { + DiscordChannelAdapter adapter = new DiscordChannelAdapter( + channel("discord", "{\"bot_token\":\"x\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader()); + } + + @Test + @DisplayName("Telegram in long-polling mode requires single leader (default mode)") + void telegramPollingRequiresLeader() { + TelegramChannelAdapter adapter = new TelegramChannelAdapter( + channel("telegram", "{\"bot_token\":\"x\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader(), + "Default Telegram mode is long-polling and must require single leader"); + } + + @Test + @DisplayName("Telegram explicitly set to polling requires single leader") + void telegramExplicitPollingRequiresLeader() { + TelegramChannelAdapter adapter = new TelegramChannelAdapter( + channel("telegram", + "{\"bot_token\":\"x\",\"connection_mode\":\"polling\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader()); + } + + @Test + @DisplayName("Telegram in webhook mode (explicit + url) does NOT require single leader") + void telegramExplicitWebhookDoesNotRequireLeader() { + TelegramChannelAdapter adapter = new TelegramChannelAdapter( + channel("telegram", + "{\"bot_token\":\"x\",\"connection_mode\":\"webhook\",\"webhook_url\":\"https://example.com/hook\"}"), + router, objectMapper); + assertFalse(adapter.requiresSingleLeader(), + "Webhook callbacks are HTTP-fanned by the LB, so all nodes may subscribe"); + } + + @Test + @DisplayName("Telegram legacy config (no connection_mode + webhook_url set) infers webhook → no leader") + void telegramLegacyWebhookInferredDoesNotRequireLeader() { + TelegramChannelAdapter adapter = new TelegramChannelAdapter( + channel("telegram", + "{\"bot_token\":\"x\",\"webhook_url\":\"https://example.com/hook\"}"), + router, objectMapper); + assertFalse(adapter.requiresSingleLeader(), + "Legacy config without connection_mode but with webhook_url must be inferred as webhook"); + } + + @Test + @DisplayName("Telegram connection_mode=webhook but webhook_url blank falls back to polling → leader required") + void telegramWebhookWithoutUrlIsPolling() { + TelegramChannelAdapter adapter = new TelegramChannelAdapter( + channel("telegram", + "{\"bot_token\":\"x\",\"connection_mode\":\"webhook\"}"), + router, objectMapper); + assertTrue(adapter.requiresSingleLeader(), + "Webhook mode without a URL falls through to polling and must require single leader"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/verifier/ChannelVerifierTest.java b/mateclaw-server/src/test/java/vip/mate/channel/verifier/ChannelVerifierTest.java new file mode 100644 index 00000000..3117d815 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/verifier/ChannelVerifierTest.java @@ -0,0 +1,132 @@ +package vip.mate.channel.verifier; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for the wizard preflight path. We only exercise the + * fail-fast branches that need no network (missing credentials), since + * the happy paths require live upstream services. End-to-end coverage + * happens in the nightly integration job described in RFC-084 §8. + */ +class ChannelVerifierTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void resultHelpers_buildExpectedShapes() { + VerificationResult ok = VerificationResult.ok(42, "hi", Map.of("a", 1)); + assertTrue(ok.ok()); + assertFalse(ok.skipped()); + assertEquals("hi", ok.headline()); + assertEquals(1, ok.identity().get("a")); + + VerificationResult bad = VerificationResult.failed(7, "nope", "token", "fix it"); + assertFalse(bad.ok()); + assertEquals("token", bad.invalidField()); + assertEquals("fix it", bad.hint()); + assertTrue(bad.identity().isEmpty()); + + VerificationResult skipped = VerificationResult.skipped("nothing to verify"); + assertTrue(skipped.ok()); + assertTrue(skipped.skipped()); + } + + @Test + void telegramVerifier_failsFast_withoutToken() { + TelegramVerifier verifier = new TelegramVerifier(objectMapper); + VerificationResult r = verifier.verify(new VerificationRequest( + "telegram", Collections.emptyMap(), 1L)); + assertFalse(r.ok()); + assertEquals("bot_token", r.invalidField()); + // Should not reach the network — duration is 0. + assertEquals(0, r.durationMs()); + } + + @Test + void discordVerifier_failsFast_withoutToken() { + DiscordVerifier verifier = new DiscordVerifier(objectMapper); + VerificationResult r = verifier.verify(new VerificationRequest( + "discord", Collections.emptyMap(), 1L)); + assertFalse(r.ok()); + assertEquals("bot_token", r.invalidField()); + assertEquals(0, r.durationMs()); + } + + @Test + void slackVerifier_failsFast_withoutToken() { + SlackVerifier verifier = new SlackVerifier(); + VerificationResult r = verifier.verify(new VerificationRequest( + "slack", Collections.emptyMap(), 1L)); + assertFalse(r.ok()); + assertEquals("bot_token", r.invalidField()); + } + + @Test + void wecomVerifier_failsFast_withoutCredentials() { + WeComVerifier verifier = new WeComVerifier(objectMapper); + VerificationResult missingBoth = verifier.verify(new VerificationRequest( + "wecom", Collections.emptyMap(), 1L)); + assertFalse(missingBoth.ok()); + assertEquals("bot_id", missingBoth.invalidField()); + assertEquals(0, missingBoth.durationMs()); + + VerificationResult missingSecret = verifier.verify(new VerificationRequest( + "wecom", Map.of("bot_id", "bot_xxxxx"), 1L)); + assertFalse(missingSecret.ok()); + assertEquals("secret", missingSecret.invalidField()); + } + + @Test + void feishuVerifier_failsFast_withoutCredentials() { + FeishuVerifier verifier = new FeishuVerifier(objectMapper); + VerificationResult missingAppId = verifier.verify(new VerificationRequest( + "feishu", Collections.emptyMap(), 1L)); + assertFalse(missingAppId.ok()); + assertEquals("app_id", missingAppId.invalidField()); + + VerificationResult missingSecret = verifier.verify(new VerificationRequest( + "feishu", Map.of("app_id", "cli_xxxxx"), 1L)); + assertFalse(missingSecret.ok()); + assertEquals("app_secret", missingSecret.invalidField()); + } + + @Test + void dingtalkVerifier_failsFast_withoutCredentials() { + DingTalkVerifier verifier = new DingTalkVerifier(objectMapper); + VerificationResult missingClientId = verifier.verify(new VerificationRequest( + "dingtalk", Collections.emptyMap(), 1L)); + assertFalse(missingClientId.ok()); + assertEquals("client_id", missingClientId.invalidField()); + + VerificationResult missingSecret = verifier.verify(new VerificationRequest( + "dingtalk", Map.of("client_id", "dingxxxxxxxx"), 1L)); + assertFalse(missingSecret.ok()); + assertEquals("client_secret", missingSecret.invalidField()); + } + + @Test + void weixinVerifier_failsFast_withoutToken() { + WeixinVerifier verifier = new WeixinVerifier(objectMapper); + VerificationResult r = verifier.verify(new VerificationRequest( + "weixin", Collections.emptyMap(), 1L)); + assertFalse(r.ok()); + assertEquals("bot_token", r.invalidField()); + } + + @Test + void registry_indexesByChannelType_andLetsLastWin() { + TelegramVerifier first = new TelegramVerifier(objectMapper); + TelegramVerifier second = new TelegramVerifier(objectMapper); + ChannelVerifierRegistry registry = new ChannelVerifierRegistry(java.util.List.of(first, second)); + registry.index(); + assertTrue(registry.find("telegram").isPresent()); + assertSame(second, registry.find("telegram").orElseThrow()); + assertTrue(registry.find("nonexistent").isEmpty()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/web/ChatControllerPersistStatusTest.java b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatControllerPersistStatusTest.java new file mode 100644 index 00000000..1249680f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatControllerPersistStatusTest.java @@ -0,0 +1,91 @@ +package vip.mate.channel.web; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.model.MessageEntity; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Truth-table for {@link ChatController#derivePersistStatus} (RFC-067 §4.6). + *

+ * The pre-RFC controller had three different inline derivations across normal / + * queued / replay {@code doOnComplete}; queued and replay hardcoded + * {@code completed} which caused {@code awaiting_approval} turns to be silently + * downgraded — the frontend would then call {@code expirePendingApprovals} and + * ghost-clear the banner. These tests pin the unified five-way truth table so a + * future refactor can't reintroduce the divergence. + */ +class ChatControllerPersistStatusTest { + + @Test + @DisplayName("awaiting_approval wins over every other condition (top of priority)") + void awaitingApprovalTakesPrecedence() { + // Even when stop+error fired AFTER the approval gate, persistence must + // still surface awaiting_approval — the turn isn't truly finished and + // the frontend's done handler skips expire on this status. + assertThat(ChatController.derivePersistStatus(true, true, true, + ChatStreamTracker.InterruptType.USER_INTERRUPT_WITH_FOLLOWUP)) + .isEqualTo("awaiting_approval"); + assertThat(ChatController.derivePersistStatus(true, false, false, null)) + .isEqualTo("awaiting_approval"); + } + + @Test + @DisplayName("error beats every non-approval state") + void errorOnTypedErrorPrefix() { + assertThat(ChatController.derivePersistStatus(false, true, false, null)) + .isEqualTo("error"); + // Stop coexists with error → still error (BaseAgent sanitization needs to + // skip these regardless of whether the user pressed Stop afterward). + assertThat(ChatController.derivePersistStatus(false, true, true, + ChatStreamTracker.InterruptType.USER_STOP)) + .isEqualTo("error"); + } + + @Test + @DisplayName("clean finish → completed") + void completedOnCleanFinish() { + assertThat(ChatController.derivePersistStatus(false, false, false, null)) + .isEqualTo("completed"); + } + + @Test + @DisplayName("user-stop without follow-up → stopped") + void stoppedOnPlainStop() { + assertThat(ChatController.derivePersistStatus(false, false, true, + ChatStreamTracker.InterruptType.USER_STOP)) + .isEqualTo("stopped"); + // Null InterruptType (defensive) also collapses to stopped — mirrors + // historical behavior where wasStopped+null was the common shape on + // doOnCancel paths before the typed enum landed. + assertThat(ChatController.derivePersistStatus(false, false, true, null)) + .isEqualTo("stopped"); + } + + @Test + @DisplayName("user interrupt-with-followup → interrupted (queued message takes over)") + void interruptedOnFollowupQueue() { + assertThat(ChatController.derivePersistStatus(false, false, true, + ChatStreamTracker.InterruptType.USER_INTERRUPT_WITH_FOLLOWUP)) + .isEqualTo("interrupted"); + } + + @Test + @DisplayName("empty completed turns persist an explicit placeholder") + void emptyCompletedTurnUsesPlaceholder() { + assertThat(ChatController.emptyAssistantPlaceholder("completed")) + .isEqualTo("[本次没有输出]"); + assertThat(ChatController.emptyAssistantPlaceholder("awaiting_approval")) + .isEqualTo("[等待审批]"); + } + + @Test + @DisplayName("done.persisted reflects whether an assistant row was actually saved") + void donePersistedFollowsSavedAssistant() { + MessageEntity saved = new MessageEntity(); + + assertThat(ChatController.isAssistantPersisted(saved)).isTrue(); + assertThat(ChatController.isAssistantPersisted(null)).isFalse(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerBatchedRelayTest.java b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerBatchedRelayTest.java new file mode 100644 index 00000000..54e1e168 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerBatchedRelayTest.java @@ -0,0 +1,146 @@ +package vip.mate.channel.web; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for {@link ChatStreamTracker#addBatchedEventRelay}: size-driven flush, + * time-driven flush, and pass-through ordering for non-batched events. + */ +class ChatStreamTrackerBatchedRelayTest { + + private ChatStreamTracker newTracker() { + return new ChatStreamTracker(new ObjectMapper()); + } + + private record Captured(String name, String json) {} + + /** + * Spin until {@code condition} is true or {@code timeoutMs} elapses. + * Polling instead of {@code Awaitility} to keep the test classpath + * dependency-free (the project doesn't bundle Awaitility). + */ + private static boolean waitUntil(java.util.function.BooleanSupplier condition, long timeoutMs) { + long deadline = System.currentTimeMillis() + timeoutMs; + while (System.currentTimeMillis() < deadline) { + if (condition.getAsBoolean()) return true; + try { + Thread.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return false; + } + } + return condition.getAsBoolean(); + } + + @Test + @DisplayName("Buffer flushes when batch size threshold is hit") + void flushAtBatchSize() { + ChatStreamTracker tracker = newTracker(); + String src = "src-batch-size"; + tracker.register(src); + List captured = new CopyOnWriteArrayList<>(); + + Runnable deregister = tracker.addBatchedEventRelay(src, "parent", + 3, 5_000L, // batch=3, flushMs large so the timer never fires + (name, json) -> captured.add(new Captured(name, json))); + try { + tracker.broadcast(src, "tool_call_started", "{\"name\":\"a\"}"); + tracker.broadcast(src, "tool_call_completed", "{\"name\":\"a\",\"ok\":true}"); + assertTrue(captured.isEmpty(), "Below batch threshold — no flush yet"); + + tracker.broadcast(src, "tool_call_started", "{\"name\":\"b\"}"); + // 3rd buffered event triggers immediate flush. + assertTrue(waitUntil(() -> !captured.isEmpty(), 1_000), + "Flush should happen at batch threshold"); + List snapshot = new ArrayList<>(captured); + assertEquals(1, snapshot.size(), + "Single delegation_batch envelope expected at the size threshold"); + assertEquals("delegation_batch", snapshot.get(0).name()); + assertTrue(snapshot.get(0).json().contains("delegation_batch")); + } finally { + deregister.run(); + } + } + + @Test + @DisplayName("Buffer flushes at the elapsed-time boundary") + void flushOnTimer() { + ChatStreamTracker tracker = newTracker(); + String src = "src-batch-time"; + tracker.register(src); + List captured = new CopyOnWriteArrayList<>(); + + Runnable deregister = tracker.addBatchedEventRelay(src, "parent", + 100, 200L, // huge batch, short timer + (name, json) -> captured.add(new Captured(name, json))); + try { + tracker.broadcast(src, "tool_call_started", "{\"name\":\"a\"}"); + tracker.broadcast(src, "tool_call_completed", "{\"name\":\"a\",\"ok\":true}"); + // Wait for the scheduler to fire (200ms + slack). + assertTrue(waitUntil(() -> !captured.isEmpty(), 2_000), + "Time-driven flush expected within 2s"); + assertEquals("delegation_batch", captured.get(0).name(), + "Time-driven flush must produce a delegation_batch envelope"); + } finally { + deregister.run(); + } + } + + @Test + @DisplayName("Pass-through events fire immediately and preserve ordering") + void passThroughPreservesOrdering() { + ChatStreamTracker tracker = newTracker(); + String src = "src-pass-through"; + tracker.register(src); + List captured = new CopyOnWriteArrayList<>(); + + Runnable deregister = tracker.addBatchedEventRelay(src, "parent", + 100, 5_000L, // size and time thresholds both far away + (name, json) -> captured.add(new Captured(name, json))); + try { + // Two batchable events buffer up. + tracker.broadcast(src, "tool_call_started", "{\"name\":\"a\"}"); + tracker.broadcast(src, "tool_call_completed", "{\"name\":\"a\",\"ok\":true}"); + // Pass-through event: must flush prior buffer, then fire itself. + tracker.broadcast(src, "phase", "{\"phase\":\"reasoning\"}"); + + assertTrue(waitUntil(() -> captured.size() >= 2, 2_000)); + // Order: delegation_batch (drained buffer) then phase. + assertEquals("delegation_batch", captured.get(0).name(), + "Pass-through must drain buffered events first"); + assertEquals("phase", captured.get(1).name(), + "Pass-through event must follow the flushed batch"); + } finally { + deregister.run(); + } + } + + @Test + @DisplayName("Deregister flushes any pending events before unsubscribing") + void deregisterFlushesPending() { + ChatStreamTracker tracker = newTracker(); + String src = "src-shutdown"; + tracker.register(src); + AtomicInteger sawBatch = new AtomicInteger(0); + Runnable deregister = tracker.addBatchedEventRelay(src, "parent", + 100, 60_000L, + (name, json) -> { + if ("delegation_batch".equals(name)) sawBatch.incrementAndGet(); + }); + tracker.broadcast(src, "tool_call_started", "{}"); + tracker.broadcast(src, "tool_call_completed", "{}"); + deregister.run(); + assertEquals(1, sawBatch.get(), + "Deregistration must drain pending events as one final batch"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerChunkedBroadcastTest.java b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerChunkedBroadcastTest.java new file mode 100644 index 00000000..152bb6c9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/web/ChatStreamTrackerChunkedBroadcastTest.java @@ -0,0 +1,185 @@ +package vip.mate.channel.web; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.web.servlet.mvc.method.annotation.ResponseBodyEmitter; +import org.springframework.web.servlet.mvc.method.annotation.SseEmitter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Verifies that {@link ChatStreamTracker#broadcastChunked} splits oversize + * payloads into ordered {@code tool_result_chunk} events with the documented + * envelope shape, and leaves small payloads intact. + */ +class ChatStreamTrackerChunkedBroadcastTest { + + private ChatStreamTracker newTracker() { + return new ChatStreamTracker(new ObjectMapper()); + } + + /** + * Captures (eventName, jsonData) tuples by intercepting Spring's + * {@code send(SseEventBuilder)} path. The builder emits multiple + * "event:..." / "data:..." entries when rendered to a Set, so we walk + * the rendered set once and collect a single logical pair. + */ + private static final class CapturingEmitter extends SseEmitter { + final List> events = new CopyOnWriteArrayList<>(); + + CapturingEmitter() { + super(60_000L); + } + + @Override + public void send(SseEventBuilder builder) throws IOException { + Set entries = builder.build(); + // Spring renders the SSE event as: + // 1) header string: "event:\ndata:" (note: data: prefix + // already attached, no payload yet) + // 2) the actual data object (here always a JSON string) + // 3) terminator string: "\n\n" + // We detect the header from prefix scanning, then take the next + // String entry as the payload. Anything else is ignored. + String name = null; + String payload = null; + boolean expectPayload = false; + for (ResponseBodyEmitter.DataWithMediaType d : entries) { + Object obj = d.getData(); + if (!(obj instanceof String text)) continue; + if (text.contains("event:") && text.contains("data:")) { + int evStart = text.indexOf("event:") + "event:".length(); + int evEnd = text.indexOf('\n', evStart); + if (evEnd < 0) evEnd = text.length(); + name = text.substring(evStart, evEnd).trim(); + expectPayload = true; + } else if (expectPayload && payload == null && !text.equals("\n\n")) { + payload = text; + expectPayload = false; + } + } + Map entry = new LinkedHashMap<>(); + entry.put("event", name != null ? name : ""); + entry.put("data", payload != null ? payload : ""); + events.add(entry); + } + } + + @Test + @DisplayName("Small payload broadcasts as a single event unchanged") + void smallPayloadBroadcastsUnchanged() { + ChatStreamTracker tracker = newTracker(); + String cid = "small-payload"; + tracker.register(cid); + CapturingEmitter emitter = new CapturingEmitter(); + tracker.attach(cid, emitter); + + Map payload = new LinkedHashMap<>(); + payload.put("toolCallId", "call-1"); + payload.put("toolName", "echo"); + payload.put("result", "small text"); + payload.put("success", true); + + tracker.broadcastChunked(cid, "tool_call_completed", payload, "call-1"); + + long completedCount = emitter.events.stream() + .filter(e -> "tool_call_completed".equals(e.get("event"))).count(); + long chunkCount = emitter.events.stream() + .filter(e -> "tool_result_chunk".equals(e.get("event"))).count(); + assertEquals(1, completedCount, "Expected single tool_call_completed event"); + assertEquals(0, chunkCount, "No chunk events for small payload"); + } + + @Test + @DisplayName("Large payload splits into ordered tool_result_chunk events with final flag") + void largePayloadChunksAndTerminates() throws Exception { + ChatStreamTracker tracker = newTracker(); + String cid = "large-payload"; + tracker.register(cid); + CapturingEmitter emitter = new CapturingEmitter(); + tracker.attach(cid, emitter); + + // Build a result well above CHUNK_SIZE (8192 bytes) so the splitter + // produces multiple chunks. 30 KiB ensures at least 4 splits even + // after envelope overhead. + StringBuilder big = new StringBuilder(30_000); + for (int i = 0; i < 3000; i++) { + big.append("0123456789"); + } + Map payload = new LinkedHashMap<>(); + payload.put("toolCallId", "call-large"); + payload.put("toolName", "shell"); + payload.put("result", big.toString()); + payload.put("success", true); + + tracker.broadcastChunked(cid, "tool_call_completed", payload, "call-large"); + + // 1. Header event preserved with empty result + chunked=true. + List> completed = new ArrayList<>(); + List> chunks = new ArrayList<>(); + for (Map e : emitter.events) { + if ("tool_call_completed".equals(e.get("event"))) completed.add(e); + else if ("tool_result_chunk".equals(e.get("event"))) chunks.add(e); + } + assertEquals(1, completed.size(), "Header event should fire exactly once"); + assertTrue(chunks.size() >= 4, + "Expected several chunk events; got " + chunks.size()); + + ObjectMapper mapper = new ObjectMapper(); + Map header = mapper.readValue(completed.get(0).get("data"), Map.class); + assertEquals(Boolean.TRUE, header.get("chunked")); + assertEquals("call-large", header.get("chunkRef")); + assertEquals("", header.get("result"), + "Header must replace long field with empty placeholder"); + + // 2. Chunk envelope: kind / scope / ref / seq monotonic / final on last. + StringBuilder reconstructed = new StringBuilder(); + for (int i = 0; i < chunks.size(); i++) { + Map chunk = mapper.readValue(chunks.get(i).get("data"), Map.class); + assertEquals("tool_result", chunk.get("kind")); + assertEquals("parent", chunk.get("scope")); + assertEquals("call-large", chunk.get("ref")); + assertEquals(i, chunk.get("seq"), "Chunks must be in seq order"); + boolean isLast = (i == chunks.size() - 1); + assertEquals(isLast, chunk.get("final"), + "Only the last chunk should set final=true"); + reconstructed.append((String) chunk.get("delta")); + } + assertEquals(big.toString(), reconstructed.toString(), + "Concatenated chunks must reproduce the original result verbatim"); + } + + @Test + @DisplayName("Disabling chunked transport keeps single-event behavior") + void disabledChunkingIsPassThrough() { + ChatStreamTracker tracker = newTracker(); + tracker.setChunkedToolResultsEnabled(false); + String cid = "disabled"; + tracker.register(cid); + CapturingEmitter emitter = new CapturingEmitter(); + tracker.attach(cid, emitter); + + StringBuilder big = new StringBuilder(15_000); + for (int i = 0; i < 1500; i++) big.append("0123456789"); + Map payload = new LinkedHashMap<>(); + payload.put("toolCallId", "call-x"); + payload.put("toolName", "shell"); + payload.put("result", big.toString()); + payload.put("success", true); + + tracker.broadcastChunked(cid, "tool_call_completed", payload, "call-x"); + + long chunkCount = emitter.events.stream() + .filter(e -> "tool_result_chunk".equals(e.get("event"))).count(); + assertEquals(0, chunkCount, "Chunking disabled — must not emit chunk events"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/web/Utf8SseEmitterTest.java b/mateclaw-server/src/test/java/vip/mate/channel/web/Utf8SseEmitterTest.java new file mode 100644 index 00000000..cbfac003 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/web/Utf8SseEmitterTest.java @@ -0,0 +1,109 @@ +package vip.mate.channel.web; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.http.MediaType; +import org.springframework.http.server.ServerHttpResponse; +import org.springframework.http.server.ServletServerHttpResponse; +import org.springframework.mock.web.MockHttpServletResponse; + +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-058 PR-1: ensure {@link Utf8SseEmitter} explicitly stamps + * {@code Content-Type: text/event-stream;charset=UTF-8} on the response. + * + *

Spring's default {@link org.springframework.web.servlet.mvc.method.annotation.SseEmitter} + * leaves the charset off, which on Windows / GBK locale Chrome and through + * certain reverse proxies leads to mojibake for Chinese characters. + */ +class Utf8SseEmitterTest { + + @Test + @DisplayName("extendResponse stamps charset=UTF-8 when Content-Type is unset") + void stampsUtf8WhenContentTypeUnset() throws Exception { + Utf8SseEmitter emitter = new Utf8SseEmitter(10_000L); + MockHttpServletResponse servlet = new MockHttpServletResponse(); + ServerHttpResponse response = new ServletServerHttpResponse(servlet); + + invokeExtendResponse(emitter, response); + + MediaType contentType = response.getHeaders().getContentType(); + assertNotNull(contentType, "Content-Type must be set"); + assertEquals("text", contentType.getType()); + assertEquals("event-stream", contentType.getSubtype()); + assertEquals(StandardCharsets.UTF_8, contentType.getCharset(), + "charset must be explicitly UTF-8 (not null)"); + } + + @Test + @DisplayName("extendResponse stamps charset=UTF-8 when Content-Type lacks charset") + void stampsUtf8WhenContentTypeMissingCharset() throws Exception { + Utf8SseEmitter emitter = new Utf8SseEmitter(10_000L); + MockHttpServletResponse servlet = new MockHttpServletResponse(); + ServerHttpResponse response = new ServletServerHttpResponse(servlet); + // Simulate Spring default: text/event-stream WITHOUT charset + response.getHeaders().setContentType(MediaType.parseMediaType("text/event-stream")); + + invokeExtendResponse(emitter, response); + + MediaType contentType = response.getHeaders().getContentType(); + assertNotNull(contentType.getCharset(), "charset must be filled in"); + assertEquals(StandardCharsets.UTF_8, contentType.getCharset()); + } + + @Test + @DisplayName("extendResponse does NOT override an explicit non-UTF8 charset") + void doesNotClobberExplicitCharset() throws Exception { + Utf8SseEmitter emitter = new Utf8SseEmitter(10_000L); + MockHttpServletResponse servlet = new MockHttpServletResponse(); + ServerHttpResponse response = new ServletServerHttpResponse(servlet); + // Caller explicitly chose ISO-8859-1 — we must respect it + MediaType iso = new MediaType("text", "event-stream", StandardCharsets.ISO_8859_1); + response.getHeaders().setContentType(iso); + + invokeExtendResponse(emitter, response); + + MediaType contentType = response.getHeaders().getContentType(); + assertEquals(StandardCharsets.ISO_8859_1, contentType.getCharset(), + "Explicit caller-set charset must not be overridden"); + } + + @Test + @DisplayName("Utf8SseEmitter constructor accepts timeout like SseEmitter") + void constructorAcceptsTimeout() { + Utf8SseEmitter emitter = new Utf8SseEmitter(60_000L); + assertEquals(60_000L, emitter.getTimeout()); + } + + @Test + @DisplayName("Default constructor works (no timeout)") + void defaultConstructorWorks() { + Utf8SseEmitter emitter = new Utf8SseEmitter(); + assertNull(emitter.getTimeout(), "Default constructor leaves timeout null"); + } + + /** + * {@code extendResponse} is {@code protected} on the framework class. + * Reflection is the cleanest way to exercise it without spinning up a + * full DispatcherServlet for a one-line behavioural assertion. + */ + private static void invokeExtendResponse(Utf8SseEmitter emitter, ServerHttpResponse response) + throws Exception { + Method m = findExtendResponseMethod(emitter.getClass()); + m.setAccessible(true); + m.invoke(emitter, response); + } + + private static Method findExtendResponseMethod(Class cls) throws NoSuchMethodException { + for (Class c = cls; c != null; c = c.getSuperclass()) { + for (Method m : c.getDeclaredMethods()) { + if ("extendResponse".equals(m.getName())) return m; + } + } + throw new NoSuchMethodException("extendResponse not found on " + cls); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/AppmsgContentTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/AppmsgContentTest.java new file mode 100644 index 00000000..346ea21a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/AppmsgContentTest.java @@ -0,0 +1,205 @@ +package vip.mate.channel.wecom; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.notification.ApprovalNotificationService; +import vip.mate.channel.wecom.cards.WeComCardDispatcher; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin {@code msgtype=appmsg} parsing — covers the four sub-variants + * users actually forward to bots in production: PDF / Word / Excel + * (file), image cards, miniprograms, and public-account article links. + * + *

Without this branch, every forwarded PDF / article / miniprogram + * fell into the inbound switch's default and got silently dropped. + * These tests pin (1) the text marker shape so prompts stay stable, + * (2) the attached-media routing for file and image variants, and + * (3) the link/miniprogram fallbacks so the agent at least knows + * something was shared. + */ +class AppmsgContentTest { + + private WeComChannelAdapter adapter; + private Method extract; + + @BeforeEach + void setUp() throws Exception { + ChannelEntity entity = new ChannelEntity(); + entity.setId(1L); + entity.setChannelType("wecom"); + entity.setConfigJson("{\"media_download_enabled\": false}"); + adapter = new WeComChannelAdapter( + entity, + Mockito.mock(ChannelMessageRouter.class), + new ObjectMapper(), + Mockito.mock(ApprovalNotificationService.class), + Mockito.mock(WeComCardDispatcher.class), + Mockito.mock(WeComKeepaliveScheduler.class)); + extract = WeComChannelAdapter.class.getDeclaredMethod( + "extractAppmsgContent", + Map.class, String.class, String.class, String.class, String.class); + extract.setAccessible(true); + } + + @SuppressWarnings("unchecked") + private Object invoke(Map body) throws Exception { + return extract.invoke(adapter, body, "msg-1", "alice", "alice", "single"); + } + + private String text(Object ctx) throws Exception { + return (String) ctx.getClass().getMethod("text").invoke(ctx); + } + + @SuppressWarnings("unchecked") + private List parts(Object ctx) throws Exception { + return (List) ctx.getClass().getMethod("attachedParts").invoke(ctx); + } + + @Test + @DisplayName("appmsg.file → file content part + [文件: filename] marker") + void fileVariant() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "report.pdf", + "file", Map.of( + "url", "https://example.com/report.pdf", + "aeskey", "k", + "filename", "report.pdf")))); + assertEquals("[文件: report.pdf]", text(ctx)); + assertEquals(1, parts(ctx).size()); + assertEquals("file", parts(ctx).get(0).getType()); + } + + @Test + @DisplayName("appmsg.image → image content part + [图片: title] marker") + void imageVariant() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "周末聚会", + "image", Map.of( + "url", "https://example.com/photo.jpg", + "aeskey", "k")))); + assertEquals("[图片: 周末聚会]", text(ctx)); + assertEquals(1, parts(ctx).size()); + assertEquals("image", parts(ctx).get(0).getType()); + } + + @Test + @DisplayName("appmsg.image with no title → bare [图片] marker") + void imageVariantNoTitle() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "image", Map.of( + "url", "https://example.com/p.jpg", + "aeskey", "k")))); + assertEquals("[图片]", text(ctx)); + assertEquals(1, parts(ctx).size()); + } + + @Test + @DisplayName("appmsg.miniprogram → [小程序: title] marker, no attached media") + void miniprogramVariant() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "外卖小程序", + "miniprogram", Map.of("title", "美团外卖")))); + assertEquals("[小程序: 美团外卖]", text(ctx)); + assertTrue(parts(ctx).isEmpty()); + } + + @Test + @DisplayName("appmsg.miniprogram with no inner title falls back to top-level title") + void miniprogramTitleFallback() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "顶层标题", + "miniprogram", Map.of()))); + assertEquals("[小程序: 顶层标题]", text(ctx)); + } + + @Test + @DisplayName("appmsg.url (public-account article) → [链接] + title + desc + url multi-line + paste-body hint") + void linkVariant() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "深度好文:AI 的未来", + "description", "本文探讨 AI 在企业的落地路径", + "url", "https://mp.weixin.qq.com/s/abc123"))); + String t = text(ctx); + assertTrue(t.startsWith("[链接] 深度好文:AI 的未来"), + "title should follow [链接] tag; got: " + t); + assertTrue(t.contains("本文探讨 AI 在企业的落地路径"), + "description must be present; got: " + t); + assertTrue(t.contains("https://mp.weixin.qq.com/s/abc123"), + "URL must be in the text so agent can reference it; got: " + t); + // Public-account body is captcha-gated — agent must be told not to + // hallucinate content from the title. + assertTrue(t.contains("公众号文章"), + "public-account article hint must be appended; got: " + t); + assertTrue(t.contains("不要凭标题猜测内容"), + "directive against title-only guessing must be present; got: " + t); + assertTrue(parts(ctx).isEmpty(), "link variant produces no attached media"); + } + + @Test + @DisplayName("non-public-account links (regular URLs) do NOT get the paste-body hint") + void linkVariantNonWeixinUrlNoHint() throws Exception { + // Generic web links don't have the captcha-gate problem — fetching + // the body via a tool is straightforward, so adding the hint would + // be misleading. + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "GitHub README", + "url", "https://github.com/example/repo"))); + String t = text(ctx); + assertTrue(t.contains("https://github.com/example/repo")); + assertFalse(t.contains("公众号文章"), + "non-mp.weixin.qq.com URLs must not trigger the public-account hint; got: " + t); + } + + @Test + @DisplayName("link with title only, no description") + void linkVariantNoDesc() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "标题", + "url", "https://example.com"))); + String t = text(ctx); + assertTrue(t.contains("[链接] 标题")); + assertTrue(t.contains("https://example.com")); + } + + @Test + @DisplayName("unknown appmsg variant with title → [appmsg: title] marker") + void unknownVariantWithTitle() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "未知卡片", + "weird_field", Map.of()))); + assertEquals("[appmsg: 未知卡片]", text(ctx)); + } + + @Test + @DisplayName("totally empty appmsg → bare [appmsg] marker (agent at least knows something arrived)") + void emptyAppmsg() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of())); + assertEquals("[appmsg]", text(ctx)); + assertTrue(parts(ctx).isEmpty()); + } + + @Test + @DisplayName("file variant uses appmsg.title as filename when file.filename missing") + void fileFilenameFallback() throws Exception { + Object ctx = invoke(Map.of("appmsg", Map.of( + "title", "周报.docx", + "file", Map.of( + "url", "https://example.com/x", + "aeskey", "k")))); + // filename comes from title since file.filename is absent + assertTrue(text(ctx).contains("周报.docx"), + "marker should carry the title as filename; got: " + text(ctx)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/GroupReplyReqIdCacheTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/GroupReplyReqIdCacheTest.java new file mode 100644 index 00000000..2c52dcd3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/GroupReplyReqIdCacheTest.java @@ -0,0 +1,109 @@ +package vip.mate.channel.wecom; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.notification.ApprovalNotificationService; +import vip.mate.channel.wecom.cards.WeComCardDispatcher; + +import java.lang.reflect.Method; +import java.util.concurrent.ConcurrentHashMap; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the group-chat reply-slot cache contract. WeCom AI Bot platform + * blocks {@code aibot_send_msg} in group chats — proactive pushes (cron + * summaries, async-task forwards, image-generation completions) must + * ride {@code aibot_respond_msg} bound to a prior frame's reqId. + * + *

Without this cache, any group push silently failed: the test rig + * here exercises the cache plumbing directly so future changes to the + * cache eviction strategy or the lookup helper don't regress group + * delivery semantics. + */ +class GroupReplyReqIdCacheTest { + + private WeComChannelAdapter adapter; + + @BeforeEach + void setUp() { + ChannelEntity entity = new ChannelEntity(); + entity.setId(1L); + entity.setChannelType("wecom"); + entity.setConfigJson("{}"); + adapter = new WeComChannelAdapter( + entity, + Mockito.mock(ChannelMessageRouter.class), + new ObjectMapper(), + Mockito.mock(ApprovalNotificationService.class), + Mockito.mock(WeComCardDispatcher.class), + Mockito.mock(WeComKeepaliveScheduler.class)); + } + + @Test + @DisplayName("unknown chatId yields null — single chats fall through to aibot_send_msg") + void unknownChatYieldsNull() { + assertNull(adapter.pickGroupReplyReqId("never-seen-chat")); + assertNull(adapter.pickGroupReplyReqId("")); + assertNull(adapter.pickGroupReplyReqId(null)); + } + + @Test + @DisplayName("remembered group reqId is returned by the lookup") + void rememberAndLookup() throws Exception { + Method remember = WeComChannelAdapter.class.getDeclaredMethod( + "rememberGroupReplyReqId", String.class, String.class); + remember.setAccessible(true); + + remember.invoke(adapter, "group-1", "req-aaa"); + assertEquals("req-aaa", adapter.pickGroupReplyReqId("group-1")); + + // Most-recent semantics: a newer reqId for the same group overwrites. + remember.invoke(adapter, "group-1", "req-bbb"); + assertEquals("req-bbb", adapter.pickGroupReplyReqId("group-1")); + } + + @Test + @DisplayName("cache stays bounded under flood — no unbounded growth") + void cacheBounded() throws Exception { + Method remember = WeComChannelAdapter.class.getDeclaredMethod( + "rememberGroupReplyReqId", String.class, String.class); + remember.setAccessible(true); + + // Exceed the 1000-entry max with 1500 distinct groups. + for (int i = 0; i < 1500; i++) { + remember.invoke(adapter, "group-" + i, "req-" + i); + } + + // Inspect the underlying cache size via reflection. + java.lang.reflect.Field f = WeComChannelAdapter.class.getDeclaredField("lastChatReqIds"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + ConcurrentHashMap map = (ConcurrentHashMap) f.get(adapter); + assertTrue(map.size() <= 1000, + "cache must not grow beyond LAST_CHAT_REQ_IDS_MAX_SIZE; got " + map.size()); + } + + @Test + @DisplayName("each group gets independent reqId tracking — no cross-group leakage") + void independentPerGroup() throws Exception { + Method remember = WeComChannelAdapter.class.getDeclaredMethod( + "rememberGroupReplyReqId", String.class, String.class); + remember.setAccessible(true); + + remember.invoke(adapter, "group-A", "req-A1"); + remember.invoke(adapter, "group-B", "req-B1"); + assertEquals("req-A1", adapter.pickGroupReplyReqId("group-A")); + assertEquals("req-B1", adapter.pickGroupReplyReqId("group-B")); + + // Updating one doesn't affect the other. + remember.invoke(adapter, "group-A", "req-A2"); + assertEquals("req-A2", adapter.pickGroupReplyReqId("group-A")); + assertEquals("req-B1", adapter.pickGroupReplyReqId("group-B")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/QuoteContextTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/QuoteContextTest.java new file mode 100644 index 00000000..f48c2c80 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/QuoteContextTest.java @@ -0,0 +1,171 @@ +package vip.mate.channel.wecom; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.notification.ApprovalNotificationService; +import vip.mate.channel.wecom.cards.WeComCardDispatcher; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@code WeComChannelAdapter.extractQuoteContext} — the + * inbound quote-message parser that converts WeCom's {@code body.quote} + * field into a prefix string + attached media parts the agent can read. + * + *

Quoted-message context is the most common reason agent replies "go + * off-topic" on IM: the user long-presses a previous bubble, types a + * follow-up like "解释一下", and assumes the agent sees both. Without + * this parser the agent only saw the new text and silently lost the + * referenced content. + * + *

These tests pin (1) the prefix string shape so prompts stay stable + * across releases, (2) flattening rules for {@code mixed} quotes, and + * (3) the empty-result contract (null when nothing useful to extract) + * so the caller can treat null as "no quote context". + */ +class QuoteContextTest { + + private WeComChannelAdapter adapter; + private Method extract; + + @BeforeEach + void setUp() throws Exception { + ChannelEntity entity = new ChannelEntity(); + entity.setId(1L); + entity.setChannelType("wecom"); + entity.setConfigJson("{\"media_download_enabled\": false}"); // skip real downloads + adapter = new WeComChannelAdapter( + entity, + Mockito.mock(ChannelMessageRouter.class), + new ObjectMapper(), + Mockito.mock(ApprovalNotificationService.class), + Mockito.mock(WeComCardDispatcher.class), + Mockito.mock(WeComKeepaliveScheduler.class)); + extract = WeComChannelAdapter.class.getDeclaredMethod( + "extractQuoteContext", + Map.class, String.class, String.class, String.class, String.class); + extract.setAccessible(true); + } + + private Object invoke(Map body) throws Exception { + return extract.invoke(adapter, body, "msg-1", "alice", "alice", "single"); + } + + @Test + @DisplayName("missing quote field returns null") + void noQuote() throws Exception { + assertNull(invoke(Map.of())); + assertNull(invoke(Map.of("text", Map.of("content", "hi")))); + } + + @Test + @DisplayName("blank msgtype returns null (defensive)") + void blankQuoteType() throws Exception { + assertNull(invoke(Map.of("quote", Map.of("msgtype", "")))); + } + + @Test + @DisplayName("text quote produces a [引用消息: ...] prefix and no attached parts") + void textQuote() throws Exception { + Object ctx = invoke(Map.of("quote", Map.of( + "msgtype", "text", + "text", Map.of("content", "你好图片是什么意思")))); + assertNotNull(ctx); + // QuoteContext is a private record — exercise via reflection on accessor methods. + String prefix = (String) ctx.getClass().getMethod("prefix").invoke(ctx); + @SuppressWarnings("unchecked") + List parts = (List) + ctx.getClass().getMethod("attachedParts").invoke(ctx); + assertEquals("[引用消息: 你好图片是什么意思]\n", prefix); + assertTrue(parts.isEmpty(), "text-only quote attaches no media"); + } + + @Test + @DisplayName("image quote attaches a part and notes [图片] in prefix") + void imageQuote() throws Exception { + Object ctx = invoke(Map.of("quote", Map.of( + "msgtype", "image", + "image", Map.of( + "url", "https://example.com/x.jpg", + "aeskey", "k")))); + assertNotNull(ctx); + String prefix = (String) ctx.getClass().getMethod("prefix").invoke(ctx); + @SuppressWarnings("unchecked") + List parts = (List) + ctx.getClass().getMethod("attachedParts").invoke(ctx); + assertEquals("[引用消息: [图片]]\n", prefix); + assertEquals(1, parts.size()); + assertEquals("image", parts.get(0).getType()); + } + + @Test + @DisplayName("file quote uses the original filename in the prefix") + void fileQuote() throws Exception { + Object ctx = invoke(Map.of("quote", Map.of( + "msgtype", "file", + "file", Map.of( + "url", "https://example.com/x.pdf", + "filename", "report.pdf")))); + String prefix = (String) ctx.getClass().getMethod("prefix").invoke(ctx); + @SuppressWarnings("unchecked") + List parts = (List) + ctx.getClass().getMethod("attachedParts").invoke(ctx); + assertEquals("[引用消息: [文件: report.pdf]]\n", prefix); + assertEquals(1, parts.size()); + assertEquals("file", parts.get(0).getType()); + } + + @Test + @DisplayName("voice quote with ASR text gets surfaced; without ASR shows [语音消息]") + void voiceQuote() throws Exception { + Object withAsr = invoke(Map.of("quote", Map.of( + "msgtype", "voice", + "voice", Map.of("content", "明天开会")))); + assertEquals("[引用消息: [语音] 明天开会]\n", + withAsr.getClass().getMethod("prefix").invoke(withAsr)); + + Object empty = invoke(Map.of("quote", Map.of( + "msgtype", "voice", + "voice", Map.of("content", "")))); + assertEquals("[引用消息: [语音消息]]\n", + empty.getClass().getMethod("prefix").invoke(empty)); + } + + @Test + @DisplayName("mixed quote flattens to a space-joined summary and merges attached parts") + void mixedQuote() throws Exception { + Object ctx = invoke(Map.of("quote", Map.of( + "msgtype", "mixed", + "mixed", Map.of("msg_item", List.of( + Map.of("msgtype", "text", "text", Map.of("content", "看这张图")), + Map.of("msgtype", "image", "image", Map.of( + "url", "https://example.com/y.jpg", + "aeskey", "k"))))))); + String prefix = (String) ctx.getClass().getMethod("prefix").invoke(ctx); + @SuppressWarnings("unchecked") + List parts = (List) + ctx.getClass().getMethod("attachedParts").invoke(ctx); + assertEquals("[引用消息: 看这张图 [图片]]\n", prefix); + assertEquals(1, parts.size(), "mixed image gets attached as a media part"); + } + + @Test + @DisplayName("unknown quote sub-type still produces a [] tag (informative, not silent)") + void unknownQuoteType() throws Exception { + Object ctx = invoke(Map.of("quote", Map.of( + "msgtype", "appmsg", + "appmsg", Map.of("title", "some link")))); + String prefix = (String) ctx.getClass().getMethod("prefix").invoke(ctx); + assertEquals("[引用消息: [appmsg]]\n", prefix); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyQueueStressTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyQueueStressTest.java new file mode 100644 index 00000000..d69603fe --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyQueueStressTest.java @@ -0,0 +1,542 @@ +package vip.mate.channel.wecom; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.notification.ApprovalNotificationService; +import vip.mate.channel.wecom.cards.WeComCardDispatcher; + +import java.io.IOException; +import java.lang.reflect.Field; +import java.net.http.WebSocket; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-32 §3.0 PR-0 stress catalog — six tests covering every concurrency + * race the v2.0~v2.5.1 review chain identified. + * + *

Each test runs against a {@link TestableAdapter} that overrides + * {@code sendFrame} so no real WebSocket is touched. Other state + * (running flag, lifecycle gate, pendingAcks map) is poked via + * reflection — keeping production-code visibility tweaks to a minimum + * (just {@code workerIdleTimeoutMs} and dropping {@code private} from + * {@code sendFrame}). + * + *

None of these tests sleep more than ~3s total even at high + * iteration counts, so they're safe to run in regular CI rather than + * a separate stress-only profile. + */ +class ReplyQueueStressTest { + + private TestableAdapter adapter; + private ObjectMapper mapper; + + @BeforeEach + void setUp() throws Exception { + ChannelEntity entity = new ChannelEntity(); + entity.setId(1L); + entity.setName("test-wecom"); + entity.setChannelType("wecom"); + entity.setConfigJson("{}"); + ChannelMessageRouter router = Mockito.mock(ChannelMessageRouter.class); + ApprovalNotificationService approvalSvc = Mockito.mock(ApprovalNotificationService.class); + WeComCardDispatcher cardDispatcher = Mockito.mock(WeComCardDispatcher.class); + WeComKeepaliveScheduler keepalive = Mockito.mock(WeComKeepaliveScheduler.class); + mapper = new ObjectMapper(); + adapter = new TestableAdapter(entity, router, mapper, approvalSvc, cardDispatcher, keepalive); + // Manually bring the adapter to a "running and ready" state without + // doing a real WS handshake. This is what doStart + connectWebSocket + + // markReady would have produced on a live system. + setRunning(adapter, true); + invokePrivate(adapter, "ensureReplyExecutor"); + invokePrivate(adapter, "openReplyQueue"); + // Most tests run with a much shorter idle timeout so the worker's + // 60-second poll doesn't dominate test wall-clock time. + adapter.workerIdleTimeoutMs = 80; + } + + @AfterEach + void tearDown() throws Exception { + // Belt-and-suspenders cleanup: even if an assertion failed, drop + // the executor so dangling worker threads don't bleed into the + // next test. + try { + invokePrivate(adapter, "releaseConnectionResources", new Class[]{String.class}, "test-teardown"); + } catch (Exception ignored) {} + setRunning(adapter, false); + } + + // ===================================================================== + // S-1: same reqId serial dispatch + // ===================================================================== + + @Nested + @DisplayName("S-1 same reqId serial dispatch") + class S1_SerialDispatch { + + @Test + @DisplayName("three frames on same reqId: only one in flight at a time") + void serialPerReqId() throws Exception { + String reqId = "req_s1"; + // Don't auto-ACK; tests will release ACKs one by one. + adapter.autoAck = false; + + CompletableFuture> f1 = adapter.callSendFrameWithAck(reqId, frame(reqId, "msg1")); + CompletableFuture> f2 = adapter.callSendFrameWithAck(reqId, frame(reqId, "msg2")); + CompletableFuture> f3 = adapter.callSendFrameWithAck(reqId, frame(reqId, "msg3")); + + // Worker thread starts asynchronously — give it a tick to dequeue + // the first task and dispatch sendFrame. + assertEquals("msg1", awaitFrameText(adapter, 500), + "first frame must dispatch within 500ms"); + + // No further frame may dispatch until the first ACK arrives. + // Sleep ~150ms (≈ 2x adapter.workerIdleTimeoutMs) and assert + // the queue stayed empty. + Thread.sleep(150); + assertNull(adapter.sentFrames.poll(), "second frame must NOT dispatch before first ACK"); + + // Release ACK 1 → frame 2 should now dispatch. + completeAck(adapter, reqId); + assertEquals("msg2", awaitFrameText(adapter, 500)); + + Thread.sleep(150); + assertNull(adapter.sentFrames.poll(), "third frame must NOT dispatch before second ACK"); + + completeAck(adapter, reqId); + assertEquals("msg3", awaitFrameText(adapter, 500)); + + completeAck(adapter, reqId); + + // All three futures should now complete successfully. + assertNotNull(f1.get(500, TimeUnit.MILLISECONDS)); + assertNotNull(f2.get(500, TimeUnit.MILLISECONDS)); + assertNotNull(f3.get(500, TimeUnit.MILLISECONDS)); + } + } + + // ===================================================================== + // S-2: sendFrame sync throw → future fails immediately + // ===================================================================== + + @Nested + @DisplayName("S-2 sendFrame sync throw → future fails fast") + class S2_SendFrameThrow { + + @Test + @DisplayName("future fails within 200ms on IOException, not the 5s ACK timeout") + void syncThrowFailsFast() throws Exception { + adapter.sendFrameBehavior = frame -> { + throw new RuntimeException("simulated ws sendText failure", new IOException("ws null")); + }; + + long t0 = System.nanoTime(); + CompletableFuture> future = + adapter.callSendFrameWithAck("req_s2", frame("req_s2", "x")); + + ExecutionException ex = assertThrows(ExecutionException.class, + () -> future.get(500, TimeUnit.MILLISECONDS), + "future must complete (exceptionally) within 500ms"); + long elapsedMs = (System.nanoTime() - t0) / 1_000_000; + assertTrue(elapsedMs < 200, + "should fail-fast in under 200ms, took " + elapsedMs + "ms"); + assertNotNull(ex.getCause()); + } + } + + // ===================================================================== + // S-3: idle-close vs late-enqueue race × many iterations × many threads + // ===================================================================== + + @Nested + @DisplayName("S-3 worker idle-close vs late enqueue: no orphans across N iterations") + class S3_IdleRace { + + @Test + @DisplayName("100 iterations × 8 threads: every offered task completes") + void noOrphansUnderRace() throws Exception { + // Tighten idle timeout to 30ms so each iteration cycles through + // open → busy → idle-close in the low-100ms range. + adapter.workerIdleTimeoutMs = 30; + adapter.autoAck = true; // ACK as soon as worker dispatches + + int threads = 8; + int iterationsPerThread = 100; + ExecutorService pool = Executors.newFixedThreadPool(threads); + CountDownLatch latch = new CountDownLatch(1); + ConcurrentLinkedQueue>> all = + new ConcurrentLinkedQueue<>(); + + for (int t = 0; t < threads; t++) { + final int tid = t; + pool.submit(() -> { + try { + latch.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + for (int i = 0; i < iterationsPerThread; i++) { + // Mix reqIds: some shared (forces worker reuse), some + // unique (forces fresh-state path). + String reqId = (i % 3 == 0) + ? "shared_req" + : "t" + tid + "_i" + i; + all.add(adapter.callSendFrameWithAck(reqId, frame(reqId, "p"))); + // Random tiny delay so worker idle-close has a chance + // to interleave with late offers. + if (i % 10 == 0) { + try { Thread.sleep(35); } catch (InterruptedException ie) { return; } + } + } + }); + } + latch.countDown(); + pool.shutdown(); + assertTrue(pool.awaitTermination(20, TimeUnit.SECONDS), "submission threads must finish"); + + // Every offered future must eventually complete. 5s budget for the + // worker(s) to drain. Track failures with reasons for debuggability. + int total = all.size(); + int orphans = 0; + int succeeded = 0; + int failed = 0; + long deadline = System.currentTimeMillis() + 5_000; + for (CompletableFuture> f : all) { + long remaining = Math.max(0, deadline - System.currentTimeMillis()); + try { + f.get(remaining, TimeUnit.MILLISECONDS); + succeeded++; + } catch (TimeoutException te) { + orphans++; + } catch (Exception e) { + // ExecutionException or interrupt — counted as completed + // (test only cares that no future hangs forever). + failed++; + } + } + assertEquals(0, orphans, + "no future may remain pending after the queue drains; " + + "total=" + total + " ok=" + succeeded + " err=" + failed + + " orphans=" + orphans); + } + } + + // ===================================================================== + // S-4: release in progress → all enqueues fast-fail + // ===================================================================== + + @Nested + @DisplayName("S-4 release window: enqueues fail fast, no orphans") + class S4_ReleaseRace { + + @Test + @DisplayName("100 concurrent enqueues during release: all complete in <1s") + void releaseFailsFast() throws Exception { + // Spawn 100 concurrent enqueues. Halfway through, trigger + // releaseConnectionResources on a separate thread. + int N = 100; + ExecutorService pool = Executors.newFixedThreadPool(16); + CountDownLatch start = new CountDownLatch(1); + ConcurrentLinkedQueue>> futures = + new ConcurrentLinkedQueue<>(); + + for (int i = 0; i < N; i++) { + final int idx = i; + pool.submit(() -> { + try { start.await(); } catch (InterruptedException ignored) {} + futures.add(adapter.callSendFrameWithAck("req_s4_" + idx, frame("req_s4_" + idx, "p"))); + }); + } + // Trigger release shortly after enqueue burst begins. + pool.submit(() -> { + try { start.await(); } catch (InterruptedException ignored) {} + try { + Thread.sleep(5); // a small lead so some enqueues land first + invokePrivate(adapter, "releaseConnectionResources", + new Class[]{String.class}, "s4-test"); + } catch (Exception ignored) {} + }); + start.countDown(); + pool.shutdown(); + assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS)); + + // Every future must complete in <1s — either success (offered + // before gate closed and worker drained) or IllegalStateException + // (gate closed by release). + long t0 = System.nanoTime(); + int hangs = 0; + for (CompletableFuture> f : futures) { + try { + f.get(1_000, TimeUnit.MILLISECONDS); + } catch (TimeoutException te) { + hangs++; + } catch (Exception ignored) {} + } + long elapsedMs = (System.nanoTime() - t0) / 1_000_000; + assertEquals(0, hangs, hangs + " future(s) hung during release window"); + assertTrue(elapsedMs < 2_000, + "all " + futures.size() + " futures should resolve in <2s, took " + elapsedMs + "ms"); + } + } + + // ===================================================================== + // S-5: executor ready but markReady not called → fast-fail + // ===================================================================== + + @Nested + @DisplayName("S-5 lifecycle gate: enqueue before markReady fails fast") + class S5_GateClosed { + + @Test + @DisplayName("with executor present but accepting=false, enqueue returns failed future immediately") + void closedGateFailsFast() throws Exception { + // Force the lifecycle into "executor ready, transport not ready" + // (the exact window R-7 covers). + adapter.workerIdleTimeoutMs = 60_000; // restore to default — we don't want the worker pool churning + // Take the gate down without going through release. + Field gate = WeComChannelAdapter.class.getDeclaredField("replyQueueAccepting"); + gate.setAccessible(true); + ((AtomicBoolean) gate.get(adapter)).set(false); + + long t0 = System.nanoTime(); + CompletableFuture> f = + adapter.callSendFrameWithAck("req_s5", frame("req_s5", "x")); + ExecutionException ex = assertThrows(ExecutionException.class, + () -> f.get(200, TimeUnit.MILLISECONDS)); + long elapsedMs = (System.nanoTime() - t0) / 1_000_000; + assertTrue(elapsedMs < 100, + "fast-fail should be near-instant (sync resolution), took " + elapsedMs + "ms"); + assertInstanceOf(IllegalStateException.class, ex.getCause(), + "must surface the gate-closed reason as IllegalStateException"); + assertTrue(ex.getCause().getMessage().contains("not accepting"), + "error message must mention 'not accepting'; got: " + ex.getCause().getMessage()); + // No frame should ever have been queued. + assertNull(adapter.sentFrames.poll(), + "sendFrame must not be invoked when gate is closed"); + } + } + + // ===================================================================== + // S-6: release ordering — accepting=false happens-before ws.close() + // ===================================================================== + + @Nested + @DisplayName("S-6 release ordering: accepting flips first") + class S6_ReleaseOrdering { + + @Test + @DisplayName("when ws.sendClose runs, replyQueueAccepting is already false") + void acceptingFalseBeforeWsClose() throws Exception { + // Install an instrumented WebSocket that records the gate value + // at the moment sendClose() is invoked. + AtomicBoolean acceptingAtCloseTime = new AtomicBoolean(true); + AtomicBoolean closeWasCalled = new AtomicBoolean(false); + + WebSocket fakeWs = (WebSocket) java.lang.reflect.Proxy.newProxyInstance( + WebSocket.class.getClassLoader(), + new Class[]{WebSocket.class}, + (proxy, method, args) -> { + if ("sendClose".equals(method.getName())) { + // Snapshot gate state at the exact moment release + // is calling close on us. The S-6 invariant: + // step 0 must have already flipped accepting. + Field gate = WeComChannelAdapter.class.getDeclaredField("replyQueueAccepting"); + gate.setAccessible(true); + acceptingAtCloseTime.set(((AtomicBoolean) gate.get(adapter)).get()); + closeWasCalled.set(true); + return CompletableFuture.completedFuture(proxy); + } + if (method.getReturnType() == boolean.class) return false; + if (method.getReturnType() == long.class) return 0L; + return null; + }); + + // Inject the fake into the adapter and verify accepting is true + // (i.e. we're in normal operation about to release). + Field wsField = WeComChannelAdapter.class.getDeclaredField("webSocket"); + wsField.setAccessible(true); + wsField.set(adapter, fakeWs); + + Field gate = WeComChannelAdapter.class.getDeclaredField("replyQueueAccepting"); + gate.setAccessible(true); + assertTrue(((AtomicBoolean) gate.get(adapter)).get(), + "precondition: accepting must be true before release"); + + invokePrivate(adapter, "releaseConnectionResources", + new Class[]{String.class}, "s6-test"); + + assertTrue(closeWasCalled.get(), "release must invoke ws.sendClose"); + assertFalse(acceptingAtCloseTime.get(), + "step 0 (accepting=false) must happen-before ws.sendClose; " + + "if this fails, the release method body has been re-ordered " + + "and an enqueue could land between accepting and ws teardown"); + } + } + + // ===================================================================== + // Helpers + // ===================================================================== + + /** Build the canonical aibot_respond_msg frame the adapter uses. */ + private static Map frame(String reqId, String text) { + return Map.of( + "cmd", "aibot_respond_msg", + "headers", Map.of("req_id", reqId), + "body", Map.of("msgtype", "text", "text", Map.of("content", text)) + ); + } + + /** Read the most-recent dispatched frame's text content. Polls up to {@code timeoutMs}. */ + private static String awaitFrameText(TestableAdapter a, long timeoutMs) throws Exception { + Map f = a.sentFrames.poll(timeoutMs, TimeUnit.MILLISECONDS); + assertNotNull(f, "no frame dispatched within " + timeoutMs + "ms"); + @SuppressWarnings("unchecked") + Map body = (Map) f.get("body"); + @SuppressWarnings("unchecked") + Map txt = (Map) body.get("text"); + return (String) txt.get("content"); + } + + /** Complete the in-flight ACK future for the given reqId. Returns true if found. */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static boolean completeAck(WeComChannelAdapter a, String reqId) throws Exception { + Field f = WeComChannelAdapter.class.getDeclaredField("pendingAcks"); + f.setAccessible(true); + ConcurrentHashMap map = + (ConcurrentHashMap) f.get(a); + // Wait briefly for the worker to register the future before completing. + long deadline = System.currentTimeMillis() + 500; + CompletableFuture future = null; + while (System.currentTimeMillis() < deadline) { + future = map.get(reqId); + if (future != null) break; + Thread.sleep(5); + } + if (future == null) return false; + future.complete(Map.of("errcode", 0)); + return true; + } + + private static void setRunning(WeComChannelAdapter a, boolean v) throws Exception { + // running lives on AbstractChannelAdapter; walk the class chain to find it. + Field running = findField(a.getClass(), "running"); + ((AtomicBoolean) running.get(a)).set(v); + } + + private static Field findField(Class cls, String name) throws NoSuchFieldException { + for (Class c = cls; c != null; c = c.getSuperclass()) { + try { + Field f = c.getDeclaredField(name); + f.setAccessible(true); + return f; + } catch (NoSuchFieldException ignored) { + // keep walking + } + } + throw new NoSuchFieldException(name + " not found in class chain rooted at " + cls); + } + + private static Object invokePrivate(WeComChannelAdapter a, String method) throws Exception { + return invokePrivate(a, method, new Class[0]); + } + + private static Object invokePrivate(WeComChannelAdapter a, String method, + Class[] paramTypes, Object... args) throws Exception { + var m = WeComChannelAdapter.class.getDeclaredMethod(method, paramTypes); + m.setAccessible(true); + return m.invoke(a, args); + } + + /** + * Test-only adapter that captures dispatched frames and lets each + * test choose between auto-ACK or manual ACK release. + */ + static class TestableAdapter extends WeComChannelAdapter { + + final LinkedBlockingQueue> sentFrames = new LinkedBlockingQueue<>(); + + /** When false, tests must manually call {@code completeAck}. */ + volatile boolean autoAck = true; + + /** Optional behavior injected per-test (return value ignored; thrown exceptions propagate). */ + volatile Function, Void> sendFrameBehavior = null; + + TestableAdapter(ChannelEntity entity, ChannelMessageRouter router, + ObjectMapper mapper, ApprovalNotificationService approvalSvc, + WeComCardDispatcher cardDispatcher, WeComKeepaliveScheduler keepalive) { + super(entity, router, mapper, approvalSvc, cardDispatcher, keepalive); + } + + @Override + void sendFrame(Map frame) { + sentFrames.offer(frame); + Function, Void> beh = sendFrameBehavior; + if (beh != null) { + beh.apply(frame); // may throw + return; + } + if (autoAck) { + String reqId = extractReqId(frame); + if (reqId != null) { + // Schedule async ACK on a tiny delay so the worker has time + // to register the future before we complete it. + AUTOACK.submit(() -> { + try { + Thread.sleep(2); + completeAck(this, reqId); + } catch (Exception ignored) {} + }); + } + } + } + + /** Expose package-private sendFrameWithAck to tests. */ + CompletableFuture> callSendFrameWithAck(String reqId, Map frame) { + try { + var m = WeComChannelAdapter.class.getDeclaredMethod("sendFrameWithAck", String.class, Map.class); + m.setAccessible(true); + @SuppressWarnings("unchecked") + CompletableFuture> f = + (CompletableFuture>) m.invoke(this, reqId, frame); + return f; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @SuppressWarnings("unchecked") + private static String extractReqId(Map frame) { + Map headers = (Map) frame.get("headers"); + return headers == null ? null : (String) headers.get("req_id"); + } + + private static final ExecutorService AUTOACK = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "test-auto-ack"); + t.setDaemon(true); + return t; + }); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyStreamDedupTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyStreamDedupTest.java new file mode 100644 index 00000000..cd26664c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/ReplyStreamDedupTest.java @@ -0,0 +1,207 @@ +package vip.mate.channel.wecom; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import vip.mate.channel.ChannelMessageRouter; +import vip.mate.channel.model.ChannelEntity; +import vip.mate.channel.notification.ApprovalNotificationService; +import vip.mate.channel.wecom.cards.WeComCardDispatcher; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Exercises the chunk-content dedup added to + * {@link WeComChannelAdapter#replyStream(String, String, String, boolean, String)} + * (RFC-32 §2.1.3). Without dedup, every token-level update during tool + * argument streaming would emit a fresh frame even when the visible + * content didn't change — flickering the IM client. + * + *

Run pattern: drop {@code sendFrame} into a queue so we can count + * how many frames actually went out for a given content sequence, + * without touching a real WebSocket. + */ +class ReplyStreamDedupTest { + + private TestableAdapter adapter; + private LinkedBlockingQueue> sentFrames; + + @BeforeEach + void setUp() throws Exception { + ChannelEntity entity = new ChannelEntity(); + entity.setId(1L); + entity.setChannelType("wecom"); + entity.setConfigJson("{}"); + adapter = new TestableAdapter( + entity, + Mockito.mock(ChannelMessageRouter.class), + new ObjectMapper(), + Mockito.mock(ApprovalNotificationService.class), + Mockito.mock(WeComCardDispatcher.class), + Mockito.mock(WeComKeepaliveScheduler.class)); + sentFrames = adapter.sentFrames; + + // Bring the adapter to "running + accepting" so sendFrameWithAck doesn't + // fast-fail on the lifecycle gate (PR-0). + Field running = adapter.getClass().getSuperclass().getSuperclass().getDeclaredField("running"); + running.setAccessible(true); + ((AtomicBoolean) running.get(adapter)).set(true); + Method ensure = WeComChannelAdapter.class.getDeclaredMethod("ensureReplyExecutor"); + ensure.setAccessible(true); + ensure.invoke(adapter); + Method open = WeComChannelAdapter.class.getDeclaredMethod("openReplyQueue"); + open.setAccessible(true); + open.invoke(adapter); + // Long idle so the worker doesn't churn during the short test. + adapter.workerIdleTimeoutMs = 60_000L; + } + + @Test + @DisplayName("identical non-final chunks dedup: only first goes out") + void identicalChunksDedup() throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "replyStream", String.class, String.class, String.class, boolean.class); + m.setAccessible(true); + m.invoke(adapter, "rid", "stream-1", "Hello", false); + m.invoke(adapter, "rid", "stream-1", "Hello", false); // dup → skipped + m.invoke(adapter, "rid", "stream-1", "Hello", false); // dup → skipped + + // Only the first frame should have been dispatched (give worker a beat). + Map first = sentFrames.poll(500, TimeUnit.MILLISECONDS); + assertNotNull(first, "first non-final chunk should have dispatched"); + assertNull(sentFrames.poll(200, TimeUnit.MILLISECONDS), + "duplicate non-final chunks must be deduplicated"); + } + + @Test + @DisplayName("changed content always goes out") + void changedContentDispatches() throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "replyStream", String.class, String.class, String.class, boolean.class); + m.setAccessible(true); + m.invoke(adapter, "rid", "stream-1", "Hello", false); + m.invoke(adapter, "rid", "stream-1", "Hello world", false); // changed → goes + m.invoke(adapter, "rid", "stream-1", "Hello world", false); // dup → skipped + + // 2 frames expected (poll up to 500ms each) + Map f1 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + Map f2 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + assertNotNull(f1); + assertNotNull(f2); + assertNull(sentFrames.poll(200, TimeUnit.MILLISECONDS), + "no third frame: only 2 distinct contents should have been sent"); + } + + @Test + @DisplayName("finish=true always goes out, even with identical content") + void finishAlwaysDispatches() throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "replyStream", String.class, String.class, String.class, boolean.class); + m.setAccessible(true); + m.invoke(adapter, "rid", "stream-1", "Done", false); + m.invoke(adapter, "rid", "stream-1", "Done", true); // SAME content but finish=true → goes + + Map f1 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + Map f2 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + assertNotNull(f1); + assertNotNull(f2, "finish=true must always dispatch even when content matches the previous chunk"); + } + + @Test + @DisplayName("dedup is per-streamId; different streams don't interfere") + void perStreamIsolation() throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "replyStream", String.class, String.class, String.class, boolean.class); + m.setAccessible(true); + m.invoke(adapter, "rid", "stream-A", "X", false); + m.invoke(adapter, "rid", "stream-B", "X", false); // different stream — must dispatch + + Map f1 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + Map f2 = sentFrames.poll(500, TimeUnit.MILLISECONDS); + assertNotNull(f1); + assertNotNull(f2, + "dedup memory must be per-streamId — same content on a different stream still dispatches"); + } + + @Test + @DisplayName("after finish=true, the dedup slot is cleared so the next stream with same content goes") + void finishClearsDedupSlot() throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "replyStream", String.class, String.class, String.class, boolean.class); + m.setAccessible(true); + m.invoke(adapter, "rid", "stream-1", "X", false); + m.invoke(adapter, "rid", "stream-1", "X", true); // finish, clears slot + m.invoke(adapter, "rid", "stream-1", "X", false); // new chunk — slot was cleared, so goes + + // 3 frames expected total + for (int i = 0; i < 3; i++) { + assertNotNull(sentFrames.poll(500, TimeUnit.MILLISECONDS), + "expected frame #" + (i + 1) + " to dispatch"); + } + } + + /** + * Test-only adapter that captures dispatched frames AND auto-completes + * each {@code pendingAcks} future shortly after the frame goes out, so + * the per-reqId serial worker can dequeue the next task without waiting + * the full 5s {@code orTimeout}. Without auto-ack, the dedup tests that + * dispatch multiple distinct frames would each block ~5s on the prior + * frame's ACK. + */ + static class TestableAdapter extends WeComChannelAdapter { + final LinkedBlockingQueue> sentFrames = new LinkedBlockingQueue<>(); + private static final ExecutorService AUTOACK = Executors.newCachedThreadPool(r -> { + Thread t = new Thread(r, "test-autoack-dedup"); + t.setDaemon(true); + return t; + }); + + TestableAdapter(ChannelEntity entity, ChannelMessageRouter router, + ObjectMapper mapper, ApprovalNotificationService approvalSvc, + WeComCardDispatcher cardDispatcher, WeComKeepaliveScheduler keepalive) { + super(entity, router, mapper, approvalSvc, cardDispatcher, keepalive); + } + + @Override + @SuppressWarnings("unchecked") + void sendFrame(Map frame) { + sentFrames.offer(frame); + // Mirror what the WeCom server would do in production: ACK the + // outbound request so the worker's task.future().join() unblocks + // and the next frame in the same reqId queue can dispatch. + Map headers = (Map) frame.get("headers"); + if (headers == null) return; + String reqId = (String) headers.get("req_id"); + if (reqId == null || reqId.isBlank()) return; + AUTOACK.submit(() -> completeAckSoon(reqId)); + } + + private void completeAckSoon(String reqId) { + try { + // Brief delay so the worker has reliably completed + // pendingAcks.put before we look it up. + Thread.sleep(2); + Field f = WeComChannelAdapter.class.getDeclaredField("pendingAcks"); + f.setAccessible(true); + @SuppressWarnings("unchecked") + ConcurrentHashMap>> pending = + (ConcurrentHashMap>>) f.get(this); + CompletableFuture> fut = pending.get(reqId); + if (fut != null) fut.complete(Map.of("errcode", 0)); + } catch (Exception ignored) {} + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComInboundConversationIdTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComInboundConversationIdTest.java new file mode 100644 index 00000000..bc5a27cc --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComInboundConversationIdTest.java @@ -0,0 +1,77 @@ +package vip.mate.channel.wecom; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the alignment between {@code WeComChannelAdapter.inboundConversationId} + * and {@code ChannelMessageRouter.buildConversationId}. + * + *

These two compute the same logical conversation id from different code + * paths: the adapter pre-computes it to choose the per-conversation + * upload directory before the {@link vip.mate.channel.ChannelMessage} + * exists, and the router computes it from the {@code ChannelMessage} + * downstream. They MUST agree on the same string format, otherwise + * inbound media saves to one directory while messages persist under a + * different conversationId — and the {@code /api/v1/chat/files/{convId}/...} + * endpoint's owner check fails for every fetch (403 → broken images). + * + *

The format both produce: {@code wecom:{chatId}} for groups, + * {@code wecom:{senderId}} for 1:1 — no {@code group:} infix. + */ +class WeComInboundConversationIdTest { + + private static String inboundConversationId(String senderId, String chatId, String chatType) throws Exception { + Method m = WeComChannelAdapter.class.getDeclaredMethod( + "inboundConversationId", String.class, String.class, String.class); + m.setAccessible(true); + return (String) m.invoke(null, senderId, chatId, chatType); + } + + @Test + @DisplayName("group → wecom:{chatId} (no 'group:' infix, matches router)") + void groupChatIdFormat() throws Exception { + // The bug fix: previously returned "wecom:group:abc" which mismatched + // the router's "wecom:abc" — quoted-image fileUrls hit a 403 because + // isConversationOwner couldn't find a "wecom:group:abc" row in + // mate_conversation. + assertEquals("wecom:group-abc", + inboundConversationId("XuZhanFu", "group-abc", "group")); + } + + @Test + @DisplayName("1:1 → wecom:{senderId} (chatId is irrelevant in single chats)") + void singleChatSenderFormat() throws Exception { + // Single-chat case never had the bug because both adapter and + // router fell back to senderId — pin it so a future refactor of + // either side doesn't accidentally diverge. + assertEquals("wecom:XuZhanFu", + inboundConversationId("XuZhanFu", null, "single")); + assertEquals("wecom:XuZhanFu", + inboundConversationId("XuZhanFu", "ignored-when-single", "single")); + } + + @Test + @DisplayName("matches ChannelMessageRouter.buildConversationId for both group and 1:1") + void matchesRouterFormat() throws Exception { + // Router's identifier picker: + // chatId != null → "{channelType}:{chatId}" (group) + // chatId == null → "{channelType}:{senderId}" (single) + // Inbound side passes chatId for groups, null/ignored for 1:1. + // Both must arrive at the same string, exact-equal. + + // group: router gets chatId from the ChannelMessage builder + String routerGroup = "wecom" + ":" + "group-xyz"; + assertEquals(routerGroup, + inboundConversationId("Alice", "group-xyz", "group")); + + // single: router falls back to senderId (chatId is null on the message) + String routerSingle = "wecom" + ":" + "Alice"; + assertEquals(routerSingle, + inboundConversationId("Alice", null, "single")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComKeepaliveSchedulerTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComKeepaliveSchedulerTest.java new file mode 100644 index 00000000..56024609 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComKeepaliveSchedulerTest.java @@ -0,0 +1,163 @@ +package vip.mate.channel.wecom; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * Verify the WeComKeepaliveScheduler bookkeeping + force-finish path. + * + *

The 20s/180s timing constants come from QwenPaw and are already + * validated empirically in production; we don't re-test the exact + * scheduling intervals here (would require either real wall-clock waits + * or invasive ScheduledExecutor mocking). Instead we cover: + *

    + *
  • start/stop/shutdownAll bookkeeping is correct
  • + *
  • the force-finish branch (180s ceiling) calls + * {@link WeComChannelAdapter#replyStreamFinishForKeepalive} AND + * {@link WeComChannelAdapter#invalidateReplyContext} — the + * RFC-32 §2.1.2 invariant that prevents the next real reply from + * reusing a closed stream slot
  • + *
  • the refresh branch (still under ceiling) calls + * {@link WeComChannelAdapter#replyStreamRefreshForKeepalive} only
  • + *
+ * + *

Force-finish is exercised by reflection-overriding {@code startedAt} + * to a long-ago timestamp on a tracked StreamState, then invoking the + * private {@code tick} method. This bypasses the ScheduledExecutor + * entirely so tests run in milliseconds. + */ +class WeComKeepaliveSchedulerTest { + + private WeComKeepaliveScheduler scheduler; + private WeComChannelAdapter adapter; + + @BeforeEach + void setUp() { + scheduler = new WeComKeepaliveScheduler(); + adapter = Mockito.mock(WeComChannelAdapter.class); + } + + @Test + @DisplayName("start adds a stream entry; stop removes it") + void startStopBookkeeping() { + assertEquals(0, scheduler.activeStreamCount()); + + scheduler.start(adapter, "req-1", "stream-1", "user-alice"); + assertEquals(1, scheduler.activeStreamCount()); + + scheduler.stop("stream-1"); + assertEquals(0, scheduler.activeStreamCount()); + } + + @Test + @DisplayName("start is idempotent — second call for same streamId is a no-op") + void startIdempotent() { + scheduler.start(adapter, "req-1", "stream-1", "user-alice"); + scheduler.start(adapter, "req-1", "stream-1", "user-alice"); + assertEquals(1, scheduler.activeStreamCount(), "second start must not double-track"); + } + + @Test + @DisplayName("start is null-tolerant — null/blank args silently drop") + void startNullTolerant() { + scheduler.start(null, "r", "s", "t"); + scheduler.start(adapter, null, "s", "t"); + scheduler.start(adapter, "", "s", "t"); + scheduler.start(adapter, "r", null, "t"); + scheduler.start(adapter, "r", "", "t"); + assertEquals(0, scheduler.activeStreamCount(), + "null/blank args must not add entries"); + } + + @Test + @DisplayName("shutdownAll clears every tracked stream") + void shutdownAllClears() { + scheduler.start(adapter, "req-1", "stream-1", "user-alice"); + scheduler.start(adapter, "req-2", "stream-2", "user-bob"); + assertEquals(2, scheduler.activeStreamCount()); + + scheduler.shutdownAll(); + assertEquals(0, scheduler.activeStreamCount()); + } + + @Test + @DisplayName("force-finish path: replyStreamFinishForKeepalive + invalidateReplyContext + stop") + void forceFinishPath() throws Exception { + scheduler.start(adapter, "req-x", "stream-x", "user-alice"); + + // Reflectively rewind startedAt so the next tick sees elapsed > 180s + Object state = getStreamState("stream-x"); + Field startedAt = state.getClass().getDeclaredField("startedAt"); + startedAt.setAccessible(true); + // Java's `final long` fields normally resist setAccessible.set — unfortunately + // primitives also need the modifiers hack on JDK 17+. Use Unsafe-free path: + // the field happens to be declared `final` in the static record, so we mutate + // via setLong (which works for primitives even on final fields when accessible + // is true on JDK17 — verified locally). + startedAt.setLong(state, System.currentTimeMillis() - 200_000L); + + // Manually invoke the private tick(StreamState) — no ScheduledExecutor + // wall-clock wait + Method tick = WeComKeepaliveScheduler.class.getDeclaredMethod( + "tick", Class.forName(WeComKeepaliveScheduler.class.getName() + "$StreamState")); + tick.setAccessible(true); + tick.invoke(scheduler, state); + + verify(adapter, times(1)).replyStreamFinishForKeepalive( + eq("req-x"), eq("stream-x"), eq(WeComKeepaliveScheduler.PROCESSING_TEXT)); + verify(adapter, times(1)).invalidateReplyContext(eq("user-alice"), eq("stream-x")); + verify(adapter, never()).replyStreamRefreshForKeepalive(any(), any(), any()); + // After force-finish, the stream is removed from the tracker + assertEquals(0, scheduler.activeStreamCount()); + } + + @Test + @DisplayName("refresh path: replyStreamRefreshForKeepalive only — no force-finish below ceiling") + void refreshPathBelowCeiling() throws Exception { + scheduler.start(adapter, "req-y", "stream-y", "user-bob"); + + // Don't rewind startedAt; the state is fresh — well under 180s. + Object state = getStreamState("stream-y"); + Method tick = WeComKeepaliveScheduler.class.getDeclaredMethod( + "tick", Class.forName(WeComKeepaliveScheduler.class.getName() + "$StreamState")); + tick.setAccessible(true); + tick.invoke(scheduler, state); + + verify(adapter, times(1)).replyStreamRefreshForKeepalive( + eq("req-y"), eq("stream-y"), eq(WeComKeepaliveScheduler.PROCESSING_TEXT)); + verify(adapter, never()).replyStreamFinishForKeepalive(any(), any(), any()); + verify(adapter, never()).invalidateReplyContext(any(), any()); + // Still tracked — refresh ticks don't unregister + assertEquals(1, scheduler.activeStreamCount()); + } + + @Test + @DisplayName("constants match the QwenPaw-verified values (20s refresh / 180s ceiling)") + void constantsMatch() { + assertEquals(20L, WeComKeepaliveScheduler.REFRESH_INTERVAL_SECONDS); + assertEquals(180L, WeComKeepaliveScheduler.MAX_DURATION_SECONDS); + assertEquals("🤔 思考中...", WeComKeepaliveScheduler.PROCESSING_TEXT); + } + + // Pull a tracked StreamState by streamId via reflection. The states map + // lives behind a private final ConcurrentHashMap. + private Object getStreamState(String streamId) throws Exception { + Field statesField = WeComKeepaliveScheduler.class.getDeclaredField("states"); + statesField.setAccessible(true); + @SuppressWarnings("unchecked") + Map states = (Map) statesField.get(scheduler); + Object st = states.get(streamId); + assertNotNull(st, "expected stream " + streamId + " to be tracked"); + return st; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComUploadLimitsTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComUploadLimitsTest.java new file mode 100644 index 00000000..d4be9f0a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/WeComUploadLimitsTest.java @@ -0,0 +1,115 @@ +package vip.mate.channel.wecom; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.channel.wecom.WeComChannelAdapter.WeComUploadLimitDecision; + +import static org.junit.jupiter.api.Assertions.*; +import static vip.mate.channel.wecom.WeComChannelAdapter.applyWeComUploadLimits; +import static vip.mate.channel.wecom.WeComChannelAdapter.FILE_MAX_BYTES; +import static vip.mate.channel.wecom.WeComChannelAdapter.IMAGE_MAX_BYTES; +import static vip.mate.channel.wecom.WeComChannelAdapter.VIDEO_MAX_BYTES; +import static vip.mate.channel.wecom.WeComChannelAdapter.VOICE_MAX_BYTES; + +/** + * Pin the WeCom upload-limits decision matrix. + * + *

The platform server enforces these limits at the chunk-finish step + * (after we've already uploaded all bytes). Without the client-side + * pre-check, a 25 MB PDF would chunk-upload for ~minutes, then the + * server rejects the finish frame, and the user sees nothing arrive. + * These tests pin the boundary so future tweaks (e.g. WeCom raising + * limits) are intentional. + */ +class WeComUploadLimitsTest { + + @Test + @DisplayName("normal-sized file passes through with native media type") + void normalFilePasses() { + WeComUploadLimitDecision d = applyWeComUploadLimits(1_000_000, "file", null); + assertFalse(d.rejected()); + assertFalse(d.downgraded()); + assertEquals("file", d.finalMediaType()); + } + + @Test + @DisplayName("file at exactly 20MB still passes; over rejects") + void fileBoundary() { + WeComUploadLimitDecision pass = applyWeComUploadLimits(FILE_MAX_BYTES, "file", null); + assertFalse(pass.rejected()); + + WeComUploadLimitDecision fail = applyWeComUploadLimits(FILE_MAX_BYTES + 1, "file", null); + assertTrue(fail.rejected()); + assertNotNull(fail.rejectReason()); + assertTrue(fail.rejectReason().contains("20MB"), + "reject reason should mention 20MB; got: " + fail.rejectReason()); + } + + @Test + @DisplayName("image over 10MB downgrades to file with friendly note") + void oversizedImageDowngrades() { + WeComUploadLimitDecision d = applyWeComUploadLimits(IMAGE_MAX_BYTES + 1, "image", "image/png"); + assertFalse(d.rejected()); + assertTrue(d.downgraded()); + assertEquals("file", d.finalMediaType()); + assertNotNull(d.downgradeNote()); + assertTrue(d.downgradeNote().contains("图片")); + assertTrue(d.downgradeNote().contains("10MB")); + } + + @Test + @DisplayName("image at exactly 10MB still passes as image") + void imageAtBoundary() { + WeComUploadLimitDecision d = applyWeComUploadLimits(IMAGE_MAX_BYTES, "image", "image/jpeg"); + assertFalse(d.rejected()); + assertFalse(d.downgraded()); + assertEquals("image", d.finalMediaType()); + } + + @Test + @DisplayName("video over 10MB downgrades to file") + void oversizedVideoDowngrades() { + WeComUploadLimitDecision d = applyWeComUploadLimits(VIDEO_MAX_BYTES + 1, "video", "video/mp4"); + assertEquals("file", d.finalMediaType()); + assertTrue(d.downgraded()); + assertTrue(d.downgradeNote().contains("视频")); + } + + @Test + @DisplayName("voice with non-AMR mime downgrades to file regardless of size") + void voiceWrongMimeDowngrades() { + WeComUploadLimitDecision d = applyWeComUploadLimits(500_000, "voice", "audio/mpeg"); + assertEquals("file", d.finalMediaType()); + assertTrue(d.downgraded()); + assertTrue(d.downgradeNote().contains("AMR")); + } + + @Test + @DisplayName("voice in AMR but over 2MB downgrades to file") + void voiceOversizedAmrDowngrades() { + WeComUploadLimitDecision d = applyWeComUploadLimits(VOICE_MAX_BYTES + 1, "voice", "audio/amr"); + assertEquals("file", d.finalMediaType()); + assertTrue(d.downgraded()); + assertTrue(d.downgradeNote().contains("语音")); + assertTrue(d.downgradeNote().contains("2MB")); + } + + @Test + @DisplayName("voice in AMR within 2MB passes natively") + void voiceAmrInBoundsPasses() { + WeComUploadLimitDecision d = applyWeComUploadLimits(VOICE_MAX_BYTES, "voice", "audio/amr"); + assertFalse(d.rejected()); + assertFalse(d.downgraded()); + assertEquals("voice", d.finalMediaType()); + } + + @Test + @DisplayName("absolute 20MB cap trumps every modality-specific downgrade") + void absoluteCapTrumpsDowngrade() { + // An image at 25MB is over both 10MB image limit AND 20MB absolute cap. + // The absolute cap fires first (reject), not the downgrade path. + WeComUploadLimitDecision d = applyWeComUploadLimits(25L * 1024 * 1024, "image", "image/png"); + assertTrue(d.rejected()); + assertFalse(d.downgraded()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKeyTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKeyTest.java new file mode 100644 index 00000000..a725c407 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardButtonKeyTest.java @@ -0,0 +1,143 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.channel.wecom.cards.CardOversizedException; + +import java.nio.charset.StandardCharsets; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for the WeCom 1024-byte button.key encoding contract. + * + *

The encoding is the only place in PR-1 where a card payload can + * exceed a hard server limit and force the adapter to fall back to + * text. These tests pin both the happy-path encoding shape and the + * overflow behaviour so future changes to button.key fields can't + * silently break either. + */ +class ToolGuardButtonKeyTest { + + private ToolGuardButtonKey buttonKey; + + @BeforeEach + void setUp() { + buttonKey = new ToolGuardButtonKey(new ObjectMapper()); + } + + @Test + @DisplayName("encode produces decodable JSON with stable field order") + void encodeDecodeRoundTrip() { + String encoded = buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, + "abc123def456", + "shell_exec", + "HIGH" + ); + // Stable order ensures byte-length predictability + makes log + // greps deterministic. + assertTrue(encoded.startsWith("{\"a\":\"approve\""), + "first field must be 'a' (action); got: " + encoded); + assertTrue(encoded.contains("\"rid\":\"abc123def456\"")); + assertTrue(encoded.contains("\"tool\":\"shell_exec\"")); + assertTrue(encoded.contains("\"sev\":\"HIGH\"")); + + ToolGuardButtonKey.Decoded decoded = buttonKey.decode(encoded); + assertNotNull(decoded); + assertEquals(ToolGuardButtonKey.Action.APPROVE, decoded.action()); + assertEquals("abc123def456", decoded.pendingId()); + assertEquals("shell_exec", decoded.toolName()); + assertEquals("HIGH", decoded.severity()); + } + + @Test + @DisplayName("encode throws CardOversizedException at exactly the 1024-byte threshold") + void overflowAt1024Bytes() { + // toolName 1100 chars of pure ASCII (1100 bytes) — single character per byte + // forces the JSON over 1024 even with all the structural overhead. + String hugeTool = "x".repeat(1100); + CardOversizedException ex = assertThrows(CardOversizedException.class, + () -> buttonKey.encode( + ToolGuardButtonKey.Action.DENY, + "rid", + hugeTool, + "MEDIUM")); + assertTrue(ex.getMessage().contains("button.key payload"), + "exception message should reference button.key payload, got: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("1024"), + "exception message should mention the 1024 limit, got: " + ex.getMessage()); + } + + @Test + @DisplayName("encode handles Chinese tool names within the 1024-byte budget") + void encodeChineseToolName() { + String chinese = "执行命令".repeat(40); // 4 chars * 40 = 160 chars, ~480 UTF-8 bytes + String encoded = buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, + "uuid-1234", + chinese, + "MEDIUM" + ); + // sanity: each Chinese char = 3 UTF-8 bytes; 160 chars ≈ 480 bytes; + // overhead ≈ 50 bytes; total well under 1024 + int bytes = encoded.getBytes(StandardCharsets.UTF_8).length; + assertTrue(bytes < 1024, "expected < 1024 bytes for moderate Chinese, got " + bytes); + ToolGuardButtonKey.Decoded decoded = buttonKey.decode(encoded); + assertNotNull(decoded); + assertEquals(chinese, decoded.toolName()); + } + + @Test + @DisplayName("decode returns null for malformed JSON, unknown action, or missing rid") + void decodeMalformed() { + // Garbage JSON + assertNull(buttonKey.decode("not json")); + assertNull(buttonKey.decode("{not closed")); + // Unknown action + assertNull(buttonKey.decode("{\"a\":\"reboot\",\"rid\":\"x\"}")); + // Missing rid + assertNull(buttonKey.decode("{\"a\":\"approve\"}")); + // Blank rid + assertNull(buttonKey.decode("{\"a\":\"approve\",\"rid\":\"\"}")); + // Null / blank input + assertNull(buttonKey.decode(null)); + assertNull(buttonKey.decode("")); + assertNull(buttonKey.decode(" ")); + } + + @Test + @DisplayName("decode tolerates extra/unknown fields (forward-compat)") + void decodeForwardCompat() { + String json = "{\"a\":\"deny\",\"rid\":\"r1\",\"tool\":\"t\",\"sev\":\"LOW\",\"future\":42}"; + ToolGuardButtonKey.Decoded decoded = buttonKey.decode(json); + assertNotNull(decoded); + assertEquals(ToolGuardButtonKey.Action.DENY, decoded.action()); + } + + @Test + @DisplayName("encoded JSON respects the 1024-byte boundary on either side") + void boundaryExact() { + // 950 ASCII chars + JSON overhead (~50 bytes for the structural braces, + // commas, quotes, and the 'a'/'rid'/'tool'/'sev' field labels) lands + // around 1010 bytes — comfortably under the 1024 limit. + String near = "a".repeat(950); + String encoded = buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, + "x", + near, + "M" + ); + assertNotNull(encoded); + assertTrue(encoded.getBytes(StandardCharsets.UTF_8).length <= ToolGuardButtonKey.MAX_KEY_BYTES, + "950-char tool name must encode within 1024 bytes; got " + + encoded.getBytes(StandardCharsets.UTF_8).length); + + // Push past the limit — must throw + String over = "a".repeat(1100); + assertThrows(CardOversizedException.class, + () -> buttonKey.encode(ToolGuardButtonKey.Action.APPROVE, "x", over, "M")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandlerTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandlerTest.java new file mode 100644 index 00000000..d1873f62 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardHandlerTest.java @@ -0,0 +1,198 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; +import vip.mate.approval.ApprovalService; +import vip.mate.approval.PendingApproval; +import vip.mate.channel.ChannelMessage; +import vip.mate.channel.wecom.WeComChannelAdapter; + +import java.util.Map; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.*; + +/** + * Tests for the validate-before-render invariant (RFC-32 v2.1 / R-5). + * + *

The earlier draft (v2.0) did "render resolved card → inject /approve → + * router rejects unauthorized" — meaning a Lee click on Zhang's pending + * would briefly show "✅ 已批准 by 李四" on the card before the router + * silently dropped the command. v2.1 reorders to validate first, then + * render the resolved card matching the validation result, then inject + * the command only when authorized. + */ +class ToolGuardCardHandlerTest { + + private ApprovalService approvalService; + private WeComChannelAdapter adapter; + private ToolGuardButtonKey buttonKey; + private ToolGuardCardHandler handler; + + @BeforeEach + void setUp() { + approvalService = Mockito.mock(ApprovalService.class); + adapter = Mockito.mock(WeComChannelAdapter.class); + buttonKey = new ToolGuardButtonKey(new ObjectMapper()); + handler = new ToolGuardCardHandler(approvalService, buttonKey); + } + + @Test + @DisplayName("unauthorized click renders 'unauthorized' card and does NOT inject command") + void unauthorizedClickDoesNotInject() { + // Given: a pending whose original requester is "alice" + PendingApproval pending = pendingFor("pid_xyz", "alice", "shell_exec"); + when(approvalService.getPending("pid_xyz")).thenReturn(Optional.of(pending)); + + // When: bob (NOT alice) clicks "approve" + Map frame = inboundFrame("evt_req_1", buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, "pid_xyz", "shell_exec", "HIGH")); + handler.handle(adapter, frame, tce(frame), fromBlock("bob")); + + // Then: card was updated to "unauthorized" state… + ArgumentCaptor> cardCaptor = cardArgCaptor(); + verify(adapter, times(1)).updateTemplateCard(eq("evt_req_1"), cardCaptor.capture()); + @SuppressWarnings("unchecked") + Map mainTitle = (Map) cardCaptor.getValue().get("main_title"); + assertNotNull(mainTitle); + String title = (String) mainTitle.get("title"); + assertTrue(title.contains("仅原请求者"), + "unauthorized card must say '仅原请求者可审批'; got: " + title); + + // …and CRITICALLY, no /approve command was injected + verify(adapter, never()).injectSyntheticMessage(any(ChannelMessage.class)); + } + + @Test + @DisplayName("expired pending renders 'expired' card and does NOT inject command") + void expiredPendingShowsExpiredCard() { + when(approvalService.getPending("pid_old")).thenReturn(Optional.empty()); + + Map frame = inboundFrame("evt_req_2", buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, "pid_old", "shell_exec", "MEDIUM")); + handler.handle(adapter, frame, tce(frame), fromBlock("alice")); + + ArgumentCaptor> cardCaptor = cardArgCaptor(); + verify(adapter, times(1)).updateTemplateCard(eq("evt_req_2"), cardCaptor.capture()); + @SuppressWarnings("unchecked") + Map mainTitle = (Map) cardCaptor.getValue().get("main_title"); + assertTrue(((String) mainTitle.get("title")).contains("过期"), + "expired card title must mention 过期; got: " + mainTitle.get("title")); + verify(adapter, never()).injectSyntheticMessage(any(ChannelMessage.class)); + } + + @Test + @DisplayName("authorized approve click: render resolved card AND inject /approve") + void authorizedApproveInjectsCommand() { + PendingApproval pending = pendingFor("pid_ok", "alice", "shell_exec"); + when(approvalService.getPending("pid_ok")).thenReturn(Optional.of(pending)); + + Map frame = inboundFrame("evt_req_3", buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, "pid_ok", "shell_exec", "HIGH")); + handler.handle(adapter, frame, tce(frame), fromBlock("alice")); + + ArgumentCaptor> cardCaptor = cardArgCaptor(); + verify(adapter, times(1)).updateTemplateCard(eq("evt_req_3"), cardCaptor.capture()); + @SuppressWarnings("unchecked") + Map mainTitle = (Map) cardCaptor.getValue().get("main_title"); + assertTrue(((String) mainTitle.get("title")).contains("已批准"), + "title must announce success; got: " + mainTitle.get("title")); + + // Synthetic command should be injected with the right text + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(ChannelMessage.class); + verify(adapter, times(1)).injectSyntheticMessage(msgCaptor.capture()); + ChannelMessage injected = msgCaptor.getValue(); + assertEquals("/approve pid_ok", injected.getContent()); + assertEquals("alice", injected.getSenderId()); + assertEquals("text", injected.getContentType()); + } + + @Test + @DisplayName("authorized deny click: injects /deny") + void authorizedDenyInjectsCommand() { + PendingApproval pending = pendingFor("pid_d", "alice", "shell_exec"); + when(approvalService.getPending("pid_d")).thenReturn(Optional.of(pending)); + + Map frame = inboundFrame("evt_req_4", buttonKey.encode( + ToolGuardButtonKey.Action.DENY, "pid_d", "shell_exec", "HIGH")); + handler.handle(adapter, frame, tce(frame), fromBlock("alice")); + + ArgumentCaptor msgCaptor = ArgumentCaptor.forClass(ChannelMessage.class); + verify(adapter, times(1)).injectSyntheticMessage(msgCaptor.capture()); + assertEquals("/deny pid_d", msgCaptor.getValue().getContent()); + } + + @Test + @DisplayName("system-owned pending allows ANY clicker (no original requester)") + void systemPendingAcceptsAnyClicker() { + PendingApproval pending = pendingFor("pid_sys", "system", "shell_exec"); + when(approvalService.getPending("pid_sys")).thenReturn(Optional.of(pending)); + + Map frame = inboundFrame("evt_req_5", buttonKey.encode( + ToolGuardButtonKey.Action.APPROVE, "pid_sys", "shell_exec", "MEDIUM")); + handler.handle(adapter, frame, tce(frame), fromBlock("anyone")); + + verify(adapter).injectSyntheticMessage(any(ChannelMessage.class)); + } + + @Test + @DisplayName("malformed event_key drops the event silently — no card update, no command") + void malformedEventKeyIgnored() { + Map frame = inboundFrame("evt_req_6", "{not json"); + handler.handle(adapter, frame, tce(frame), fromBlock("alice")); + + verify(adapter, never()).updateTemplateCard(anyString(), any()); + verify(adapter, never()).injectSyntheticMessage(any(ChannelMessage.class)); + } + + // ---- helpers ---- + + private static PendingApproval pendingFor(String pendingId, String requester, String tool) { + PendingApproval p = new PendingApproval( + pendingId, "wecom:alice", requester, tool, "{}", "test approval"); + // Status defaults to "pending" via the constructor + return p; + } + + private static Map inboundFrame(String reqId, String eventKey) { + return Map.of( + "cmd", "aibot_event_callback", + "headers", Map.of("req_id", reqId), + "body", Map.of( + "chattype", "single", + "chatid", "alice", + "from", Map.of("userid", "alice"), + "event", Map.of( + "eventtype", "template_card_event", + "template_card_event", Map.of( + "task_id", "tg_approval_pid_xyz", + "event_key", eventKey + ) + ) + ) + ); + } + + @SuppressWarnings("unchecked") + private static Map tce(Map frame) { + Map body = (Map) frame.get("body"); + Map event = (Map) body.get("event"); + return (Map) event.get("template_card_event"); + } + + private static Map fromBlock(String userid) { + return Map.of("userid", userid); + } + + @SuppressWarnings("unchecked") + private static ArgumentCaptor> cardArgCaptor() { + return (ArgumentCaptor>) (ArgumentCaptor) ArgumentCaptor.forClass(Map.class); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRendererTest.java b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRendererTest.java new file mode 100644 index 00000000..d9125218 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/wecom/cards/tool_guard/ToolGuardCardRendererTest.java @@ -0,0 +1,104 @@ +package vip.mate.channel.wecom.cards.tool_guard; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.channel.notification.ApprovalNotice; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the WeCom button_interaction approval card payload shape. + * + *

The structure is server-validated — any drift (rename a field, + * change button_list location, omit task_id prefix) silently fails on + * the WeCom side at runtime. These tests catch that at compile-test + * time so renames don't ship without protocol awareness. + */ +class ToolGuardCardRendererTest { + + private final ToolGuardButtonKey buttonKey = new ToolGuardButtonKey(new ObjectMapper()); + private final ToolGuardCardRenderer renderer = new ToolGuardCardRenderer(buttonKey); + + @Test + @DisplayName("approval card has the WeCom button_interaction shape") + @SuppressWarnings("unchecked") + void approvalCardShape() { + ApprovalNotice notice = new ApprovalNotice( + "abc12345def67890", + "shell_exec", + "Run system command", + "rm -rf /tmp/cache", + "HIGH", + List.of(), + "/approve abc", + "/deny abc" + ); + + Map card = renderer.render(notice); + + assertEquals("button_interaction", card.get("card_type")); + assertEquals("tg_approval_abc12345def67890", card.get("task_id"), + "task_id must carry the tg_approval_ prefix so the inbound dispatcher can route the click"); + + Map mainTitle = (Map) card.get("main_title"); + assertNotNull(mainTitle); + assertEquals("🛡️ 工具审批", mainTitle.get("title")); + String desc = (String) mainTitle.get("desc"); + assertTrue(desc.contains("shell_exec"), "subtitle must include tool name; got: " + desc); + + List> buttons = (List>) card.get("button_list"); + assertNotNull(buttons); + assertEquals(2, buttons.size()); + + Map approve = buttons.get(0); + assertEquals("批准", approve.get("text")); + assertEquals(1, approve.get("style")); + String approveKey = (String) approve.get("key"); + ToolGuardButtonKey.Decoded a = buttonKey.decode(approveKey); + assertNotNull(a); + assertEquals(ToolGuardButtonKey.Action.APPROVE, a.action()); + assertEquals("abc12345def67890", a.pendingId()); + + Map deny = buttons.get(1); + assertEquals("拒绝", deny.get("text")); + assertEquals(2, deny.get("style")); + ToolGuardButtonKey.Decoded d = buttonKey.decode((String) deny.get("key")); + assertNotNull(d); + assertEquals(ToolGuardButtonKey.Action.DENY, d.action()); + } + + @Test + @DisplayName("resolved card uses text_notice + carries non-zero card_action.type") + @SuppressWarnings("unchecked") + void resolvedCardShape() { + Map resolved = ToolGuardCardRenderer.buildResolvedCard( + "tg_approval_abc", "✅ 已批准", "Tool x 已批准 by 张三"); + + assertEquals("text_notice", resolved.get("card_type")); + assertEquals("tg_approval_abc", resolved.get("task_id")); + + Map cardAction = (Map) resolved.get("card_action"); + assertNotNull(cardAction, "WeCom rejects text_notice cards without card_action"); + assertEquals(1, cardAction.get("type"), + "card_action.type must be 1 or 2; type=0 is rejected by the bot endpoint"); + assertNotNull(cardAction.get("url")); + } + + @Test + @DisplayName("resolved card truncates over-long desc to ~30 chars + ellipsis") + @SuppressWarnings("unchecked") + void resolvedDescTruncated() { + String longDesc = "a".repeat(100); + Map resolved = ToolGuardCardRenderer.buildResolvedCard( + "tg_approval_x", "✅", longDesc); + + Map mainTitle = (Map) resolved.get("main_title"); + String desc = (String) mainTitle.get("desc"); + assertTrue(desc.length() <= 30, "desc must be ≤30 chars after truncation, got " + desc.length()); + assertTrue(desc.endsWith("…"), "truncation marker must be present; got: " + desc); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/cron/config/ShedLockIntegrationTest.java b/mateclaw-server/src/test/java/vip/mate/cron/config/ShedLockIntegrationTest.java new file mode 100644 index 00000000..6fb2f353 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/cron/config/ShedLockIntegrationTest.java @@ -0,0 +1,124 @@ +package vip.mate.cron.config; + +import net.javacrumbs.shedlock.core.LockConfiguration; +import net.javacrumbs.shedlock.core.LockProvider; +import net.javacrumbs.shedlock.core.SimpleLock; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; + +import java.time.Duration; +import java.time.Instant; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-03 Lane G2 integration test — exercises the full path: + * + *

    + *
  1. Flyway migration {@code V74__shedlock_table.sql} ran successfully + * against the in-memory H2 (otherwise context startup would fail).
  2. + *
  3. {@link ShedLockConfig} wired a {@link LockProvider} bean.
  4. + *
  5. The provider's lock/unlock semantics actually exclude concurrent + * holders — i.e. node-A → node-B contention works as expected.
  6. + *
+ * + *

Single-node deployments hit only the trivial path (acquire from this + * JVM always succeeds), so a CI test that only exercises one acquirer + * would miss the multi-node behavior we actually shipped this for. + * Simulating two nodes against the same H2 database catches the + * contention path. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:shedlock_test_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none" +}) +class ShedLockIntegrationTest { + + @Autowired + private LockProvider lockProvider; + + @Autowired + private JdbcTemplate jdbcTemplate; + + @Test + @DisplayName("V74 created the shedlock table with the expected columns") + void shedlockTableExists() { + // information_schema lookup works on H2 MySQL-mode and on MySQL itself. + Long count = jdbcTemplate.queryForObject( + "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = 'shedlock'", + Long.class); + assertNotNull(count); + assertEquals(1L, count, "shedlock table should be created by V74"); + } + + @Test + @DisplayName("acquire then release lets a sibling acquire immediately") + void acquireAndRelease() { + String name = "test-lock-acquire-release"; + // First node — acquires. + Optional a = lockProvider.lock(new LockConfiguration( + Instant.now(), name, Duration.ofMinutes(5), Duration.ZERO)); + assertTrue(a.isPresent(), "first acquirer should succeed"); + + // Sibling tries while A holds it — must be excluded. + Optional b = lockProvider.lock(new LockConfiguration( + Instant.now(), name, Duration.ofMinutes(5), Duration.ZERO)); + assertFalse(b.isPresent(), "second acquirer should be blocked while first holds the lock"); + + // A releases. + a.get().unlock(); + + // Sibling tries again — should now succeed. + Optional c = lockProvider.lock(new LockConfiguration( + Instant.now(), name, Duration.ofMinutes(5), Duration.ZERO)); + assertTrue(c.isPresent(), "third acquirer should succeed after release"); + c.get().unlock(); + } + + @Test + @DisplayName("different lock names are independent — two jobs both proceed") + void independentLocks() { + Optional jobA = lockProvider.lock(new LockConfiguration( + Instant.now(), "cron-job-A", Duration.ofMinutes(5), Duration.ZERO)); + Optional jobB = lockProvider.lock(new LockConfiguration( + Instant.now(), "cron-job-B", Duration.ofMinutes(5), Duration.ZERO)); + + assertTrue(jobA.isPresent()); + assertTrue(jobB.isPresent(), + "different lock names must not block each other — multi-job parallelism is the whole point"); + + jobA.get().unlock(); + jobB.get().unlock(); + } + + @Test + @DisplayName("lockAtLeastFor prevents instant re-acquire by the same caller") + void lockAtLeastForHonored() { + String name = "test-lock-at-least"; + // Hold the lock for at least 2 seconds even if we release immediately. + Optional first = lockProvider.lock(new LockConfiguration( + Instant.now(), name, Duration.ofMinutes(5), Duration.ofSeconds(2))); + assertTrue(first.isPresent()); + first.get().unlock(); // unlock returns, but lockAtLeastFor still applies + + // Immediate re-acquire should fail because lockAtLeastFor=2s hasn't elapsed. + Optional second = lockProvider.lock(new LockConfiguration( + Instant.now(), name, Duration.ofMinutes(5), Duration.ZERO)); + assertFalse(second.isPresent(), + "lockAtLeastFor must keep the entry inaccessible for its duration even after unlock"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/cron/delivery/AbstractCronResultDeliveryTest.java b/mateclaw-server/src/test/java/vip/mate/cron/delivery/AbstractCronResultDeliveryTest.java new file mode 100644 index 00000000..49c8155a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/cron/delivery/AbstractCronResultDeliveryTest.java @@ -0,0 +1,176 @@ +package vip.mate.cron.delivery; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import vip.mate.cron.model.CronJobEntity; +import vip.mate.dashboard.model.CronJobRunEntity; +import vip.mate.dashboard.repository.CronJobRunMapper; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * RFC-063r §2.6.1: Template-Method invariants — SQL CAS claim, marker + * methods after success / failure, exception propagation. + */ +class AbstractCronResultDeliveryTest { + + private CronJobRunMapper runMapper; + private CronJobEntity job; + private CronJobRunEntity run; + + /** + * Pre-warm MyBatis Plus's lambda → column cache. Without this the + * production code's {@code new LambdaUpdateWrapper()} + * throws "can not find lambda cache" — the cache is normally populated + * during Spring context init, which we skip in unit tests. + */ + @BeforeAll + static void initMpLambdaCache() { + MybatisConfiguration cfg = new MybatisConfiguration(); + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(cfg, ""), CronJobRunEntity.class); + } + + @BeforeEach + void setUp() { + runMapper = mock(CronJobRunMapper.class); + job = new CronJobEntity(); + job.setId(1L); + run = new CronJobRunEntity(); + run.setId(42L); + run.setStatus("succeeded"); + } + + @Test + void deliver_claimsSuccessfully_marksDelivered() { + // First update = the claim CAS, returns 1 (won the race) + // Second update = the markDelivered, returns 1 + when(runMapper.update(any(), any(Wrapper.class))).thenReturn(1, 1); + + AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) { + @Override public boolean supports(CronJobEntity j) { return true; } + @Override + protected DeliveryOutcome doDeliver(CronJobEntity j, AssistantMessage r, CronJobRunEntity run) { + return DeliveryOutcome.delivered("user-x"); + } + }; + + DeliveryOutcome outcome = strategy.deliver(job, new AssistantMessage("hi"), run); + + assertEquals(DeliveryOutcome.Status.DELIVERED, outcome.status()); + assertEquals("user-x", outcome.target()); + verify(runMapper, times(2)).update(any(), any(Wrapper.class)); // claim + markDelivered + } + + @Test + void deliver_claimAlreadyTaken_returnsSkippedAndDoesNotInvokeDoDeliver() { + // Claim returns 0 → another listener already won the CAS + when(runMapper.update(any(), any(Wrapper.class))).thenReturn(0); + + AtomicReference doDeliverInvoked = new AtomicReference<>(false); + AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) { + @Override public boolean supports(CronJobEntity j) { return true; } + @Override + protected DeliveryOutcome doDeliver(CronJobEntity j, AssistantMessage r, CronJobRunEntity run) { + doDeliverInvoked.set(true); + return DeliveryOutcome.delivered("never"); + } + }; + + DeliveryOutcome outcome = strategy.deliver(job, new AssistantMessage("hi"), run); + + assertEquals(DeliveryOutcome.Status.SKIPPED, outcome.status()); + assertEquals("already-claimed-by-other-instance", outcome.reason()); + assertFalse(doDeliverInvoked.get(), "doDeliver must not run after a failed CAS claim"); + verify(runMapper, times(1)).update(any(), any(Wrapper.class)); // only the failed claim + } + + @Test + void deliver_doDeliverThrows_marksNotDeliveredAndRethrows() { + // Claim returns 1, then markNotDelivered returns 1 + when(runMapper.update(any(), any(Wrapper.class))).thenReturn(1, 1); + + RuntimeException oops = new RuntimeException("Slack 503 Service Unavailable"); + AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) { + @Override public boolean supports(CronJobEntity j) { return true; } + @Override + protected DeliveryOutcome doDeliver(CronJobEntity j, AssistantMessage r, CronJobRunEntity run) { + throw oops; + } + }; + + RuntimeException thrown = assertThrows(RuntimeException.class, + () -> strategy.deliver(job, new AssistantMessage("hi"), run)); + assertSame(oops, thrown, "exception must propagate verbatim so the listener can audit it"); + verify(runMapper, times(2)).update(any(), any(Wrapper.class)); // claim + markNotDelivered + } + + @Test + void claimRun_concurrentInvocations_onlyOneSucceeds() throws Exception { + // Simulates the cluster scenario: the SQL CAS guarantees exactly one + // listener instance wins. Mock the mapper so the FIRST update() call + // returns 1, all subsequent return 0 — matches DB semantics. + Set winnerThreadIds = java.util.Collections.synchronizedSet(new HashSet<>()); + AtomicReference firstClaim = new AtomicReference<>(true); + when(runMapper.update(any(), any(Wrapper.class))).thenAnswer(inv -> { + // First caller wins, others lose + return firstClaim.compareAndSet(true, false) ? 1 : 0; + }); + + AbstractCronResultDelivery strategy = new AbstractCronResultDelivery(runMapper) { + @Override public boolean supports(CronJobEntity j) { return true; } + @Override + protected DeliveryOutcome doDeliver(CronJobEntity j, AssistantMessage r, CronJobRunEntity run) { + winnerThreadIds.add((int) Thread.currentThread().threadId()); + return DeliveryOutcome.delivered("winner"); + } + }; + + int threadCount = 8; + CountDownLatch start = new CountDownLatch(1); + var pool = Executors.newFixedThreadPool(threadCount); + try { + var futures = IntStream.range(0, threadCount).mapToObj(i -> pool.submit(() -> { + start.await(); + return strategy.deliver(job, new AssistantMessage("hi"), run); + })).toList(); + start.countDown(); + + int delivered = 0; + int skipped = 0; + for (var f : futures) { + try { + DeliveryOutcome o = f.get(); + if (o.status() == DeliveryOutcome.Status.DELIVERED) delivered++; + else skipped++; + } catch (ExecutionException ignored) { + // doDeliver throws are OK; counted as not-delivered + } + } + + assertEquals(1, delivered, + "Exactly one winner under concurrent claim — RFC-063r §2.6.1 invariant"); + assertEquals(threadCount - 1, skipped, "All others must observe SKIPPED"); + assertEquals(1, winnerThreadIds.size(), + "doDeliver must execute on exactly one thread"); + } finally { + pool.shutdownNow(); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/cron/delivery/ChannelCronResultDeliveryTest.java b/mateclaw-server/src/test/java/vip/mate/cron/delivery/ChannelCronResultDeliveryTest.java new file mode 100644 index 00000000..e3b2d030 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/cron/delivery/ChannelCronResultDeliveryTest.java @@ -0,0 +1,117 @@ +package vip.mate.cron.delivery; + +import com.baomidou.mybatisplus.core.MybatisConfiguration; +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.messages.AssistantMessage; +import vip.mate.channel.ChannelManager; +import vip.mate.channel.DeliveryOptions; +import vip.mate.cron.model.CronJobEntity; +import vip.mate.cron.model.DeliveryConfig; +import vip.mate.dashboard.model.CronJobRunEntity; +import vip.mate.dashboard.repository.CronJobRunMapper; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * RFC-063r §2.6: ChannelCronResultDelivery dispatch contract. + */ +class ChannelCronResultDeliveryTest { + + private CronJobRunMapper runMapper; + private ChannelManager channelManager; + private ChannelCronResultDelivery strategy; + + @BeforeAll + static void initMpLambdaCache() { + MybatisConfiguration cfg = new MybatisConfiguration(); + TableInfoHelper.initTableInfo(new MapperBuilderAssistant(cfg, ""), CronJobRunEntity.class); + } + + @BeforeEach + void setUp() { + runMapper = mock(CronJobRunMapper.class); + channelManager = mock(ChannelManager.class); + when(runMapper.update(any(), any(Wrapper.class))).thenReturn(1); + strategy = new ChannelCronResultDelivery(runMapper, channelManager); + } + + @Test + void supports_channelIdNull_returnsFalse() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(null); + job.setDeliveryConfig(new DeliveryConfig("u", null, null)); + assertFalse(strategy.supports(job), + "web-origin runs (no channelId) must not match the channel strategy"); + } + + @Test + void supports_targetIdNull_returnsFalse() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(9L); + job.setDeliveryConfig(new DeliveryConfig(null, "thread-1", null)); + assertFalse(strategy.supports(job), + "channel binding without targetId must not deliver"); + } + + @Test + void supports_targetIdBlank_returnsFalse() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(9L); + job.setDeliveryConfig(new DeliveryConfig(" ", null, null)); + assertFalse(strategy.supports(job), + "blank targetId must be treated as missing"); + } + + @Test + void supports_channelAndTargetSet_returnsTrue() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(9L); + job.setDeliveryConfig(new DeliveryConfig("user-7", null, null)); + assertTrue(strategy.supports(job)); + } + + @Test + void doDeliver_callsChannelManagerWithDeliveryOptions() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(9L); + job.setDeliveryConfig(new DeliveryConfig("user-7", "thread-abc", "bot-001")); + CronJobRunEntity run = new CronJobRunEntity(); + run.setId(42L); + + DeliveryOutcome outcome = strategy.deliver(job, new AssistantMessage("Daily summary"), run); + + assertEquals(DeliveryOutcome.Status.DELIVERED, outcome.status()); + assertEquals("user-7", outcome.target()); + verify(channelManager).sendToChannel(eq(9L), eq("user-7"), any(String.class), + argThat(opts -> "thread-abc".equals(opts.threadId()) + && "bot-001".equals(opts.accountId()))); + } + + @Test + void doDeliver_adapterDisabled_propagatesIllegalStateAndMarksNotDelivered() { + CronJobEntity job = new CronJobEntity(); + job.setChannelId(9L); + job.setDeliveryConfig(new DeliveryConfig("user-7", null, null)); + CronJobRunEntity run = new CronJobRunEntity(); + run.setId(42L); + + // Simulate channel adapter unavailable — ChannelManager throws. + IllegalStateException disabled = new IllegalStateException("Channel not active: 9"); + doThrow(disabled).when(channelManager) + .sendToChannel(eq(9L), eq("user-7"), any(String.class), any(DeliveryOptions.class)); + + IllegalStateException thrown = assertThrows(IllegalStateException.class, + () -> strategy.deliver(job, new AssistantMessage("hi"), run)); + assertSame(disabled, thrown); + // Two updates: claim + markNotDelivered + verify(runMapper, times(2)).update(any(), any(Wrapper.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/cron/model/DeliveryConfigTest.java b/mateclaw-server/src/test/java/vip/mate/cron/model/DeliveryConfigTest.java new file mode 100644 index 00000000..00167ad6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/cron/model/DeliveryConfigTest.java @@ -0,0 +1,96 @@ +package vip.mate.cron.model; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import vip.mate.agent.context.ChannelTarget; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-063r §2.9: DeliveryConfig must round-trip through Jackson cleanly so + * MyBatis Plus JacksonTypeHandler can persist + restore it on + * {@code mate_cron_job.delivery_config}. + */ +class DeliveryConfigTest { + + @Test + void from_nullChannelTarget_returnsNull() { + assertNull(DeliveryConfig.from(null)); + } + + @Test + void roundTripThroughChannelTarget() { + ChannelTarget t = new ChannelTarget("user-1", "thread-a", "bot-x"); + DeliveryConfig dc = DeliveryConfig.from(t); + assertEquals(t, dc.toChannelTarget()); + } + + @Test + void jsonRoundTrip_preservesAllFields() throws Exception { + ObjectMapper om = new ObjectMapper(); + DeliveryConfig original = new DeliveryConfig("user-1", "thread-a", "bot-x"); + String json = om.writeValueAsString(original); + DeliveryConfig restored = om.readValue(json, DeliveryConfig.class); + assertEquals(original, restored); + } + + @Test + void jsonDeserialize_unknownFieldsAreIgnored() throws Exception { + ObjectMapper om = new ObjectMapper(); + String json = "{\"targetId\":\"u\",\"threadId\":null,\"accountId\":null,\"newFieldFromFuture\":\"y\"}"; + DeliveryConfig dc = om.readValue(json, DeliveryConfig.class); + assertEquals("u", dc.targetId()); + } + + // ── RFC-03 Lane C1: suppressAgentReply ───────────────────────────────── + + @Test + void suppressAgentReply_defaultsToFalse_legacyCtor3arg() { + // Pre-RFC-03 callsite — no suppress arg means historical behavior. + DeliveryConfig dc = new DeliveryConfig("u", null, null); + assertFalse(dc.isAgentReplySuppressed()); + assertNull(dc.suppressAgentReply()); + } + + @Test + void suppressAgentReply_defaultsToFalse_legacyCtor4arg() { + // 4-arg legacy ctor (post-userId, pre-suppress). + DeliveryConfig dc = new DeliveryConfig("u", null, null, "sender"); + assertFalse(dc.isAgentReplySuppressed()); + assertNull(dc.suppressAgentReply()); + } + + @Test + void suppressAgentReply_explicitFalseStillDelivers() { + DeliveryConfig dc = new DeliveryConfig("u", null, null, null, Boolean.FALSE); + assertFalse(dc.isAgentReplySuppressed(), + "explicit FALSE must be treated identically to null — both deliver"); + } + + @Test + void suppressAgentReply_trueShortCircuits() { + DeliveryConfig dc = new DeliveryConfig("u", null, null, null, Boolean.TRUE); + assertTrue(dc.isAgentReplySuppressed()); + } + + @Test + void suppressAgentReply_jsonRoundTrip() throws Exception { + ObjectMapper om = new ObjectMapper(); + DeliveryConfig original = new DeliveryConfig("u", "t", "a", "sender", Boolean.TRUE); + String json = om.writeValueAsString(original); + DeliveryConfig restored = om.readValue(json, DeliveryConfig.class); + assertEquals(original, restored); + assertTrue(restored.isAgentReplySuppressed()); + } + + @Test + void suppressAgentReply_preV75JsonRow_treatedAsFalse() throws Exception { + // Rows persisted before V75 don't have suppressAgentReply at all — + // round-trip must surface as null and isAgentReplySuppressed=false. + ObjectMapper om = new ObjectMapper(); + String legacyJson = "{\"targetId\":\"u\",\"threadId\":\"t\",\"accountId\":\"a\",\"userId\":\"sender\"}"; + DeliveryConfig dc = om.readValue(legacyJson, DeliveryConfig.class); + assertNull(dc.suppressAgentReply()); + assertFalse(dc.isAgentReplySuppressed()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/cron/service/CronJobRunnerDeliveryGuardTest.java b/mateclaw-server/src/test/java/vip/mate/cron/service/CronJobRunnerDeliveryGuardTest.java new file mode 100644 index 00000000..8d633db7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/cron/service/CronJobRunnerDeliveryGuardTest.java @@ -0,0 +1,45 @@ +package vip.mate.cron.service; + +import org.junit.jupiter.api.Test; +import vip.mate.agent.context.ChannelTarget; +import vip.mate.agent.context.ChatOrigin; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-063r §2.13 (Issue #25 — second symptom): + * {@link CronJobRunner#wrapWithDeliveryGuard} must prepend a system note + * for channel-bound cron runs and pass through web-origin runs unchanged. + */ +class CronJobRunnerDeliveryGuardTest { + + @Test + void channelBoundCron_prependsDeliveryGuard() { + ChatOrigin channelOrigin = new ChatOrigin( + /* agentId */ 7L, "cron_7", "system", 1L, null, + /* channelId */ 9L, new ChannelTarget("group-a", null, null)); + String input = "提醒我喝水并发到微信"; + String wrapped = CronJobRunner.wrapWithDeliveryGuard(input, channelOrigin); + + assertTrue(wrapped.contains("[系统说明]"), + "Channel-bound cron must include system note (RFC-063r §2.13)"); + assertTrue(wrapped.contains("不要尝试调用 CLI"), + "system note must explicitly forbid CLI hallucination"); + assertTrue(wrapped.endsWith(input), + "user message must be appended after the system note"); + } + + @Test + void webOriginCron_passesThroughUnchanged() { + ChatOrigin webOrigin = ChatOrigin.web("cron_1", "system", 1L, null); + String input = "Daily wiki update"; + assertEquals(input, CronJobRunner.wrapWithDeliveryGuard(input, webOrigin), + "web-origin cron must keep pre-RFC behavior"); + } + + @Test + void emptyOrigin_passesThroughUnchanged() { + assertEquals("hello", CronJobRunner.wrapWithDeliveryGuard("hello", ChatOrigin.EMPTY)); + assertEquals("hello", CronJobRunner.wrapWithDeliveryGuard("hello", null)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/hook/action/HttpActionHmacTest.java b/mateclaw-server/src/test/java/vip/mate/hook/action/HttpActionHmacTest.java new file mode 100644 index 00000000..3f32e0a1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/hook/action/HttpActionHmacTest.java @@ -0,0 +1,108 @@ +package vip.mate.hook.action; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.web.client.RestClient; + +import java.net.URI; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-03 Lane H1 — covers {@link HttpAction#hmacSign(String)} and the + * default-header convention used to deliver outbound webhook signatures. + * + *

Validating the signature on the receiver side requires the digest to be: + *

    + *
  1. computed over the exact bytes that were sent (no JSON re-encode),
  2. + *
  3. formatted as {@code "sha256="} so off-the-shelf + * GitHub-style validators work without changes,
  4. + *
  5. deterministic — same secret + same body always yields the same + * digest (no timestamp / nonce mixed in here).
  6. + *
+ * + *

The reference vector is from RFC 4231 §4.7 (HMAC-SHA-256 with the + * canonical "Test 7" inputs) so any divergence from the standard surfaces + * here, not in production. + */ +class HttpActionHmacTest { + + /** Build an HttpAction with the given secret; restClient is a no-op stub + * because hmacSign() doesn't touch it. */ + private static HttpAction action(String secret) { + return new HttpAction( + RestClient.builder().build(), + "POST", + URI.create("https://hooks.example.com/test"), + null, + List.of("hooks.example.com"), + 3000L, + secret, + null); + } + + @Test + @DisplayName("hmacSign produces lowercase-hex 'sha256=' format") + void formatIsGitHubCompatible() { + String sig = action("secret").hmacSign("hello"); + assertTrue(sig.startsWith("sha256="), + "header value must be sha256-prefixed for GitHub-compatible validators"); + // SHA-256 hex digest is 64 lowercase chars, no separators. + String hex = sig.substring("sha256=".length()); + assertEquals(64, hex.length()); + assertTrue(hex.matches("[0-9a-f]+"), + "digest must be lowercase hex; got: " + hex); + } + + @Test + @DisplayName("Wikipedia reference vector — known input → known digest") + void referenceVector() { + // From the canonical HMAC-SHA-256 worked example + // (Wikipedia "HMAC" article — same input/output as Bruce Schneier's + // applied-cryptography vector). Hardcoding the expected digest catches + // any divergence from the JCA reference impl — e.g. if someone later + // swaps in a third-party Mac or a Bouncy Castle provider that returns + // a different byte order. + String sig = action("key").hmacSign("The quick brown fox jumps over the lazy dog"); + assertEquals( + "sha256=f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8", + sig); + } + + @Test + @DisplayName("same secret + same body → identical digest (deterministic)") + void deterministic() { + HttpAction a = action("shared-secret-123"); + String first = a.hmacSign("{\"event\":\"agent.completed\"}"); + String second = a.hmacSign("{\"event\":\"agent.completed\"}"); + assertEquals(first, second); + } + + @Test + @DisplayName("different secrets → different digests") + void secretMattersForDigest() { + String body = "{\"event\":\"x\"}"; + String s1 = action("secret-A").hmacSign(body); + String s2 = action("secret-B").hmacSign(body); + assertTrue(!s1.equals(s2), + "swapping the secret must change the digest — otherwise signing is theatre"); + } + + @Test + @DisplayName("different body bytes → different digests") + void bodyMattersForDigest() { + HttpAction a = action("secret"); + String s1 = a.hmacSign("{\"a\":1}"); + String s2 = a.hmacSign("{\"a\":2}"); + assertTrue(!s1.equals(s2), + "swapping a byte must change the digest — otherwise tampering goes undetected"); + } + + @Test + @DisplayName("default signature header constant matches MateClaw convention") + void defaultHeaderName() { + assertEquals("X-MateClaw-Signature", HttpAction.DEFAULT_SIGNATURE_HEADER); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/i18n/LocaleAwareToolCallbackToolContextTest.java b/mateclaw-server/src/test/java/vip/mate/i18n/LocaleAwareToolCallbackToolContextTest.java new file mode 100644 index 00000000..5edcf775 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/i18n/LocaleAwareToolCallbackToolContextTest.java @@ -0,0 +1,80 @@ +package vip.mate.i18n; + +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-063r §2.3 (P0): regression guard — {@link LocaleAwareToolCallback} must + * forward both the input string and the ToolContext to the wrapped callback. + * Pre-fix this class only overrode {@code call(String)}, silently dropping the + * context (and thus ChatOrigin) for every builtin tool. + */ +class LocaleAwareToolCallbackToolContextTest { + + @Test + void callWithToolContext_forwardsToDelegate() { + RecordingDelegate delegate = new RecordingDelegate(); + LocaleAwareToolCallback decorator = + new LocaleAwareToolCallback(delegate, "本地化描述"); + + ToolContext ctx = new ToolContext(Map.of("k", "v")); + String out = decorator.call("{\"x\":1}", ctx); + + assertEquals("ok", out); + assertEquals("{\"x\":1}", delegate.lastInput); + assertSame(ctx, delegate.lastContext, + "ToolContext must reach the underlying tool unchanged"); + } + + @Test + void getToolMetadata_isForwardedSoReturnDirectIsPreserved() { + ToolMetadata directMetadata = ToolMetadata.builder().returnDirect(true).build(); + RecordingDelegate delegate = new RecordingDelegate(); + delegate.metadata = directMetadata; + + LocaleAwareToolCallback decorator = new LocaleAwareToolCallback(delegate, "本地化描述"); + assertSame(directMetadata, decorator.getToolMetadata(), + "decorator must not flip returnDirect by inheriting the framework default"); + } + + private static final class RecordingDelegate implements ToolCallback { + String lastInput; + ToolContext lastContext; + ToolMetadata metadata = ToolMetadata.builder().build(); + + @Override + public ToolDefinition getToolDefinition() { + return ToolDefinition.builder() + .name("recording-tool") + .description("...") + .inputSchema("{}") + .build(); + } + + @Override + public ToolMetadata getToolMetadata() { + return metadata; + } + + @Override + public String call(String toolInput) { + this.lastInput = toolInput; + this.lastContext = null; + return "ok"; + } + + @Override + public String call(String toolInput, ToolContext toolContext) { + this.lastInput = toolInput; + this.lastContext = toolContext; + return "ok"; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeApiHeadersTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeApiHeadersTest.java new file mode 100644 index 00000000..c14124c7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeApiHeadersTest.java @@ -0,0 +1,79 @@ +package vip.mate.llm.anthropic.oauth; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Header-construction guarantees for OAuth-authenticated Anthropic requests. + * + *

The two non-negotiable invariants Anthropic's edge enforces: + *

    + *
  1. {@code anthropic-beta} must contain both {@code claude-code-20250219} + * AND {@code oauth-2025-04-20}, comma-joined (no spaces).
  2. + *
  3. {@code User-Agent} must be the bare {@code claude-cli/} — + * NOT {@code claude-cli/ (external, cli)}. The {@code (external, cli)} + * suffix is what hermes-agent and other third-party clients append, and + * Anthropic uses it as a fingerprint to rate-limit the anti-abuse path. + * Real Claude Code emits the bare form via the official JS SDK.
  4. + *
+ */ +class ClaudeCodeApiHeadersTest { + + private ClaudeCodeApiHeaders headers; + + @BeforeEach + void setUp() { + // Stub detector returns a stable version string so assertions stay deterministic. + ClaudeCodeVersionDetector stub = new ClaudeCodeVersionDetector() { + @Override + public String get() { return "2.1.114"; } + }; + headers = new ClaudeCodeApiHeaders(stub); + } + + @Test + @DisplayName("allBetas: common betas appear before OAuth-only betas (matches hermes-agent ordering)") + void allBetas_orderedCommonFirst() { + String result = headers.allBetas(); + int oauthIdx = result.indexOf("oauth-2025-04-20"); + int interleavedIdx = result.indexOf("interleaved-thinking-2025-05-14"); + assertTrue(oauthIdx >= 0, "oauth beta missing"); + assertTrue(interleavedIdx >= 0, "interleaved-thinking beta missing"); + assertTrue(interleavedIdx < oauthIdx, "common betas must precede OAuth-only betas"); + } + + @Test + @DisplayName("allBetas: comma-joined with no whitespace") + void allBetas_commaJoined() { + String result = headers.allBetas(); + // Anthropic's edge is strict — a stray space breaks the header parser. + assertTrue(result.contains("claude-code-20250219")); + assertTrue(result.contains("oauth-2025-04-20")); + assertTrue(result.contains(",")); + assertEquals(-1, result.indexOf(", ")); + assertEquals(-1, result.indexOf(" ,")); + } + + @Test + @DisplayName("userAgent: bare claude-cli/ (no suffix — anti-abuse fingerprint)") + void userAgent_format() { + // Critical: must NOT contain "(external, cli)" — see class javadoc. + assertEquals("claude-cli/2.1.114", headers.userAgent()); + } + + @Test + @DisplayName("xApp: returns the literal cli identifier") + void xApp() { + assertEquals("cli", headers.xApp()); + } + + @Test + @DisplayName("bearerAuth: prepends Bearer prefix exactly once") + void bearerAuth() { + assertEquals("Bearer abc123", headers.bearerAuth("abc123")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsReaderTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsReaderTest.java new file mode 100644 index 00000000..190252ae --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsReaderTest.java @@ -0,0 +1,145 @@ +package vip.mate.llm.anthropic.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Covers the JSON parsing path of {@link ClaudeCodeCredentialsReader}, which + * is the only path exercised on Linux/Windows servers. Keychain reading is + * a macOS-only ProcessBuilder integration — left to manual / live testing. + */ +class ClaudeCodeCredentialsReaderTest { + + private ClaudeCodeCredentialsReader reader; + + @BeforeEach + void setUp() { + reader = new ClaudeCodeCredentialsReader(new ObjectMapper()); + } + + @Test + @DisplayName("parseCredentials extracts all fields from the canonical envelope") + void parseCredentials_fullPayload() { + String json = """ + { + "claudeAiOauth": { + "accessToken": "sk-ant-oat01-test", + "refreshToken": "sk-ant-ort01-test", + "expiresAt": 1735689600000, + "scopes": ["user:inference", "user:profile"] + } + } + """; + Optional result = + reader.parseCredentials(json, ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertTrue(result.isPresent()); + ClaudeCodeCredentials c = result.get(); + assertEquals("sk-ant-oat01-test", c.accessToken()); + assertEquals("sk-ant-ort01-test", c.refreshToken()); + assertEquals(1735689600000L, c.expiresAtMs()); + assertEquals(ClaudeCodeCredentials.Source.CREDENTIALS_FILE, c.source()); + } + + @Test + @DisplayName("parseCredentials returns empty when claudeAiOauth missing") + void parseCredentials_missingEnvelope() { + // Some users have only {primaryApiKey: "..."} in ~/.claude.json — that's + // an Anthropic console managed key, not OAuth, so we must NOT pretend + // it's a Claude Code credential. + Optional result = reader.parseCredentials( + "{\"primaryApiKey\":\"sk-ant-test\"}", + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertFalse(result.isPresent()); + } + + @Test + @DisplayName("parseCredentials returns empty when accessToken blank") + void parseCredentials_blankToken() { + String json = """ + { "claudeAiOauth": { "accessToken": "", "refreshToken": "rt" } } + """; + Optional result = + reader.parseCredentials(json, ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertFalse(result.isPresent()); + } + + @Test + @DisplayName("parseCredentials handles missing refreshToken gracefully") + void parseCredentials_missingRefreshToken() { + // Older Claude Code versions wrote the access token without a refresh + // token. Reader must still surface those — refresh just won't be possible. + String json = """ + { "claudeAiOauth": { "accessToken": "at-only", "expiresAt": 0 } } + """; + Optional result = + reader.parseCredentials(json, ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertTrue(result.isPresent()); + assertEquals("at-only", result.get().accessToken()); + assertFalse(result.get().canRefresh()); + } + + @Test + @DisplayName("parseCredentials rejects malformed JSON without throwing") + void parseCredentials_badJson() { + Optional result = reader.parseCredentials( + "{not json", ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertFalse(result.isPresent()); + } + + @Test + @DisplayName("parseCredentials returns empty for null/blank input") + void parseCredentials_blankInput() { + assertFalse(reader.parseCredentials(null, ClaudeCodeCredentials.Source.CREDENTIALS_FILE).isPresent()); + assertFalse(reader.parseCredentials("", ClaudeCodeCredentials.Source.CREDENTIALS_FILE).isPresent()); + assertFalse(reader.parseCredentials(" ", ClaudeCodeCredentials.Source.CREDENTIALS_FILE).isPresent()); + } + + @Test + @DisplayName("readFromJsonFile returns empty for missing path") + void readFromJsonFile_missing(@TempDir Path tmp) { + Path absent = tmp.resolve("nonexistent.json"); + assertFalse(reader.readFromJsonFile(absent).isPresent()); + } + + @Test + @DisplayName("readFromJsonFile reads + parses an existing file") + void readFromJsonFile_present(@TempDir Path tmp) throws IOException { + Path file = tmp.resolve(".credentials.json"); + Files.writeString(file, """ + { "claudeAiOauth": { + "accessToken": "from-file", + "refreshToken": "rt-from-file", + "expiresAt": 0 + } } + """, StandardCharsets.UTF_8); + + Optional result = reader.readFromJsonFile(file); + assertTrue(result.isPresent()); + assertEquals("from-file", result.get().accessToken()); + assertEquals(ClaudeCodeCredentials.Source.CREDENTIALS_FILE, result.get().source()); + } + + @Test + @DisplayName("readFromKeychain returns empty on non-macOS hosts") + void readFromKeychain_nonMacOs() { + // Override isMacOs() to false so the test passes regardless of CI host. + ClaudeCodeCredentialsReader linux = new ClaudeCodeCredentialsReader(new ObjectMapper()) { + @Override + boolean isMacOs() { return false; } + }; + assertFalse(linux.readFromKeychain().isPresent()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsTest.java new file mode 100644 index 00000000..e13fc543 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsTest.java @@ -0,0 +1,57 @@ +package vip.mate.llm.anthropic.oauth; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates the pure-data invariants of {@link ClaudeCodeCredentials} — + * specifically the {@code isValid(buffer)} expiry math and {@code canRefresh} + * predicate. Exercising these here means downstream services can rely on the + * record without re-implementing the same checks. + */ +class ClaudeCodeCredentialsTest { + + @Test + @DisplayName("isValid: blank access token always invalid") + void isValid_blankToken_false() { + assertFalse(creds("", "rt", System.currentTimeMillis() + 60_000).isValid(0L)); + assertFalse(creds(null, "rt", System.currentTimeMillis() + 60_000).isValid(0L)); + } + + @Test + @DisplayName("isValid: expiresAt=0 means no expiry — always valid when token present") + void isValid_zeroExpiry_alwaysValid() { + assertTrue(creds("at", "rt", 0L).isValid(60_000L)); + } + + @Test + @DisplayName("isValid: returns false within buffer window") + void isValid_withinBuffer_false() { + long now = System.currentTimeMillis(); + // Token expires in 30s; buffer is 60s → invalid (must refresh before expiry). + assertFalse(creds("at", "rt", now + 30_000L).isValid(60_000L)); + } + + @Test + @DisplayName("isValid: returns true outside buffer window") + void isValid_outsideBuffer_true() { + long now = System.currentTimeMillis(); + // Token expires in 5 minutes; 60s buffer → still valid. + assertTrue(creds("at", "rt", now + 300_000L).isValid(60_000L)); + } + + @Test + @DisplayName("canRefresh: requires non-blank refresh token") + void canRefresh() { + assertTrue(creds("at", "rt", 0L).canRefresh()); + assertFalse(creds("at", "", 0L).canRefresh()); + assertFalse(creds("at", null, 0L).canRefresh()); + } + + private static ClaudeCodeCredentials creds(String at, String rt, long expiresAt) { + return new ClaudeCodeCredentials(at, rt, expiresAt, ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsWriterTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsWriterTest.java new file mode 100644 index 00000000..eadfbd56 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeCredentialsWriterTest.java @@ -0,0 +1,182 @@ +package vip.mate.llm.anthropic.oauth; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates the JSON-file write path of {@link ClaudeCodeCredentialsWriter}, + * with focus on the two correctness-critical behaviors: + * + *
    + *
  1. Concurrent-write defence: when Claude Code itself rewrites the file + * while MateClaw is mid-refresh, the writer must NOT clobber.
  2. + *
  3. Scope preservation: the writer must keep the {@code scopes} array + * (Claude Code >= 2.1.81 needs {@code user:inference} or it shows + * the user as logged-out).
  4. + *
+ */ +class ClaudeCodeCredentialsWriterTest { + + private ObjectMapper mapper; + private ClaudeCodeCredentialsWriter writer; + + @BeforeEach + void setUp() { + mapper = new ObjectMapper(); + writer = new ClaudeCodeCredentialsWriter(mapper); + } + + @Test + @DisplayName("writeJsonFile creates a new file when none exists") + void writeJsonFile_createsNew(@TempDir Path tmp) throws IOException { + Path target = tmp.resolve(".credentials.json"); + ClaudeCodeCredentials fresh = new ClaudeCodeCredentials( + "new-access", "new-refresh", 9_999_999_999L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + + boolean ok = writer.writeJsonFile(target, null, fresh); + assertTrue(ok); + assertTrue(Files.exists(target)); + + JsonNode root = mapper.readTree(Files.readString(target, StandardCharsets.UTF_8)); + JsonNode oauth = root.path("claudeAiOauth"); + assertEquals("new-access", oauth.path("accessToken").asText()); + assertEquals("new-refresh", oauth.path("refreshToken").asText()); + assertEquals(9_999_999_999L, oauth.path("expiresAt").asLong()); + // Default scope must be present so Claude Code 2.1.81+ keeps recognising + // the credential after MateClaw writes to it. + assertTrue(oauth.path("scopes").isArray()); + assertEquals("user:inference", oauth.path("scopes").get(0).asText()); + } + + @Test + @DisplayName("writeJsonFile preserves existing scopes") + void writeJsonFile_preservesScopes(@TempDir Path tmp) throws IOException { + Path target = tmp.resolve(".credentials.json"); + Files.writeString(target, """ + { "claudeAiOauth": { + "accessToken": "old-token", + "refreshToken": "old-refresh", + "expiresAt": 1, + "scopes": ["user:inference", "user:profile", "extra:scope"] + } } + """, StandardCharsets.UTF_8); + + ClaudeCodeCredentials fresh = new ClaudeCodeCredentials( + "new-access", "new-refresh", 9_999_999_999L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + boolean ok = writer.writeJsonFile(target, "old-token", fresh); + assertTrue(ok); + + JsonNode oauth = mapper.readTree(Files.readString(target, StandardCharsets.UTF_8)) + .path("claudeAiOauth"); + assertEquals("new-access", oauth.path("accessToken").asText()); + // All three original scopes survive — the writer mutates only the + // fields it owns (access/refresh/expiresAt). + assertEquals(3, oauth.path("scopes").size()); + assertEquals("user:profile", oauth.path("scopes").get(1).asText()); + assertEquals("extra:scope", oauth.path("scopes").get(2).asText()); + } + + @Test + @DisplayName("writeJsonFile preserves unknown top-level fields") + void writeJsonFile_preservesUnknownFields(@TempDir Path tmp) throws IOException { + // Defends against future Claude Code releases that add new fields: + // we must not strip them on rewrite. + Path target = tmp.resolve(".credentials.json"); + Files.writeString(target, """ + { + "claudeAiOauth": { "accessToken": "x", "expiresAt": 1 }, + "futureField": { "foo": "bar" } + } + """, StandardCharsets.UTF_8); + + ClaudeCodeCredentials fresh = new ClaudeCodeCredentials( + "new-access", "new-refresh", 100L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + writer.writeJsonFile(target, "x", fresh); + + JsonNode root = mapper.readTree(Files.readString(target, StandardCharsets.UTF_8)); + assertEquals("bar", root.path("futureField").path("foo").asText()); + } + + @Test + @DisplayName("writeJsonFile bails out when on-disk token already changed") + void writeJsonFile_concurrentWriteDetected(@TempDir Path tmp) throws IOException { + // Simulate: MateClaw started a refresh from token "T1", Claude Code + // beat us to it and wrote "T2". MateClaw must NOT overwrite. + Path target = tmp.resolve(".credentials.json"); + Files.writeString(target, """ + { "claudeAiOauth": { + "accessToken": "T2", + "refreshToken": "rt2", + "expiresAt": 99, + "scopes": ["user:inference"] + } } + """, StandardCharsets.UTF_8); + + ClaudeCodeCredentials fresh = new ClaudeCodeCredentials( + "T3", "rt3", 100L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + boolean ok = writer.writeJsonFile(target, "T1", fresh); + assertFalse(ok, "writer must refuse to overwrite a concurrently-updated file"); + + // Disk contents unchanged + JsonNode oauth = mapper.readTree(Files.readString(target, StandardCharsets.UTF_8)) + .path("claudeAiOauth"); + assertEquals("T2", oauth.path("accessToken").asText()); + } + + @Test + @DisplayName("writeJsonFile proceeds when previousAccessToken is null (first-time write)") + void writeJsonFile_nullPrevious_proceeds(@TempDir Path tmp) throws IOException { + Path target = tmp.resolve(".credentials.json"); + Files.writeString(target, """ + { "claudeAiOauth": { "accessToken": "existing", "scopes": ["user:inference"] } } + """, StandardCharsets.UTF_8); + + ClaudeCodeCredentials fresh = new ClaudeCodeCredentials( + "fresh-token", "fresh-refresh", 0L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + // Null previous → caller doesn't have a baseline (e.g. first import) + // → skip concurrency check and just write. + assertTrue(writer.writeJsonFile(target, null, fresh)); + + JsonNode oauth = mapper.readTree(Files.readString(target, StandardCharsets.UTF_8)) + .path("claudeAiOauth"); + assertEquals("fresh-token", oauth.path("accessToken").asText()); + } + + @Test + @DisplayName("write rejects blank access tokens") + void write_rejectsBlankToken() { + ClaudeCodeCredentials blank = new ClaudeCodeCredentials( + " ", "rt", 0L, ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertFalse(writer.write(null, blank)); + } + + @Test + @DisplayName("writeKeychain returns false on non-macOS hosts") + void writeKeychain_nonMacOs() { + ClaudeCodeCredentialsWriter linux = new ClaudeCodeCredentialsWriter(mapper) { + @Override + boolean isMacOs() { return false; } + }; + ClaudeCodeCredentials creds = new ClaudeCodeCredentials( + "at", "rt", 0L, ClaudeCodeCredentials.Source.MACOS_KEYCHAIN); + assertFalse(linux.writeKeychain(null, creds)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeOAuthServiceTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeOAuthServiceTest.java new file mode 100644 index 00000000..5636235f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeOAuthServiceTest.java @@ -0,0 +1,225 @@ +package vip.mate.llm.anthropic.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.exception.MateClawException; + +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Tests the orchestration logic of {@link ClaudeCodeOAuthService} — the + * decision tree for "return cached token" / "refresh + persist" / "fail with + * actionable error". Uses test-double subclasses for Reader / Refresher / + * Writer to avoid hitting the filesystem or network. + */ +class ClaudeCodeOAuthServiceTest { + + private ObjectMapper mapper; + + @BeforeEach + void setUp() { + mapper = new ObjectMapper(); + } + + @Test + @DisplayName("getValidToken returns existing token when still valid") + void getValidToken_cached() { + ClaudeCodeCredentials valid = new ClaudeCodeCredentials( + "still-good", "rt", System.currentTimeMillis() + 600_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + ClaudeCodeOAuthService svc = serviceWith(valid, /* refreshShouldBeCalled */ false); + assertEquals("still-good", svc.getValidToken()); + } + + @Test + @DisplayName("getValidToken refreshes when within buffer window") + void getValidToken_refreshesNearExpiry() { + // Token expires in 30s; buffer is 60s → must refresh. + ClaudeCodeCredentials nearExpiry = new ClaudeCodeCredentials( + "old-token", "rt", System.currentTimeMillis() + 30_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + + AtomicReference capturedPreviousToken = new AtomicReference<>(); + AtomicReference capturedWritten = new AtomicReference<>(); + + ClaudeCodeCredentialsReader reader = stubReader(nearExpiry); + ClaudeCodeTokenRefresher refresher = stubRefresher(rt -> new ClaudeCodeCredentials( + "fresh-token", "fresh-rt", System.currentTimeMillis() + 3_600_000L, + ClaudeCodeCredentials.Source.REFRESH_RESPONSE)); + ClaudeCodeCredentialsWriter writer = stubWriter((prev, creds) -> { + capturedPreviousToken.set(prev); + capturedWritten.set(creds); + return true; + }); + + ClaudeCodeOAuthService svc = new ClaudeCodeOAuthService(reader, refresher, writer); + assertEquals("fresh-token", svc.getValidToken()); + + // Writer must receive the prior access token (for concurrency check) + // AND the credential pinned to the original source — not REFRESH_RESPONSE. + assertEquals("old-token", capturedPreviousToken.get()); + assertNotNull(capturedWritten.get()); + assertEquals("fresh-token", capturedWritten.get().accessToken()); + assertEquals(ClaudeCodeCredentials.Source.CREDENTIALS_FILE, capturedWritten.get().source(), + "write must target the source the credential was originally read from"); + } + + @Test + @DisplayName("getValidToken throws actionable error when no credentials on disk") + void getValidToken_noCredentials() { + ClaudeCodeOAuthService svc = new ClaudeCodeOAuthService( + stubReader(null), + stubRefresher(rt -> { throw new IllegalStateException("should not be called"); }), + stubWriter((prev, creds) -> { throw new IllegalStateException("should not be called"); })); + + MateClawException ex = assertThrows(MateClawException.class, svc::getValidToken); + assertEquals("err.anthropic.no_claude_code", ex.getMsgKey()); + } + + @Test + @DisplayName("getValidToken throws when token expired and no refresh available") + void getValidToken_expiredNoRefresh() { + ClaudeCodeCredentials expired = new ClaudeCodeCredentials( + "expired", "", System.currentTimeMillis() - 60_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + ClaudeCodeOAuthService svc = serviceWith(expired, false); + MateClawException ex = assertThrows(MateClawException.class, svc::getValidToken); + assertEquals("err.anthropic.token_expired_no_refresh", ex.getMsgKey()); + } + + @Test + @DisplayName("getValidToken still returns fresh token when persistence fails") + void getValidToken_writeFailureNonFatal() { + // Writer returning false (e.g. concurrent-write detected) must NOT + // turn into a request failure — the in-memory token is still good. + ClaudeCodeCredentials nearExpiry = new ClaudeCodeCredentials( + "stale", "rt", System.currentTimeMillis() - 60_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + ClaudeCodeOAuthService svc = new ClaudeCodeOAuthService( + stubReader(nearExpiry), + stubRefresher(rt -> new ClaudeCodeCredentials( + "refreshed", "rt2", System.currentTimeMillis() + 600_000L, + ClaudeCodeCredentials.Source.REFRESH_RESPONSE)), + stubWriter((prev, creds) -> false)); + assertEquals("refreshed", svc.getValidToken()); + } + + @Test + @DisplayName("isLoggedIn reflects on-disk state without triggering refresh") + void isLoggedIn() { + ClaudeCodeCredentials valid = new ClaudeCodeCredentials( + "tok", "rt", System.currentTimeMillis() + 600_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertTrue(serviceWith(valid, false).isLoggedIn()); + + // Expired token → not logged in (we don't auto-refresh from a status check). + ClaudeCodeCredentials expired = new ClaudeCodeCredentials( + "tok", "rt", System.currentTimeMillis() - 60_000L, + ClaudeCodeCredentials.Source.CREDENTIALS_FILE); + assertFalse(serviceWith(expired, false).isLoggedIn()); + + // No file → not logged in. + ClaudeCodeOAuthService noCreds = new ClaudeCodeOAuthService( + stubReader(null), + stubRefresher(rt -> { throw new IllegalStateException(); }), + stubWriter((p, c) -> { throw new IllegalStateException(); })); + assertFalse(noCreds.isLoggedIn()); + } + + @Test + @DisplayName("getStatus surfaces source + expiry without exposing the token") + void getStatus_disconnected() { + ClaudeCodeOAuthService svc = new ClaudeCodeOAuthService( + stubReader(null), + stubRefresher(rt -> { throw new IllegalStateException(); }), + stubWriter((p, c) -> { throw new IllegalStateException(); })); + ClaudeCodeOAuthService.OAuthStatus status = svc.getStatus(); + assertFalse(status.connected()); + assertFalse(status.expired()); + } + + @Test + @DisplayName("getStatus reports expired flag correctly") + void getStatus_expired() { + ClaudeCodeCredentials expired = new ClaudeCodeCredentials( + "tok", "rt", System.currentTimeMillis() - 1_000L, + ClaudeCodeCredentials.Source.MACOS_KEYCHAIN); + ClaudeCodeOAuthService svc = serviceWith(expired, false); + ClaudeCodeOAuthService.OAuthStatus status = svc.getStatus(); + assertTrue(status.connected()); + assertTrue(status.expired()); + assertEquals(ClaudeCodeCredentials.Source.MACOS_KEYCHAIN, status.source()); + } + + /* ---------- Test-double helpers ---------- */ + + /** Build a service whose reader returns the given credentials and whose refresher/writer fail loudly if invoked. */ + private ClaudeCodeOAuthService serviceWith(ClaudeCodeCredentials creds, boolean expectRefresh) { + return new ClaudeCodeOAuthService( + stubReader(creds), + stubRefresher(rt -> { + if (!expectRefresh) { + throw new IllegalStateException("refresher should not have been called"); + } + return new ClaudeCodeCredentials("refreshed", "rt2", + System.currentTimeMillis() + 3_600_000L, + ClaudeCodeCredentials.Source.REFRESH_RESPONSE); + }), + stubWriter((prev, c) -> { + if (!expectRefresh) { + throw new IllegalStateException("writer should not have been called"); + } + return true; + })); + } + + private ClaudeCodeCredentialsReader stubReader(ClaudeCodeCredentials toReturn) { + return new ClaudeCodeCredentialsReader(mapper) { + @Override + public Optional read() { + return Optional.ofNullable(toReturn); + } + }; + } + + @FunctionalInterface + private interface RefreshFn { + ClaudeCodeCredentials apply(String refreshToken); + } + + private ClaudeCodeTokenRefresher stubRefresher(RefreshFn fn) { + ClaudeCodeVersionDetector ver = new ClaudeCodeVersionDetector() { + @Override + public String get() { return "2.1.114"; } + }; + return new ClaudeCodeTokenRefresher(mapper, ver) { + @Override + public ClaudeCodeCredentials refresh(String refreshToken) { + return fn.apply(refreshToken); + } + }; + } + + @FunctionalInterface + private interface WriteFn { + boolean apply(String previousAccessToken, ClaudeCodeCredentials creds); + } + + private ClaudeCodeCredentialsWriter stubWriter(WriteFn fn) { + return new ClaudeCodeCredentialsWriter(mapper) { + @Override + public boolean write(String previousAccessToken, ClaudeCodeCredentials refreshed) { + return fn.apply(previousAccessToken, refreshed); + } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeTokenRefresherTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeTokenRefresherTest.java new file mode 100644 index 00000000..07046ea4 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeTokenRefresherTest.java @@ -0,0 +1,115 @@ +package vip.mate.llm.anthropic.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.exception.MateClawException; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Validates the response-parsing logic of {@link ClaudeCodeTokenRefresher}. + * Network-bound paths (the actual POST to platform.claude.com) require either + * a wiremock or live fixtures and are out of scope for unit tests. + */ +class ClaudeCodeTokenRefresherTest { + + private ClaudeCodeTokenRefresher refresher; + + @BeforeEach + void setUp() { + ClaudeCodeVersionDetector versionStub = new ClaudeCodeVersionDetector() { + @Override + public String get() { return "2.1.114"; } + }; + refresher = new ClaudeCodeTokenRefresher(new ObjectMapper(), versionStub); + } + + @Test + @DisplayName("parseTokenResponse handles standard expires_in seconds") + void parseTokenResponse_expiresIn() { + long before = System.currentTimeMillis(); + String body = """ + { "access_token": "fresh-at", "refresh_token": "fresh-rt", "expires_in": 3600 } + """; + ClaudeCodeCredentials c = refresher.parseTokenResponse(body, "old-rt"); + assertEquals("fresh-at", c.accessToken()); + assertEquals("fresh-rt", c.refreshToken()); + // expires_in=3600 → expiresAt should be ~1h from now. + long expectedMin = before + 3_590_000L; + long expectedMax = System.currentTimeMillis() + 3_610_000L; + assertTrue(c.expiresAtMs() >= expectedMin && c.expiresAtMs() <= expectedMax, + "expiresAtMs " + c.expiresAtMs() + " out of expected range"); + assertEquals(ClaudeCodeCredentials.Source.REFRESH_RESPONSE, c.source()); + } + + @Test + @DisplayName("parseTokenResponse uses absolute expires_at when provided") + void parseTokenResponse_expiresAtMs() { + // Some Anthropic deployments return expires_at as an absolute ms value. + String body = """ + { "access_token": "at2", "expires_at": 1234567890000 } + """; + ClaudeCodeCredentials c = refresher.parseTokenResponse(body, "old-rt"); + assertEquals(1234567890000L, c.expiresAtMs()); + } + + @Test + @DisplayName("parseTokenResponse falls back to old refresh_token when response omits one") + void parseTokenResponse_keepsOldRefreshToken() { + // Anthropic docs say refresh_token may be omitted on rotation-disabled + // grants. We must NOT lose the original; otherwise the next refresh fails. + String body = """ + { "access_token": "at3", "expires_in": 3600 } + """; + ClaudeCodeCredentials c = refresher.parseTokenResponse(body, "preserved-rt"); + assertEquals("preserved-rt", c.refreshToken()); + } + + @Test + @DisplayName("parseTokenResponse rejects blank access_token") + void parseTokenResponse_blankToken_throws() { + // Edge case where Anthropic returns 200 with empty access_token — + // surface as a domain error rather than persisting garbage. + String body = """ + { "access_token": "", "expires_in": 3600 } + """; + MateClawException ex = assertThrows(MateClawException.class, + () -> refresher.parseTokenResponse(body, "rt")); + assertEquals("err.anthropic.refresh_failed", ex.getMsgKey()); + } + + @Test + @DisplayName("parseTokenResponse wraps malformed JSON") + void parseTokenResponse_badJson_throws() { + MateClawException ex = assertThrows(MateClawException.class, + () -> refresher.parseTokenResponse("not-json", "rt")); + assertEquals("err.anthropic.refresh_failed", ex.getMsgKey()); + } + + @Test + @DisplayName("refresh rejects blank refresh_token without making a network call") + void refresh_blankInput_throws() { + MateClawException ex = assertThrows(MateClawException.class, + () -> refresher.refresh("")); + // No network call made — the failure mode here is "no refresh available", + // not "refresh attempt failed". + assertNotEquals("err.anthropic.refresh_failed", ex.getMsgKey()); + assertEquals("err.anthropic.token_expired_no_refresh", ex.getMsgKey()); + } + + @Test + @DisplayName("ENDPOINTS includes both platform.claude.com and console.anthropic.com") + void endpoints_haveBothHosts() { + // Constants pinned by RFC-062. If Anthropic deprecates one, change here + // AND in the RFC; do not silently drop a fallback. + assertTrue(ClaudeCodeTokenRefresher.ENDPOINTS.stream() + .anyMatch(s -> s.contains("platform.claude.com"))); + assertTrue(ClaudeCodeTokenRefresher.ENDPOINTS.stream() + .anyMatch(s -> s.contains("console.anthropic.com"))); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeVersionDetectorTest.java b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeVersionDetectorTest.java new file mode 100644 index 00000000..643452c9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/anthropic/oauth/ClaudeCodeVersionDetectorTest.java @@ -0,0 +1,59 @@ +package vip.mate.llm.anthropic.oauth; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Static-helper coverage for {@link ClaudeCodeVersionDetector#parseVersion}. + * + *

The {@code claude --version} output format has shifted between Claude Code + * releases (early builds prefixed with the binary name; recent ones print just + * the number). The regex must match both so MateClaw stays in sync without + * manual config when users upgrade. + */ +class ClaudeCodeVersionDetectorTest { + + @Test + @DisplayName("parseVersion accepts the modern bare-number format") + void parseVersion_modern() { + assertEquals("2.1.114", ClaudeCodeVersionDetector.parseVersion("2.1.114")); + assertEquals("2.1.74", ClaudeCodeVersionDetector.parseVersion("2.1.74\n")); + } + + @Test + @DisplayName("parseVersion ignores trailing whitespace and extra suffix") + void parseVersion_withSuffix() { + assertEquals("2.1.114", ClaudeCodeVersionDetector.parseVersion("2.1.114 (Claude Code)")); + assertEquals("2.1.114", ClaudeCodeVersionDetector.parseVersion(" 2.1.114 ")); + } + + @Test + @DisplayName("parseVersion accepts a two-segment version") + void parseVersion_twoSegments() { + // Some legacy --version outputs printed only major.minor. + assertEquals("2.1", ClaudeCodeVersionDetector.parseVersion("2.1")); + } + + @Test + @DisplayName("parseVersion rejects non-numeric prefixes") + void parseVersion_rejectsNonNumeric() { + assertNull(ClaudeCodeVersionDetector.parseVersion("claude-code v2.1.114")); + assertNull(ClaudeCodeVersionDetector.parseVersion("")); + assertNull(ClaudeCodeVersionDetector.parseVersion(null)); + assertNull(ClaudeCodeVersionDetector.parseVersion("not a version")); + } + + @Test + @DisplayName("FALLBACK_VERSION constant is a real semver-shape string") + void fallbackVersion_isSemver() { + // Sanity-check the static fallback so a bad edit (e.g. typo) is caught + // before it ships in a User-Agent header. + String parsed = ClaudeCodeVersionDetector.parseVersion(ClaudeCodeVersionDetector.FALLBACK_VERSION); + assertNotNull(parsed); + assertEquals(ClaudeCodeVersionDetector.FALLBACK_VERSION, parsed); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/chatmodel/HttpTimeoutsTest.java b/mateclaw-server/src/test/java/vip/mate/llm/chatmodel/HttpTimeoutsTest.java new file mode 100644 index 00000000..5b469fd9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/chatmodel/HttpTimeoutsTest.java @@ -0,0 +1,71 @@ +package vip.mate.llm.chatmodel; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * RFC-03 Lane B1 — covers {@link HttpTimeouts#resolveReadTimeout(Integer)}, + * the central resolver that backs {@code mate_model_config.request_timeout_seconds}. + * + *

Behavioral contract under test: + *

    + *
  • null / non-positive → 180s (the historical hardcoded default; preserves + * behavior for every existing row before V75 ran).
  • + *
  • positive integer → that many seconds, no clamp (caller decides + * reasonable upper bound at the model-config level — we don't want to + * silently rewrite a user's deliberate 30-min override).
  • + *
  • connect timeout stays at 10s and is never overridable — long-tail + * latency manifests on the read path, not on connect.
  • + *
+ */ +class HttpTimeoutsTest { + + @Test + @DisplayName("null override → default 180s read timeout") + void nullFallsBack() { + assertEquals(Duration.ofSeconds(180), + HttpTimeouts.resolveReadTimeout(null)); + } + + @Test + @DisplayName("zero → default 180s (treated as unset)") + void zeroFallsBack() { + assertEquals(Duration.ofSeconds(180), + HttpTimeouts.resolveReadTimeout(0)); + } + + @Test + @DisplayName("negative → default 180s (defensively treats nonsense values as unset)") + void negativeFallsBack() { + assertEquals(Duration.ofSeconds(180), + HttpTimeouts.resolveReadTimeout(-30)); + } + + @Test + @DisplayName("positive integer → exact seconds, no clamp on either side") + void positiveHonored() { + assertEquals(Duration.ofSeconds(30), + HttpTimeouts.resolveReadTimeout(30)); + assertEquals(Duration.ofSeconds(600), + HttpTimeouts.resolveReadTimeout(600)); + // o1-pro / claude opus extended-thinking can legitimately need 30 min. + assertEquals(Duration.ofSeconds(1800), + HttpTimeouts.resolveReadTimeout(1800)); + } + + @Test + @DisplayName("connect timeout is the canonical 10s") + void connectTimeoutIsCanonical() { + assertEquals(Duration.ofSeconds(10), HttpTimeouts.CONNECT_TIMEOUT); + } + + @Test + @DisplayName("default read timeout matches the legacy hardcoded 180s") + void defaultMatchesLegacy() { + assertEquals(Duration.ofSeconds(180), HttpTimeouts.DEFAULT_READ_TIMEOUT); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java new file mode 100644 index 00000000..33f4d7f3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/AvailableProviderPoolTest.java @@ -0,0 +1,153 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.*; +import static vip.mate.llm.failover.AvailableProviderPool.RemovalSource; + +/** + * Unit tests for {@link AvailableProviderPool} — the membership data structure + * that gates the failover walker. + */ +class AvailableProviderPoolTest { + + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + pool = new AvailableProviderPool(); + } + + @Test + @DisplayName("New pool: nothing is in it") + void newPoolEmpty() { + assertFalse(pool.contains("openai")); + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("add then contains") + void addThenContains() { + pool.add("openai"); + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + } + + @Test + @DisplayName("Adding twice is idempotent") + void addIdempotent() { + pool.add("openai"); + pool.add("openai"); + assertTrue(pool.contains("openai")); + assertEquals(1, pool.snapshot().size()); + } + + @Test + @DisplayName("Remove after add: pool no longer contains, snapshot exposes reason") + void removeAfterAdd() { + pool.add("openai"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "401 Unauthorized"); + + assertFalse(pool.contains("openai")); + var snap = pool.snapshot(); + assertEquals(1, snap.size()); + assertNotNull(snap.get("openai")); + assertEquals(RemovalSource.AUTH_ERROR, snap.get("openai").source()); + assertEquals("401 Unauthorized", snap.get("openai").message()); + assertTrue(snap.get("openai").removedAtMs() > 0, "removedAtMs must be set"); + } + + @Test + @DisplayName("Remove without prior add still records reason (idempotent removal)") + void removeWithoutAddIsIdempotent() { + pool.remove("openai", RemovalSource.INIT_PROBE, "init failed"); + assertFalse(pool.contains("openai")); + assertNotNull(pool.snapshot().get("openai")); + } + + @Test + @DisplayName("Re-add after remove: contains true, removal reason cleared") + void readdClearsRemovalReason() { + pool.add("openai"); + pool.remove("openai", RemovalSource.AUTH_ERROR, "bad key"); + assertNotNull(pool.snapshot().get("openai")); + + pool.add("openai"); + assertTrue(pool.contains("openai")); + // Snapshot now shows openai in pool (value null), no stale reason + assertNull(pool.snapshot().get("openai"), + "re-adding a provider must clear its prior removal reason"); + } + + @Test + @DisplayName("Snapshot mixes in-pool (value=null) and removed (value=reason) entries") + void snapshotMixedView() { + pool.add("openai"); + pool.add("dashscope"); + pool.remove("anthropic", RemovalSource.MODEL_NOT_FOUND, "model claude-99 not found"); + + var snap = pool.snapshot(); + assertEquals(3, snap.size()); + assertNull(snap.get("openai"), "in-pool members appear with null value"); + assertNull(snap.get("dashscope")); + assertNotNull(snap.get("anthropic")); + assertEquals(RemovalSource.MODEL_NOT_FOUND, snap.get("anthropic").source()); + } + + @Test + @DisplayName("Null/empty providerId is a no-op (defensive)") + void nullEmptySafe() { + pool.add(null); + pool.add(""); + pool.remove(null, RemovalSource.AUTH_ERROR, "x"); + pool.remove("", RemovalSource.AUTH_ERROR, "x"); + assertFalse(pool.contains(null)); + assertFalse(pool.contains("")); + assertTrue(pool.snapshot().isEmpty(), + "null/empty inputs must not pollute the snapshot"); + } + + @Test + @DisplayName("Concurrent add + remove + contains is thread-safe") + void concurrentAccess() throws Exception { + int threads = 16; + int opsPerThread = 5_000; + ExecutorService pool2 = Executors.newFixedThreadPool(threads); + CountDownLatch start = new CountDownLatch(1); + CountDownLatch done = new CountDownLatch(threads); + + for (int t = 0; t < threads; t++) { + int worker = t; + pool2.submit(() -> { + try { + start.await(); + for (int i = 0; i < opsPerThread; i++) { + String id = "p" + (worker * 10 + (i % 10)); // shared id space + if (i % 3 == 0) pool.add(id); + else if (i % 3 == 1) pool.remove(id, RemovalSource.AUTH_ERROR, "race"); + else pool.contains(id); + } + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } finally { + done.countDown(); + } + }); + } + start.countDown(); + assertTrue(done.await(30, TimeUnit.SECONDS), "concurrent workload must complete in 30s"); + pool2.shutdown(); + + // Internal state must remain consistent — each id is either in members OR has a removal reason + // (or both — the union is also fine), and snapshot doesn't NPE. + var snap = pool.snapshot(); + assertNotNull(snap); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java new file mode 100644 index 00000000..97d3b478 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderHealthTrackerTest.java @@ -0,0 +1,117 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-009 P3.3: per-provider failure-count + cooldown logic. + */ +class ProviderHealthTrackerTest { + + private ProviderHealthProperties props; + private ProviderHealthTracker tracker; + + @BeforeEach + void setUp() { + props = new ProviderHealthProperties(); + props.setFailureThreshold(3); + props.setCooldownMs(60_000L); + tracker = new ProviderHealthTracker(props); + } + + @Test + @DisplayName("New provider is not in cooldown") + void newProviderNotInCooldown() { + assertFalse(tracker.isInCooldown("openai")); + } + + @Test + @DisplayName("Failures below threshold do not trigger cooldown") + void belowThresholdNoCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertFalse(tracker.isInCooldown("openai"), + "two failures < threshold of 3 must not enter cooldown"); + } + + @Test + @DisplayName("Failures hitting threshold enter cooldown") + void thresholdReachedTriggersCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai"), + "third failure must enter cooldown"); + } + + @Test + @DisplayName("Success resets failure counter and clears cooldown") + void successResetsCounterAndCooldown() { + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai")); + + tracker.recordSuccess("openai"); + assertFalse(tracker.isInCooldown("openai"), + "success must clear cooldown so the provider becomes eligible again"); + } + + @Test + @DisplayName("After cooldown expires, provider becomes eligible again") + void cooldownExpires() throws Exception { + // Bypass the min-1000ms clamp in setCooldownMs via reflection — the + // clamp is there to prevent prod misconfiguration, but for this test + // we want a fast-expiring window to avoid sleeping 1+ seconds. + java.lang.reflect.Field f = ProviderHealthProperties.class.getDeclaredField("cooldownMs"); + f.setAccessible(true); + f.setLong(props, 50L); + + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai"), "sanity: still in cooldown right after trigger"); + Thread.sleep(120); + assertFalse(tracker.isInCooldown("openai"), + "cooldown should expire once the window has passed"); + } + + @Test + @DisplayName("Disabled tracker never reports cooldown") + void disabledTrackerInert() { + props.setEnabled(false); + for (int i = 0; i < 10; i++) tracker.recordFailure("openai"); + assertFalse(tracker.isInCooldown("openai"), + "disabled tracker must report no cooldown regardless of failures"); + } + + @Test + @DisplayName("Null providerId is a safe no-op") + void nullProviderIdSafe() { + tracker.recordFailure(null); + tracker.recordSuccess(null); + assertFalse(tracker.isInCooldown(null), + "null providerId must not crash and must report no cooldown"); + } + + @Test + @DisplayName("Per-provider isolation: cooldown on A does not affect B") + void perProviderIsolation() { + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + assertTrue(tracker.isInCooldown("openai")); + assertFalse(tracker.isInCooldown("dashscope"), + "cooldown must be scoped per provider id"); + } + + @Test + @DisplayName("Snapshot reports both failure count and remaining cooldown") + void snapshotReportsState() { + for (int i = 0; i < 3; i++) tracker.recordFailure("openai"); + var snap = tracker.snapshot(); + assertNotNull(snap.get("openai")); + assertEquals(3L, snap.get("openai").consecutiveFailures()); + assertTrue(snap.get("openai").cooldownRemainingMs() > 0, + "cooldown remaining ms must be positive while active"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java new file mode 100644 index 00000000..841994ce --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderInitProbeTest.java @@ -0,0 +1,264 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.model.ModelProtocol; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.repository.ModelProviderMapper; +import vip.mate.llm.service.ModelProviderService; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Verifies the startup-probe orchestration: + *
    + *
  • Healthy probes → provider added to pool.
  • + *
  • Failed probes → provider removed with INIT_PROBE source.
  • + *
  • Slow probe → fail-open (in-pool) so chat isn't gated by a stalled probe.
  • + *
  • Missing strategy → fail-open (in-pool).
  • + *
  • {@code probeOne} updates pool state on demand.
  • + *
  • Duplicate strategies for the same protocol fail-fast at construction.
  • + *
+ * + *

Strategies are real test-double instances (not Mockito mocks) so we can + * inject latency or throw cheaply; the mapper / service collaborators are + * stock Mockito mocks because they're MyBatis-Plus / Spring beans.

+ */ +class ProviderInitProbeTest { + + private ModelProviderMapper mapper; + private ModelProviderService providerService; + private AvailableProviderPool pool; + + @BeforeEach + void setUp() { + mapper = mock(ModelProviderMapper.class); + providerService = mock(ModelProviderService.class); + pool = new AvailableProviderPool(); + } + + @Test + @DisplayName("All strategies pass: every configured provider lands in the pool") + void allHealthy() { + ModelProviderEntity openai = provider("openai", ModelProtocol.OPENAI_COMPATIBLE); + ModelProviderEntity anthropic = provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES); + ModelProviderEntity dashscope = provider("dashscope", ModelProtocol.DASHSCOPE_NATIVE); + configure(List.of(openai, anthropic, dashscope), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(10)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> ProbeResult.ok(20)), + stub(ModelProtocol.DASHSCOPE_NATIVE, p -> ProbeResult.ok(30)))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertTrue(pool.contains("anthropic")); + assertTrue(pool.contains("dashscope")); + } + + @Test + @DisplayName("Failed probe removes provider with INIT_PROBE source and the error message") + void failurePathRemovesWithReason() { + configure(List.of(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.fail(50, "401 Unauthorized")))); + probe.probeAllConfigured(); + + assertFalse(pool.contains("openai")); + var reason = pool.snapshot().get("openai"); + assertNotNull(reason); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, reason.source()); + assertTrue(reason.message().contains("401 Unauthorized"), + "removal message must surface the underlying probe error"); + } + + @Test + @DisplayName("Mixed batch: pass + fail in one run leaves correct pool state") + void mixedBatch() { + configure(List.of( + provider("openai", ModelProtocol.OPENAI_COMPATIBLE), + provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(10)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> ProbeResult.fail(15, "auth")))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, + pool.snapshot().get("anthropic").source()); + } + + @Test + @DisplayName("Strategy throwing is treated as a probe failure (no startup crash)") + void strategyThrowsHandledAsFailure() { + configure(List.of(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)), id -> true); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> { + throw new RuntimeException("network down"); + }))); + probe.probeAllConfigured(); + + assertFalse(pool.contains("openai"), + "a throwing strategy must not leave the provider falsely in-pool"); + assertNotNull(pool.snapshot().get("openai")); + } + + @Test + @DisplayName("No strategy registered for protocol: fail-open (provider stays in pool)") + void missingStrategyFailsOpen() { + configure(List.of(provider("gemini", ModelProtocol.GEMINI_NATIVE)), id -> true); + + // Empty strategy list — no GEMINI_NATIVE handler. + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + probe.probeAllConfigured(); + + assertTrue(pool.contains("gemini"), + "without a probe strategy we must default to in-pool, not block chat"); + } + + @Test + @DisplayName("No configured providers: probe is a no-op, pool stays empty") + void emptyConfigurationIsNoOp() { + configure(List.of(), id -> false); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)))); + probe.probeAllConfigured(); + + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("Unconfigured providers are skipped (not probed and not added)") + void unconfiguredSkipped() { + ModelProviderEntity openai = provider("openai", ModelProtocol.OPENAI_COMPATIBLE); + ModelProviderEntity anthropic = provider("anthropic", ModelProtocol.ANTHROPIC_MESSAGES); + // mapper returns both, but only openai is "configured" + when(mapper.selectList(any())).thenReturn(List.of(openai, anthropic)); + when(providerService.isProviderConfigured("openai")).thenReturn(true); + when(providerService.isProviderConfigured("anthropic")).thenReturn(false); + + AtomicInteger anthropicCalls = new AtomicInteger(); + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)), + stub(ModelProtocol.ANTHROPIC_MESSAGES, p -> { + anthropicCalls.incrementAndGet(); + return ProbeResult.ok(0); + }))); + probe.probeAllConfigured(); + + assertTrue(pool.contains("openai")); + assertFalse(pool.contains("anthropic")); + assertEquals(0, anthropicCalls.get(), + "unconfigured providers must not even be probed"); + } + + @Test + @DisplayName("probeOne(unknown) returns failure and does not pollute pool") + void probeOneUnknownProvider() { + when(mapper.selectById(anyString())).thenReturn(null); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + ProbeResult r = probe.probeOne("ghost"); + + assertFalse(r.success()); + assertTrue(pool.snapshot().isEmpty()); + } + + @Test + @DisplayName("probeOne(unconfigured) HARD-removes from pool") + void probeOneUnconfigured() { + when(mapper.selectById("openai")).thenReturn(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)); + when(providerService.isProviderConfigured("openai")).thenReturn(false); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of()); + ProbeResult r = probe.probeOne("openai"); + + assertFalse(r.success()); + assertFalse(pool.contains("openai")); + assertEquals(AvailableProviderPool.RemovalSource.INIT_PROBE, + pool.snapshot().get("openai").source()); + } + + @Test + @DisplayName("probeOne(healthy) re-adds previously-removed provider to pool") + void probeOneRecoversRemovedProvider() { + when(mapper.selectById("openai")).thenReturn(provider("openai", ModelProtocol.OPENAI_COMPATIBLE)); + when(providerService.isProviderConfigured("openai")).thenReturn(true); + + // Pre-remove openai to simulate a HARD-error eviction. + pool.remove("openai", AvailableProviderPool.RemovalSource.AUTH_ERROR, "401"); + assertFalse(pool.contains("openai")); + + ProviderInitProbe probe = new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(5)))); + ProbeResult r = probe.probeOne("openai"); + + assertTrue(r.success()); + assertTrue(pool.contains("openai"), + "a successful reprobe must rehabilitate a previously removed provider"); + } + + @Test + @DisplayName("Duplicate strategy for same protocol fails-fast at construction") + void duplicateStrategyRejected() { + IllegalStateException ex = assertThrows(IllegalStateException.class, () -> + new ProviderInitProbe(mapper, providerService, pool, List.of( + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0)), + stub(ModelProtocol.OPENAI_COMPATIBLE, p -> ProbeResult.ok(0))))); + assertTrue(ex.getMessage().contains("OPENAI_COMPATIBLE")); + } + + // ============================================================ + // Helpers + // ============================================================ + + private static ModelProviderEntity provider(String id, ModelProtocol protocol) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setChatModel(protocol.getChatModelClass()); + p.setApiKey("sk-test"); + p.setBaseUrl("https://example.com"); + // RFC-074: probe filters out enabled=false rows. The pre-RFC-074 default + // for these test fixtures was "everything participates" — preserve that. + p.setEnabled(true); + return p; + } + + /** Wires the mapper and service so {@code listConfiguredProviders()} returns the given list, + * filtered through {@code configuredPredicate}. */ + private void configure(List all, Function configuredPredicate) { + when(mapper.selectList(any())).thenReturn(all); + Map map = new HashMap<>(); + for (ModelProviderEntity p : all) { + map.put(p.getProviderId(), configuredPredicate.apply(p.getProviderId())); + } + when(providerService.isProviderConfigured(anyString())) + .thenAnswer(inv -> map.getOrDefault(inv.getArgument(0), false)); + } + + /** Lambda-driven fake of {@link ProviderProbeStrategy}. */ + private static ProviderProbeStrategy stub(ModelProtocol protocol, + Function body) { + return new ProviderProbeStrategy() { + @Override public ModelProtocol supportedProtocol() { return protocol; } + @Override public ProbeResult probe(ModelProviderEntity provider) { return body.apply(provider); } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderRequirementsTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderRequirementsTest.java new file mode 100644 index 00000000..b51699fb --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/ProviderRequirementsTest.java @@ -0,0 +1,179 @@ +package vip.mate.llm.failover; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.model.ModelProviderEntity; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Issue #81: row-based required-fields decision. Replaces the v1 protocol-keyed + * lookup, which couldn't tell OpenAI cloud (needs api_key) apart from llama.cpp + * local (needs base_url) because both ride the OPENAI_COMPATIBLE protocol enum. + * + *

Each test is one cell of the truth table in the RFC §2.2 / §2.3. + */ +class ProviderRequirementsTest { + + @Test + @DisplayName("OpenAI cloud: needs api key, no base url, no hint") + void openaiCloud() { + ProviderRequirements.Required r = ProviderRequirements.of(cloud("openai", true)); + assertTrue(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + assertNull(r.hintKey()); + } + + @Test + @DisplayName("Kimi cloud: same shape as OpenAI") + void kimiCloud() { + ProviderRequirements.Required r = ProviderRequirements.of(cloud("kimi", true)); + assertTrue(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + } + + @Test + @DisplayName("DeepSeek cloud: same shape") + void deepseekCloud() { + ProviderRequirements.Required r = ProviderRequirements.of(cloud("deepseek", true)); + assertTrue(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + } + + @Test + @DisplayName("llama.cpp local: no api key, needs base url, llamacpp hint") + void llamacppLocal() { + ProviderRequirements.Required r = ProviderRequirements.of(local("llamacpp")); + assertFalse(r.needsApiKey()); + assertTrue(r.needsBaseUrl()); + assertEquals("provider.hint.llamacppBaseUrlExample", r.hintKey()); + assertEquals("http://127.0.0.1:8080/v1", r.hintArgs().get("example")); + } + + @Test + @DisplayName("Ollama local: ollama-specific hint") + void ollamaLocal() { + ProviderRequirements.Required r = ProviderRequirements.of(local("ollama")); + assertFalse(r.needsApiKey()); + assertTrue(r.needsBaseUrl()); + assertEquals("provider.hint.ollamaBaseUrlExample", r.hintKey()); + assertEquals("http://127.0.0.1:11434", r.hintArgs().get("example")); + } + + @Test + @DisplayName("LM Studio local: lmstudio-specific hint, also matches lm-studio / lm_studio") + void lmstudioLocal() { + ProviderRequirements.Required r = ProviderRequirements.of(local("lmstudio")); + assertEquals("provider.hint.lmstudioBaseUrlExample", r.hintKey()); + assertEquals("provider.hint.lmstudioBaseUrlExample", + ProviderRequirements.of(local("lm-studio")).hintKey()); + assertEquals("provider.hint.lmstudioBaseUrlExample", + ProviderRequirements.of(local("lm_studio")).hintKey()); + } + + @Test + @DisplayName("vLLM local: vllm-specific hint") + void vllmLocal() { + ProviderRequirements.Required r = ProviderRequirements.of(local("vllm")); + assertEquals("provider.hint.vllmBaseUrlExample", r.hintKey()); + assertEquals("http://127.0.0.1:8000/v1", r.hintArgs().get("example")); + } + + @Test + @DisplayName("Custom OpenAI-compat needing API key: needs both, generic hint") + void customOpenAiCompatNeedingKey() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("my-llm-server"); + p.setIsCustom(true); + p.setIsLocal(false); + p.setRequireApiKey(true); + + ProviderRequirements.Required r = ProviderRequirements.of(p); + assertTrue(r.needsApiKey()); + assertTrue(r.needsBaseUrl()); + assertEquals("provider.hint.openaiCompatBaseUrlExample", r.hintKey()); + } + + @Test + @DisplayName("Custom OpenAI-compat without API key: only base url + generic hint") + void customOpenAiCompatNoKey() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("my-llm-server"); + p.setIsCustom(true); + p.setRequireApiKey(false); + + ProviderRequirements.Required r = ProviderRequirements.of(p); + assertFalse(r.needsApiKey()); + assertTrue(r.needsBaseUrl()); + assertEquals("provider.hint.openaiCompatBaseUrlExample", r.hintKey()); + } + + @Test + @DisplayName("OAuth provider: no api key, no base url, no hint") + void oauthProvider() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("anthropic-claude-code"); + p.setAuthType("oauth"); + p.setRequireApiKey(true); // ignored under oauth + p.setIsLocal(true); // ignored under oauth + + ProviderRequirements.Required r = ProviderRequirements.of(p); + assertFalse(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + assertNull(r.hintKey()); + } + + @Test + @DisplayName("Generic OAuth (non-Claude-Code): same shape") + void genericOauth() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("some-oauth-provider"); + p.setAuthType("oauth"); + + ProviderRequirements.Required r = ProviderRequirements.of(p); + assertFalse(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + } + + @Test + @DisplayName("Null provider: safe defaults") + void nullProvider() { + ProviderRequirements.Required r = ProviderRequirements.of(null); + assertFalse(r.needsApiKey()); + assertFalse(r.needsBaseUrl()); + assertNull(r.hintKey()); + assertNotNull(r.hintArgs()); + } + + @Test + @DisplayName("isCustom=true with empty providerId: still needs base url, generic hint") + void customEmptyProviderId() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setIsCustom(true); + p.setRequireApiKey(false); + + ProviderRequirements.Required r = ProviderRequirements.of(p); + assertTrue(r.needsBaseUrl()); + assertEquals("provider.hint.openaiCompatBaseUrlExample", r.hintKey()); + } + + // ===== helpers ===== + + private static ModelProviderEntity cloud(String id, boolean requireApiKey) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setIsLocal(false); + p.setIsCustom(false); + p.setRequireApiKey(requireApiKey); + return p; + } + + private static ModelProviderEntity local(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setIsLocal(true); + p.setIsCustom(false); + p.setRequireApiKey(false); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbeTest.java b/mateclaw-server/src/test/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbeTest.java new file mode 100644 index 00000000..4991a38e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/failover/probe/OpenAiCompatibleListModelsProbeTest.java @@ -0,0 +1,69 @@ +package vip.mate.llm.failover.probe; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Locks down the URL-resolution rule for {@link OpenAiCompatibleListModelsProbe}: + * + *

    + *
  • Vendors that point at the API root (OpenAI / Kimi / DeepSeek) get + * {@code /v1/models} appended.
  • + *
  • Vendors that include a {@code /vN} segment in their Base URL + * (LMStudio's {@code /v1}, ZhipuAI's {@code /v4}, etc.) get only + * {@code /models} appended — preventing the {@code /v1/v1/models} or + * {@code /v4/v1/models} 404s the original implementation produced.
  • + *
+ */ +class OpenAiCompatibleListModelsProbeTest { + + @Test + @DisplayName("API-root base URL → append /v1/models (OpenAI / DeepSeek / Kimi)") + void apiRootBaseGetsV1Models() { + assertEquals("/v1/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://api.openai.com")); + assertEquals("/v1/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://api.deepseek.com")); + assertEquals("/v1/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://api.moonshot.cn")); + } + + @Test + @DisplayName("Base URL ends in /v1 → append only /models (LMStudio)") + void v1SuffixGetsOnlyModels() { + assertEquals("/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("http://localhost:1234/v1")); + } + + @Test + @DisplayName("Base URL ends in /v4 → append only /models (ZhipuAI)") + void v4SuffixGetsOnlyModels() { + assertEquals("/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://open.bigmodel.cn/api/paas/v4")); + } + + @Test + @DisplayName("Base URL ends in /v2 (hypothetical) → append only /models") + void otherVersionSuffixGetsOnlyModels() { + assertEquals("/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://example.com/api/v2")); + assertEquals("/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://example.com/v3")); + } + + @Test + @DisplayName("Base URL contains /vN mid-path but doesn't end with it → append /v1/models") + void midPathVersionDoesNotMatch() { + assertEquals("/v1/models", + OpenAiCompatibleListModelsProbe.resolveModelsPath("https://api.example.com/v1/proxy")); + } + + @Test + @DisplayName("Edge: null / blank base URL falls back to /v1/models (caller validates emptiness separately)") + void nullOrBlankBase() { + assertEquals("/v1/models", OpenAiCompatibleListModelsProbe.resolveModelsPath(null)); + assertEquals("/v1/models", OpenAiCompatibleListModelsProbe.resolveModelsPath("")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/model/ModelFamilyTest.java b/mateclaw-server/src/test/java/vip/mate/llm/model/ModelFamilyTest.java new file mode 100644 index 00000000..707c0175 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/model/ModelFamilyTest.java @@ -0,0 +1,68 @@ +package vip.mate.llm.model; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pinpoint regression tests for {@link ModelFamily#detect(String)}. + * + *

Each new model family added here should pin its detect rule so accidental + * code-style cleanups (e.g. reordering branches in {@code detect()}) can't + * silently route a thinking model to {@link ModelFamily#STANDARD} and break + * reasoning_effort propagation. + */ +class ModelFamilyTest { + + @Test + @DisplayName("DeepSeek V4 (flash + pro) → DEEPSEEK_V4_REASONING with reasoning_effort enabled") + void deepSeekV4_reasoning() { + // Critical assertion: V4 differs from v3.2 deepseek-reasoner — V4 ACCEPTS + // the reasoning_effort field, while v3.2 doesn't (DeepSeek API rejects it). + // Routing V4 to DEEPSEEK_REASONER would suppress the field and forfeit + // openclaw's documented thinking control. + assertEquals(ModelFamily.DEEPSEEK_V4_REASONING, ModelFamily.detect("deepseek-v4-flash")); + assertEquals(ModelFamily.DEEPSEEK_V4_REASONING, ModelFamily.detect("deepseek-v4-pro")); + assertTrue(ModelFamily.DEEPSEEK_V4_REASONING.supportsReasoningEffort(), + "V4 must accept reasoning_effort (key differentiator from v3.2 reasoner)"); + assertTrue(ModelFamily.DEEPSEEK_V4_REASONING.isThinking(), + "V4 is a thinking family — DeepSeekV4ThinkingDecorator gates on this"); + assertFalse(ModelFamily.DEEPSEEK_V4_REASONING.fixedTemperatureOne(), + "V4 allows configurable temperature (unlike v3.2 reasoner)"); + } + + @Test + @DisplayName("Legacy deepseek-reasoner stays in DEEPSEEK_REASONER family (does not catch V4 rule)") + void deepSeekReasoner_unchanged() { + // Defensive: if the V4 detect rule were too broad (e.g. startsWith "deepseek-") + // it would catch deepseek-reasoner too and break that model's working config. + assertEquals(ModelFamily.DEEPSEEK_REASONER, ModelFamily.detect("deepseek-reasoner")); + assertFalse(ModelFamily.DEEPSEEK_REASONER.supportsReasoningEffort(), + "v3.2 reasoner must NOT advertise reasoning_effort support"); + } + + @Test + @DisplayName("deepseek-chat stays STANDARD") + void deepSeekChat_standard() { + // Smoke check: non-reasoning DeepSeek model unaffected. + assertEquals(ModelFamily.STANDARD, ModelFamily.detect("deepseek-chat")); + } + + @Test + @DisplayName("Case + whitespace tolerance — uppercased / padded model name routes the same") + void detect_caseInsensitive() { + assertEquals(ModelFamily.DEEPSEEK_V4_REASONING, ModelFamily.detect("DeepSeek-V4-Flash")); + assertEquals(ModelFamily.DEEPSEEK_V4_REASONING, ModelFamily.detect(" deepseek-v4-pro ")); + } + + @Test + @DisplayName("Null / blank model name → STANDARD (no NPE)") + void detect_nullSafe() { + assertEquals(ModelFamily.STANDARD, ModelFamily.detect(null)); + assertEquals(ModelFamily.STANDARD, ModelFamily.detect("")); + assertEquals(ModelFamily.STANDARD, ModelFamily.detect(" ")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIDeviceCodeServiceTest.java b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIDeviceCodeServiceTest.java new file mode 100644 index 00000000..fab211a9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIDeviceCodeServiceTest.java @@ -0,0 +1,325 @@ +package vip.mate.llm.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; +import vip.mate.exception.MateClawException; +import vip.mate.llm.oauth.OpenAIDeviceCodeService.DeviceCodePollResult; +import vip.mate.llm.oauth.OpenAIDeviceCodeService.DeviceCodeStartResult; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.content; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.jsonPath; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +/** + * Unit tests for the device authorization grant flow. + * + *

{@link OpenAIDeviceCodeService} is exercised through a mocked OpenAI endpoint + * (via {@link MockRestServiceServer}). The token exchange path + * ({@code OpenAIOAuthService#exchangeTokenWithVerifier}) is mocked so we never + * touch the database — we only verify it is invoked with the correct args. + */ +class OpenAIDeviceCodeServiceTest { + + private OpenAIOAuthService oauthService; + private OpenAIDeviceCodeService deviceCodeService; + private MockRestServiceServer mockServer; + + @BeforeEach + void setUp() throws Exception { + oauthService = mock(OpenAIOAuthService.class); + deviceCodeService = new OpenAIDeviceCodeService(oauthService, new ObjectMapper()); + + // Tighten config knobs so tests don't sleep + setField(deviceCodeService, "pollMinIntervalMs", 0L); + setField(deviceCodeService, "defaultSessionTtlSeconds", 900L); + setField(deviceCodeService, "userAgent", "test-agent/0.0"); + + RestClient.Builder builder = RestClient.builder(); + mockServer = MockRestServiceServer.bindTo(builder).build(); + deviceCodeService.setRestClient(builder.build()); + } + + private static void setField(Object target, String name, Object value) throws Exception { + Field f = OpenAIDeviceCodeService.class.getDeclaredField(name); + f.setAccessible(true); + f.set(target, value); + } + + // --------------------------------------------------------------------- + // start() + // --------------------------------------------------------------------- + + @Test + @DisplayName("start sends JSON body with client_id and parses all response fields") + void start_parsesAllFields() { + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_USERCODE_URL)) + .andExpect(header(org.springframework.http.HttpHeaders.CONTENT_TYPE, + MediaType.APPLICATION_JSON_VALUE)) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.client_id").value(OpenAIDeviceCodeService.CLIENT_ID)) + .andRespond(withSuccess( + "{\"device_auth_id\":\"dev-abc-123\"," + + "\"user_code\":\"WXYZ-1234\"," + + "\"interval\":7," + + "\"expires_in\":600," + + "\"verification_uri\":\"https://auth.openai.com/codex/device\"," + + "\"verification_uri_complete\":\"https://auth.openai.com/codex/device?user_code=WXYZ-1234\"}", + MediaType.APPLICATION_JSON)); + + DeviceCodeStartResult result = deviceCodeService.start(); + + assertEquals("dev-abc-123", result.deviceAuthId()); + assertEquals("WXYZ-1234", result.userCode()); + assertEquals(7, result.intervalSeconds()); + assertEquals(600, result.expiresInSeconds()); + assertEquals("https://auth.openai.com/codex/device", result.verificationUrl()); + assertEquals("https://auth.openai.com/codex/device?user_code=WXYZ-1234", + result.verificationUrlComplete()); + assertEquals(1, deviceCodeService.activeSessionCount()); + + mockServer.verify(); + } + + @Test + @DisplayName("start defaults verification URL when not returned by OpenAI") + void start_defaultsVerificationUrl() { + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_USERCODE_URL)) + .andRespond(withSuccess( + "{\"device_auth_id\":\"d1\",\"user_code\":\"AB-CD\"," + + "\"interval\":5,\"expires_in\":300}", + MediaType.APPLICATION_JSON)); + + DeviceCodeStartResult result = deviceCodeService.start(); + assertEquals(OpenAIDeviceCodeService.DEFAULT_VERIFICATION_URL, result.verificationUrl()); + } + + @Test + @DisplayName("start throws MateClawException on transport failure") + void start_propagatesTransportFailures() { + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_USERCODE_URL)) + .andRespond(withStatus(HttpStatus.SERVICE_UNAVAILABLE)); + + MateClawException ex = assertThrows(MateClawException.class, + () -> deviceCodeService.start()); + assertEquals("err.llm.device_code_start_failed", ex.getMsgKey()); + } + + @Test + @DisplayName("start throws when response is missing required fields") + void start_rejectsIncompleteResponse() { + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_USERCODE_URL)) + .andRespond(withSuccess("{\"interval\":5}", MediaType.APPLICATION_JSON)); + + MateClawException ex = assertThrows(MateClawException.class, + () -> deviceCodeService.start()); + assertEquals("err.llm.device_code_start_failed", ex.getMsgKey()); + } + + // --------------------------------------------------------------------- + // poll() + // --------------------------------------------------------------------- + + @Test + @DisplayName("poll returns EXPIRED for unknown session") + void poll_unknownSessionExpired() { + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("not-a-real-session").status()); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll(null).status()); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("").status()); + } + + @Test + @DisplayName("poll sends JSON body and returns PENDING for HTTP 403 (user has not finished yet)") + void poll_403MapsToPending() { + expectStart("dev-1", "USER-1"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andExpect(content().contentType(MediaType.APPLICATION_JSON)) + .andExpect(jsonPath("$.device_auth_id").value("dev-1")) + .andExpect(jsonPath("$.user_code").value("USER-1")) + .andRespond(withStatus(HttpStatus.FORBIDDEN)); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.PENDING, + deviceCodeService.poll("dev-1").status()); + verifyNoInteractions(oauthService); + } + + @Test + @DisplayName("poll returns PENDING for HTTP 404 (per OpenAI deviceauth contract)") + void poll_404MapsToPending() { + expectStart("dev-1b", "USER-1B"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withStatus(HttpStatus.NOT_FOUND)); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.PENDING, + deviceCodeService.poll("dev-1b").status()); + } + + @Test + @DisplayName("poll still maps RFC 8628 400+authorization_pending to PENDING for forward-compat") + void poll_rfcAuthorizationPendingMapsToPending() { + expectStart("dev-1c", "USER-1C"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withStatus(HttpStatus.BAD_REQUEST) + .contentType(MediaType.APPLICATION_JSON) + .body("{\"error\":\"authorization_pending\"}")); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.PENDING, + deviceCodeService.poll("dev-1c").status()); + verifyNoInteractions(oauthService); + } + + @Test + @DisplayName("poll returns PENDING when OpenAI replies 400 slow_down") + void poll_slowDownMapsToPending() { + expectStart("dev-2", "USER-2"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withStatus(HttpStatus.BAD_REQUEST) + .contentType(MediaType.APPLICATION_JSON) + .body("{\"error\":\"slow_down\"}")); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.PENDING, + deviceCodeService.poll("dev-2").status()); + } + + @Test + @DisplayName("poll returns EXPIRED + drops session when OpenAI replies 400 expired_token") + void poll_expiredTokenDropsSession() { + expectStart("dev-3", "USER-3"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withStatus(HttpStatus.BAD_REQUEST) + .contentType(MediaType.APPLICATION_JSON) + .body("{\"error\":\"expired_token\"}")); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("dev-3").status()); + // session was removed — next poll returns EXPIRED without hitting the network + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("dev-3").status()); + } + + @Test + @DisplayName("poll returns EXPIRED when user denies access") + void poll_accessDeniedDropsSession() { + expectStart("dev-4", "USER-4"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withStatus(HttpStatus.BAD_REQUEST) + .contentType(MediaType.APPLICATION_JSON) + .body("{\"error\":\"access_denied\"}")); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("dev-4").status()); + } + + @Test + @DisplayName("poll returns COMPLETED + invokes token exchange when authorization_code arrives") + void poll_completedExchangesToken() { + expectStart("dev-5", "USER-5"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withSuccess( + "{\"authorization_code\":\"auth-code-xyz\"," + + "\"code_verifier\":\"verifier-xyz\"}", + MediaType.APPLICATION_JSON)); + + deviceCodeService.start(); + DeviceCodePollResult result = deviceCodeService.poll("dev-5"); + + assertEquals(DeviceCodePollResult.Status.COMPLETED, result.status()); + verify(oauthService).exchangeTokenWithVerifier( + eq("auth-code-xyz"), + eq("verifier-xyz"), + eq(OpenAIDeviceCodeService.DEVICE_REDIRECT_URI)); + assertEquals(0, deviceCodeService.activeSessionCount()); + } + + @Test + @DisplayName("poll keeps session and returns PENDING when 200 body has no authorization_code") + void poll_inlinePendingErrorMapsToPending() { + expectStart("dev-6", "USER-6"); + // Some flavours of the endpoint reply 200 with {error: authorization_pending} + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withSuccess( + "{\"error\":\"authorization_pending\"}", + MediaType.APPLICATION_JSON)); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.PENDING, + deviceCodeService.poll("dev-6").status()); + assertEquals(1, deviceCodeService.activeSessionCount()); + } + + @Test + @DisplayName("poll returns EXPIRED when authorization_code present but code_verifier missing") + void poll_missingCodeVerifierDropsSession() { + expectStart("dev-7", "USER-7"); + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_TOKEN_URL)) + .andRespond(withSuccess( + "{\"authorization_code\":\"only-code\"}", + MediaType.APPLICATION_JSON)); + + deviceCodeService.start(); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("dev-7").status()); + verifyNoInteractions(oauthService); + } + + // --------------------------------------------------------------------- + // cancel() + // --------------------------------------------------------------------- + + @Test + @DisplayName("cancel removes the session so subsequent poll returns EXPIRED") + void cancel_dropsSession() { + expectStart("dev-cancel", "USER-CANCEL"); + + deviceCodeService.start(); + assertEquals(1, deviceCodeService.activeSessionCount()); + + deviceCodeService.cancel("dev-cancel"); + assertEquals(0, deviceCodeService.activeSessionCount()); + assertEquals(DeviceCodePollResult.Status.EXPIRED, + deviceCodeService.poll("dev-cancel").status()); + } + + @Test + @DisplayName("cancel handles null/missing IDs without throwing") + void cancel_nullSafe() { + assertDoesNotThrow(() -> deviceCodeService.cancel(null)); + assertDoesNotThrow(() -> deviceCodeService.cancel("never-existed")); + } + + // --------------------------------------------------------------------- + // helpers + // --------------------------------------------------------------------- + + /** Register the usercode-endpoint expectation; caller must invoke start() afterwards. */ + private void expectStart(String deviceAuthId, String userCode) { + mockServer.expect(requestTo(OpenAIDeviceCodeService.DEVICE_USERCODE_URL)) + .andRespond(withSuccess( + "{\"device_auth_id\":\"" + deviceAuthId + "\"," + + "\"user_code\":\"" + userCode + "\"," + + "\"interval\":5,\"expires_in\":900}", + MediaType.APPLICATION_JSON)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceFlowModeTest.java b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceFlowModeTest.java new file mode 100644 index 00000000..ab06b532 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceFlowModeTest.java @@ -0,0 +1,180 @@ +package vip.mate.llm.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.exception.MateClawException; +import vip.mate.llm.oauth.OpenAIOAuthService.OAuthFlowMode; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Issue: OAuth callback fails on Linux server deployment because the + * redirect_uri is hardcoded to http://localhost:1455/auth/callback. When the + * user's browser hits this URL it tries to reach the user's own machine, not + * the remote MateClaw server, so the auth code never reaches the server. + * + *

Tests focus on the deployment-mode resolution logic (Host header heuristic + * + config override + paste-URL parser). Network-bound paths (token exchange, + * Keychain reads) are out of scope here — they need either a wiremock or live + * fixtures. + */ +class OpenAIOAuthServiceFlowModeTest { + + private OpenAIOAuthService service; + + @BeforeEach + void setUp() { + // null collaborators OK because the helpers we exercise (resolveFlowMode, + // completeFromPastedUrl up to state validation) don't touch them. The + // compile-time @RequiredArgsConstructor accepts nulls. + service = new OpenAIOAuthService(null, new ObjectMapper(), null); + } + + @AfterEach + void clearOverride() { + System.clearProperty("mateclaw.oauth.openai.deployment-mode"); + } + + // ============== resolveFlowMode (private — accessed via reflection) === + + private OAuthFlowMode invokeResolve(String host) throws Exception { + Method m = OpenAIOAuthService.class.getDeclaredMethod("resolveFlowMode", String.class); + m.setAccessible(true); + return (OAuthFlowMode) m.invoke(service, host); + } + + @Test + @DisplayName("localhost variants resolve to LOCAL mode") + void localhostHosts_resolveToLocal() throws Exception { + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("localhost")); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("localhost:18088")); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("127.0.0.1")); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("127.0.0.1:18088")); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("LocalHost")); // case-insensitive + } + + @Test + @DisplayName("public hosts resolve to DEVICE_CODE (browser-agnostic, no callback server needed)") + void publicHosts_resolveToDeviceCode() throws Exception { + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("mateclaw.example.com")); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("api.mate.vip")); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("api.mate.vip:443")); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("192.168.1.10"), + "private LAN IP — not localhost, browser still won't reach server's localhost"); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("10.0.0.5:8080")); + } + + @Test + @DisplayName("null/blank host falls back to LOCAL (legacy behaviour preservation)") + void nullOrBlankHost_legacyLocal() throws Exception { + assertEquals(OAuthFlowMode.LOCAL, invokeResolve(null)); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("")); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve(" ")); + } + + @Test + @DisplayName("config override mateclaw.oauth.openai.deployment-mode=local forces LOCAL even on remote host") + void configOverride_forcesLocal() throws Exception { + System.setProperty("mateclaw.oauth.openai.deployment-mode", "local"); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("mateclaw.example.com")); + } + + @Test + @DisplayName("config override =device_code forces DEVICE_CODE even on localhost") + void configOverride_forcesDeviceCode() throws Exception { + System.setProperty("mateclaw.oauth.openai.deployment-mode", "device_code"); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("localhost")); + + // 'server' kept as alias for backwards compatibility (now points to DEVICE_CODE) + System.setProperty("mateclaw.oauth.openai.deployment-mode", "server"); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("localhost")); + } + + @Test + @DisplayName("config override =manual_paste forces MANUAL_PASTE") + void configOverride_forcesManualPaste() throws Exception { + System.setProperty("mateclaw.oauth.openai.deployment-mode", "manual_paste"); + assertEquals(OAuthFlowMode.MANUAL_PASTE, invokeResolve("localhost")); + assertEquals(OAuthFlowMode.MANUAL_PASTE, invokeResolve("api.mate.vip")); + } + + @Test + @DisplayName("config override 'auto' or unknown falls back to heuristic") + void configOverride_autoFallsThrough() throws Exception { + System.setProperty("mateclaw.oauth.openai.deployment-mode", "auto"); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("localhost")); + assertEquals(OAuthFlowMode.DEVICE_CODE, invokeResolve("api.mate.vip")); + + System.setProperty("mateclaw.oauth.openai.deployment-mode", "garbage"); + assertEquals(OAuthFlowMode.LOCAL, invokeResolve("localhost")); + } + + // ============== completeFromPastedUrl ================================ + + @Test + @DisplayName("completeFromPastedUrl rejects empty / null input") + void pastedUrl_emptyRejected() { + assertThrows(MateClawException.class, () -> service.completeFromPastedUrl(null)); + assertThrows(MateClawException.class, () -> service.completeFromPastedUrl("")); + assertThrows(MateClawException.class, () -> service.completeFromPastedUrl(" ")); + } + + @Test + @DisplayName("completeFromPastedUrl rejects URL without query string") + void pastedUrl_noQueryRejected() { + MateClawException ex = assertThrows(MateClawException.class, + () -> service.completeFromPastedUrl("http://localhost:1455/auth/callback")); + assertTrue(ex.getMessage().contains("查询参数")); + } + + @Test + @DisplayName("completeFromPastedUrl rejects URL missing code") + void pastedUrl_missingCode() { + MateClawException ex = assertThrows(MateClawException.class, + () -> service.completeFromPastedUrl( + "http://localhost:1455/auth/callback?state=xyz")); + assertTrue(ex.getMessage().contains("code")); + } + + @Test + @DisplayName("completeFromPastedUrl rejects URL missing state") + void pastedUrl_missingState() { + MateClawException ex = assertThrows(MateClawException.class, + () -> service.completeFromPastedUrl( + "http://localhost:1455/auth/callback?code=abc")); + assertTrue(ex.getMessage().contains("state")); + } + + @Test + @DisplayName("completeFromPastedUrl strips fragment after #") + void pastedUrl_stripsFragment() { + // Should successfully extract code and state, but throw because + // state isn't in pendingStates map (no real authorize was called). + // We're verifying the parser gets past the parsing stage. + MateClawException ex = assertThrows(MateClawException.class, + () -> service.completeFromPastedUrl( + "http://localhost:1455/auth/callback?code=abc&state=xyz#fragment")); + // The error must be from exchangeToken (state not in pendingStates), + // not from a parsing failure. + assertTrue(ex.getMsgKey() != null && ex.getMsgKey().contains("oauth_state_invalid"), + "Expected state validation failure (parser succeeded), got: " + ex.getMessage()); + } + + @Test + @DisplayName("completeFromPastedUrl handles URL-encoded code values") + void pastedUrl_handlesEncodedValues() { + // The exchangeToken stage will fail, but parser must have decoded + // the percent-encoded characters before getting there. + MateClawException ex = assertThrows(MateClawException.class, + () -> service.completeFromPastedUrl( + "http://localhost:1455/auth/callback?code=abc%2B123&state=test%3Dvalue")); + // Should fail at state validation, not parsing + assertTrue(ex.getMsgKey() != null && ex.getMsgKey().contains("oauth_state_invalid"), + "Parser should accept percent-encoded values; got: " + ex.getMessage()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceTest.java b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceTest.java new file mode 100644 index 00000000..30d4fa79 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/oauth/OpenAIOAuthServiceTest.java @@ -0,0 +1,38 @@ +package vip.mate.llm.oauth; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import vip.mate.llm.repository.ModelProviderMapper; +import vip.mate.llm.service.ModelProviderService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; + +class OpenAIOAuthServiceTest { + + @AfterEach + void clearProperties() { + System.clearProperty("mateclaw.oauth.openai.callback-bind-host"); + } + + @Test + void resolveCallbackBindHostDefaultsToLoopback() { + OpenAIOAuthService service = service(); + + assertEquals("127.0.0.1", service.resolveCallbackBindHost()); + } + + @Test + void resolveCallbackBindHostUsesConfiguredProperty() { + System.setProperty("mateclaw.oauth.openai.callback-bind-host", "0.0.0.0"); + OpenAIOAuthService service = service(); + + assertEquals("0.0.0.0", service.resolveCallbackBindHost()); + } + + private OpenAIOAuthService service() { + return new OpenAIOAuthService(mock(ModelProviderMapper.class), new ObjectMapper(), + mock(ModelProviderService.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/routing/MultimodalRouterTest.java b/mateclaw-server/src/test/java/vip/mate/llm/routing/MultimodalRouterTest.java new file mode 100644 index 00000000..ee7ced22 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/routing/MultimodalRouterTest.java @@ -0,0 +1,207 @@ +package vip.mate.llm.routing; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.routing.model.MultimodalRoutingDecision; +import vip.mate.llm.service.ModelCapabilityService; +import vip.mate.llm.service.ModelCapabilityService.Modality; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.workspace.conversation.model.MessageContentPart; + +import java.util.EnumSet; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class MultimodalRouterTest { + + @Mock + private SystemSettingService systemSettingService; + + @Mock + private ModelConfigService modelConfigService; + + @Mock + private ModelCapabilityService capabilityService; + + @InjectMocks + private MultimodalRouter router; + + private SystemSettingsDTO settings; + + @BeforeEach + void setUp() { + settings = new SystemSettingsDTO(); + lenient().when(systemSettingService.getSettings()).thenReturn(settings); + } + + @Test + @DisplayName("No attachments → strategy NONE, no reads to settings") + void noAttachmentsReturnsNone() { + // No capabilityService stubbing here — the router must short-circuit before + // touching capabilities when no attachments are present. + MultimodalRoutingDecision decision = router.route( + List.of(), chatModel("deepseek", "deepseek-chat", null)); + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + assertTrue(decision.skipped().isEmpty()); + assertNull(decision.sidecarModel()); + } + + @Test + @DisplayName("Primary already supports vision → strategy NONE") + void primaryCoversVisionReturnsNone() { + ModelConfigEntity primary = chatModel("zhipu", "glm-4v", "[\"vision\"]"); + when(capabilityService.resolve("glm-4v", "[\"vision\"]")) + .thenReturn(EnumSet.of(Modality.VISION)); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), primary); + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + } + + @Test + @DisplayName("Image attachment + text-only primary + configured vision sidecar → SIDECAR") + void textPrimaryImageWithSidecarConfigured() { + ModelConfigEntity primary = chatModel("deepseek", "deepseek-chat", null); + ModelConfigEntity vision = chatModel("zhipu", "glm-4v", "[\"vision\"]"); + vision.setId(42L); + vision.setEnabled(true); + + when(capabilityService.resolve("deepseek-chat", null)) + .thenReturn(EnumSet.of(Modality.TEXT)); + settings.setDefaultVisionModelId(42L); + when(modelConfigService.getModel(42L)).thenReturn(vision); + when(capabilityService.supports(eq("glm-4v"), eq("[\"vision\"]"), eq(Modality.VISION))) + .thenReturn(true); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), primary); + + assertEquals(MultimodalRoutingDecision.Strategy.SIDECAR, decision.strategy()); + assertNotNull(decision.sidecarModel()); + assertEquals(42L, decision.sidecarModel().getId()); + assertTrue(decision.skipped().isEmpty()); + } + + @Test + @DisplayName("Image + text-only primary + sidecar NOT configured → NONE with skipped reason") + void textPrimaryImageNoSidecar() { + ModelConfigEntity primary = chatModel("deepseek", "deepseek-chat", null); + when(capabilityService.resolve("deepseek-chat", null)) + .thenReturn(EnumSet.of(Modality.TEXT)); + settings.setDefaultVisionModelId(null); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), primary); + + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + assertEquals(1, decision.skipped().size()); + assertEquals("vision_model_not_configured", decision.skipped().get(0).reason()); + } + + @Test + @DisplayName("Image + sidecar configured but model disabled → NONE with vision_model_unavailable") + void textPrimaryImageSidecarDisabled() { + ModelConfigEntity primary = chatModel("deepseek", "deepseek-chat", null); + ModelConfigEntity vision = chatModel("zhipu", "glm-4v", "[\"vision\"]"); + vision.setId(42L); + vision.setEnabled(false); + + when(capabilityService.resolve("deepseek-chat", null)) + .thenReturn(EnumSet.of(Modality.TEXT)); + settings.setDefaultVisionModelId(42L); + when(modelConfigService.getModel(42L)).thenReturn(vision); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), primary); + + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + assertEquals("vision_model_unavailable", decision.skipped().get(0).reason()); + } + + @Test + @DisplayName("Video attachment never sidecarred in v1 → NONE with reserved reason") + void videoAttachmentSkippedInV1() { + ModelConfigEntity primary = chatModel("deepseek", "deepseek-chat", null); + when(capabilityService.resolve("deepseek-chat", null)) + .thenReturn(EnumSet.of(Modality.TEXT)); + + MultimodalRoutingDecision decision = router.route(List.of(videoPart("b.mp4")), primary); + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + assertEquals(1, decision.skipped().size()); + assertEquals("video_sidecar_not_supported_in_v1", decision.skipped().get(0).reason()); + } + + @Test + @DisplayName("Configured sidecar that does not actually support VISION → fallback to NONE") + void sidecarLacksClaimedCapability() { + ModelConfigEntity primary = chatModel("deepseek", "deepseek-chat", null); + ModelConfigEntity vision = chatModel("acme", "acme-chat", "[]"); + vision.setId(42L); + vision.setEnabled(true); + + when(capabilityService.resolve("deepseek-chat", null)) + .thenReturn(EnumSet.of(Modality.TEXT)); + settings.setDefaultVisionModelId(42L); + when(modelConfigService.getModel(42L)).thenReturn(vision); + when(capabilityService.supports(anyString(), anyString(), eq(Modality.VISION))) + .thenReturn(false); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), primary); + + assertEquals(MultimodalRoutingDecision.Strategy.NONE, decision.strategy()); + assertEquals("vision_model_unavailable", decision.skipped().get(0).reason()); + } + + @Test + @DisplayName("Null primary → routing returns SIDECAR if vision configured, else NONE") + void nullPrimaryHonorsSidecarConfig() { + ModelConfigEntity vision = chatModel("zhipu", "glm-4v", "[\"vision\"]"); + vision.setId(42L); + vision.setEnabled(true); + settings.setDefaultVisionModelId(42L); + when(modelConfigService.getModel(42L)).thenReturn(vision); + when(capabilityService.supports(anyString(), anyString(), eq(Modality.VISION))).thenReturn(true); + + MultimodalRoutingDecision decision = router.route(List.of(imagePart("a.png")), null); + assertEquals(MultimodalRoutingDecision.Strategy.SIDECAR, decision.strategy()); + } + + private static MessageContentPart imagePart(String fileName) { + MessageContentPart part = new MessageContentPart(); + part.setType("image"); + part.setContentType("image/png"); + part.setFileName(fileName); + return part; + } + + private static MessageContentPart videoPart(String fileName) { + MessageContentPart part = new MessageContentPart(); + part.setType("video"); + part.setContentType("video/mp4"); + part.setFileName(fileName); + return part; + } + + private static ModelConfigEntity chatModel(String provider, String modelName, String modalitiesJson) { + ModelConfigEntity m = new ModelConfigEntity(); + m.setProvider(provider); + m.setModelName(modelName); + m.setModalities(modalitiesJson); + m.setEnabled(true); + return m; + } + +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelCapabilityServiceTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelCapabilityServiceTest.java new file mode 100644 index 00000000..0c485548 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelCapabilityServiceTest.java @@ -0,0 +1,261 @@ +package vip.mate.llm.service; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.service.ModelCapabilityService.Modality; + +import java.util.EnumSet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pinpoint regression tests for {@link ModelCapabilityService}. + * + *

Per-model granularity is the whole point — the prior hardcoded + * {@code n.contains("glm") && n.contains("v")} matcher (issue #44) collapsed + * {@code glm-4v} and {@code glm-4v-plus} into the same bucket even though only + * the latter accepts video. The cases below pin that boundary so a future + * "let's just add another contains() rule" cleanup can't bring the bug back. + */ +class ModelCapabilityServiceTest { + + private final ModelCapabilityService service = new ModelCapabilityService(); + + // ---------- Heuristic table: per-model granularity ---------- + + @Test + @DisplayName("glm-4v-plus → VIDEO; glm-4v → no VIDEO (issue #44 root cause)") + void glm4v_videoCapabilityDiffers() { + assertTrue(service.supports("glm-4v-plus", null, Modality.VIDEO), + "glm-4v-plus is multimodal incl. video"); + assertFalse(service.supports("glm-4v", null, Modality.VIDEO), + "plain glm-4v is image-only — must NOT pass through video Media"); + assertFalse(service.supports("glm-4v-flash", null, Modality.VIDEO), + "glm-4v-flash is image-only"); + // All three still support vision + assertTrue(service.supports("glm-4v-plus", null, Modality.VISION)); + assertTrue(service.supports("glm-4v", null, Modality.VISION)); + assertTrue(service.supports("glm-4v-flash", null, Modality.VISION)); + } + + @Test + @DisplayName("glm-5v-turbo / glm-4.5v / glm-4.1v lines all support VIDEO") + void glmNewGenerations_supportVideo() { + // glm-5v-turbo regression: original heuristic table only had glm-4v lineage, + // so a user uploading a video to glm-5v-turbo got a "model unsupported" notice + // even though Zhipu's 5V line is built for video understanding. + assertTrue(service.supports("glm-5v-turbo", null, Modality.VIDEO), + "glm-5v-turbo is Zhipu's video-understanding model — must accept video"); + assertTrue(service.supports("glm-5v-flash", null, Modality.VIDEO)); + assertTrue(service.supports("glm-5v", null, Modality.VIDEO)); + assertTrue(service.supports("glm-4.5v", null, Modality.VIDEO)); + assertTrue(service.supports("glm-4.1v-thinking-flashx", null, Modality.VIDEO)); + } + + @Test + @DisplayName("Longest-prefix-wins: glm-4v-plus does NOT degrade to glm-4v entry") + void longestPrefixWins() { + // If matcher used shortest-or-first, "glm-4v-plus" might match the "glm-4v" entry + // first and lose its VIDEO modality. Pin the iteration order independence. + EnumSet caps = service.resolve("glm-4v-plus", null); + assertTrue(caps.contains(Modality.VIDEO), "longest prefix glm-4v-plus must win"); + } + + @Test + @DisplayName("Qwen-VL family: max → VIDEO, plus → image-only") + void qwenVl_familyDiffers() { + assertTrue(service.supports("qwen-vl-max", null, Modality.VIDEO)); + assertFalse(service.supports("qwen-vl-plus", null, Modality.VIDEO)); + assertTrue(service.supports("qwen-vl-plus", null, Modality.VISION)); + } + + @Test + @DisplayName("Qwen omni line accepts vision + video + audio") + void qwenOmni_fullyMultimodal() { + EnumSet caps = service.resolve("qwen3-omni", null); + assertTrue(caps.contains(Modality.VISION)); + assertTrue(caps.contains(Modality.VIDEO)); + assertTrue(caps.contains(Modality.AUDIO)); + } + + @Test + @DisplayName("OpenAI: vision yes across the line, but native video NO (API limitation)") + void openai_neverNativeVideo() { + // The Chat Completions / Responses APIs do not accept video files for any + // OpenAI model as of 2026-04. Granting VIDEO would cause patchVideoMediaContent + // to send video_url, and OpenAI would 400. Pin this so a future "marketing-led" + // table edit can't silently re-introduce the failure mode. + assertTrue(service.supports("gpt-5", null, Modality.VISION)); + assertTrue(service.supports("gpt-4.1", null, Modality.VISION)); + assertTrue(service.supports("gpt-4o", null, Modality.VISION)); + assertTrue(service.supports("gpt-4o-mini", null, Modality.VISION)); + assertFalse(service.supports("gpt-5", null, Modality.VIDEO)); + assertFalse(service.supports("gpt-4.1", null, Modality.VIDEO)); + assertFalse(service.supports("gpt-4o", null, Modality.VIDEO)); + assertFalse(service.supports("gpt-4o-mini", null, Modality.VIDEO)); + } + + @Test + @DisplayName("DeepSeek V4 / V4-Pro → VIDEO; V3 (text-only) gets nothing") + void deepseekV4_supportsVideo() { + // DeepSeek V4 (Apr 2026) introduced native multimodal incl. video to the line. + // V3 and earlier remain text-only and must NOT match the V4 entry. + assertTrue(service.supports("deepseek-v4", null, Modality.VIDEO)); + assertTrue(service.supports("deepseek-v4-pro", null, Modality.VIDEO)); + assertTrue(service.supports("deepseek-v4-flash", null, Modality.VIDEO)); + assertFalse(service.supports("deepseek-v3", null, Modality.VIDEO), + "V3 must NOT inherit V4 capabilities — text-only base differs from V4 entirely"); + assertFalse(service.supports("deepseek-v3.2", null, Modality.VIDEO)); + assertFalse(service.supports("deepseek-r1", null, Modality.VIDEO)); + } + + @Test + @DisplayName("Qwen3-VL (all sizes) and Qwen3.5-Omni support VIDEO") + void qwen3Generation_supportsVideo() { + assertTrue(service.supports("qwen3-vl-8b-instruct", null, Modality.VIDEO)); + assertTrue(service.supports("qwen3-vl-235b-a22b", null, Modality.VIDEO)); + assertTrue(service.supports("qwen3.5-omni", null, Modality.VIDEO)); + assertTrue(service.supports("qwen3.5-omni", null, Modality.AUDIO)); + } + + @Test + @DisplayName("Moonshot Kimi K2.6 → VIDEO; K2.5 → image only") + void kimiK26_supportsVideo() { + assertTrue(service.supports("kimi-k2.6", null, Modality.VIDEO)); + assertFalse(service.supports("kimi-k2.5", null, Modality.VIDEO)); + assertTrue(service.supports("kimi-k2.5", null, Modality.VISION)); + } + + @Test + @DisplayName("ByteDance Doubao Seed 2.0 supports VIDEO") + void doubaoSeed2_supportsVideo() { + assertTrue(service.supports("doubao-seed-2.0-pro", null, Modality.VIDEO)); + assertTrue(service.supports("doubao-seed-2.0", null, Modality.VIDEO)); + } + + @Test + @DisplayName("Gemini 2.5 (pro/flash/flash-lite) is fully multimodal") + void gemini25_fullyMultimodal() { + assertTrue(service.supports("gemini-2.5-pro", null, Modality.VIDEO)); + assertTrue(service.supports("gemini-2.5-flash", null, Modality.VIDEO)); + assertTrue(service.supports("gemini-2.5-flash-lite", null, Modality.VIDEO)); + assertTrue(service.supports("gemini-2.5-flash", null, Modality.AUDIO)); + } + + @Test + @DisplayName("Claude family: vision yes, native video no") + void claude_visionOnly() { + assertTrue(service.supports("claude-3.7-sonnet", null, Modality.VISION)); + assertTrue(service.supports("claude-opus-4-5", null, Modality.VISION)); + assertFalse(service.supports("claude-3.7-sonnet", null, Modality.VIDEO), + "Claude does not natively ingest video frames"); + } + + @Test + @DisplayName("Unknown model name: only TEXT, no vision/video/audio") + void unknownModel_textOnly() { + EnumSet caps = service.resolve("totally-made-up-model-9000", null); + assertEquals(EnumSet.of(Modality.TEXT), caps, + "unknown model must default to text-only — failsafe for issue #44 silent skip"); + } + + @Test + @DisplayName("Null/blank model name resolves cleanly to TEXT only") + void nullModelName_safe() { + assertEquals(EnumSet.of(Modality.TEXT), service.resolve(null, null)); + assertEquals(EnumSet.of(Modality.TEXT), service.resolve("", null)); + assertEquals(EnumSet.of(Modality.TEXT), service.resolve(" ", null)); + } + + @Test + @DisplayName("Case-insensitive model name match") + void caseInsensitiveMatch() { + assertTrue(service.supports("GLM-4V-PLUS", null, Modality.VIDEO)); + assertTrue(service.supports("Gpt-4o", null, Modality.VISION), + "case-insensitive match still resolves the entry; OpenAI grants vision (not video)"); + } + + @Test + @DisplayName("Llama 4 Scout / Maverick support VIDEO; Llama 3 does not") + void llama4_supportsVideo() { + assertTrue(service.supports("llama-4-scout", null, Modality.VIDEO)); + assertTrue(service.supports("llama-4-maverick", null, Modality.VIDEO)); + assertFalse(service.supports("llama-3.3-70b", null, Modality.VIDEO)); + } + + @Test + @DisplayName("Mistral / Pixtral / Grok / Hunyuan vision: image yes, video no") + void imageOnlyVendors() { + assertTrue(service.supports("pixtral-12b", null, Modality.VISION)); + assertFalse(service.supports("pixtral-12b", null, Modality.VIDEO)); + assertTrue(service.supports("mistral-small-4", null, Modality.VISION)); + assertFalse(service.supports("mistral-small-4", null, Modality.VIDEO)); + assertTrue(service.supports("grok-3", null, Modality.VISION)); + assertFalse(service.supports("grok-3", null, Modality.VIDEO), + "Grok Imagine is video generation, not input — pin this to prevent confusion"); + assertTrue(service.supports("hunyuan-vision", null, Modality.VISION)); + assertTrue(service.supports("hunyuan-large-vision", null, Modality.VISION)); + } + + @Test + @DisplayName("MiniMax-VL is vision-only (Hailuo / video-01 are generation, not input)") + void minimaxVl_visionOnly() { + assertTrue(service.supports("minimax-vl-01", null, Modality.VISION)); + assertFalse(service.supports("minimax-vl-01", null, Modality.VIDEO), + "MiniMax video models generate video, they don't ingest it"); + } + + // ---------- DB modalities override (user opt-in) ---------- + + @Test + @DisplayName("DB modalities JSON overrides heuristics — user can grant video to image-only model") + void dbOverride_grantsCapability() { + // User declares glm-4v supports video (e.g. they tested a custom endpoint that does). + // Override wins. TEXT always implicit. + EnumSet caps = service.resolve("glm-4v", "[\"vision\",\"video\"]"); + assertTrue(caps.contains(Modality.VIDEO), + "DB override must take precedence — heuristic alone says no video"); + } + + @Test + @DisplayName("DB modalities JSON overrides heuristics — user can revoke capability") + void dbOverride_revokesCapability() { + // User declares gpt-4o as vision-only (e.g. their proxy strips video). + EnumSet caps = service.resolve("gpt-4o", "[\"vision\"]"); + assertFalse(caps.contains(Modality.VIDEO), + "Empty modalities array means user explicitly opted out of video for this model"); + } + + @Test + @DisplayName("DB JSON case-insensitive on modality names") + void dbOverride_caseInsensitive() { + assertTrue(service.supports("anything", "[\"VIDEO\",\"Vision\"]", Modality.VIDEO)); + assertTrue(service.supports("anything", "[\"VIDEO\",\"Vision\"]", Modality.VISION)); + } + + @Test + @DisplayName("Invalid JSON falls back to heuristics, does not throw") + void dbOverride_invalidJson_fallsBack() { + EnumSet caps = service.resolve("glm-4v-plus", "this is not json"); + assertTrue(caps.contains(Modality.VIDEO), + "When DB JSON is malformed, fall back to heuristics so service stays available"); + } + + @Test + @DisplayName("Unknown modality string in JSON is logged and ignored, others still apply") + void dbOverride_unknownModalityIgnored() { + EnumSet caps = service.resolve("anything", "[\"vision\",\"telepathy\"]"); + assertTrue(caps.contains(Modality.VISION)); + // unknown one silently skipped, no exception + } + + @Test + @DisplayName("TEXT is always implicit, even with empty DB declaration") + void textAlwaysImplicit() { + assertTrue(service.resolve("anything", "[]").contains(Modality.TEXT)); + assertTrue(service.resolve("anything", null).contains(Modality.TEXT)); + assertTrue(service.resolve(null, null).contains(Modality.TEXT)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceDefaultModelTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceDefaultModelTest.java new file mode 100644 index 00000000..36d055d1 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceDefaultModelTest.java @@ -0,0 +1,147 @@ +package vip.mate.llm.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.test.util.ReflectionTestUtils; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.repository.ModelConfigMapper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Regression tests for ModelConfigService.getDefaultModel() provider-availability filtering. + * + * Scenario: system has a default chat model but its provider is unconfigured (e.g. DashScope + * marked as default but no API key). The method must skip it and return the first chat model + * whose provider IS configured instead of blindly returning the unconfigured default. + */ +@ExtendWith(MockitoExtension.class) +class ModelConfigServiceDefaultModelTest { + + @Mock + private ModelConfigMapper modelConfigMapper; + + @Mock + private ApplicationEventPublisher eventPublisher; + + @Mock + private ModelProviderService modelProviderService; + + @InjectMocks + private ModelConfigService service; + + @BeforeEach + void injectLazyDep() { + // Simulate the @Lazy @Autowired field injection Spring does at runtime. + ReflectionTestUtils.setField(service, "modelProviderService", modelProviderService); + } + + private static ModelConfigEntity chatModel(String provider, String modelName, boolean isDefault) { + ModelConfigEntity m = new ModelConfigEntity(); + m.setProvider(provider); + m.setModelName(modelName); + m.setIsDefault(isDefault); + m.setEnabled(true); + m.setModelType("chat"); + return m; + } + + // ── Scenario 1: default model available ──────────────────────────────────── + + @Test + @DisplayName("configured default model is returned directly") + void defaultModelConfigured_returnsIt() { + ModelConfigEntity dashscopeDefault = chatModel("dashscope", "qwen-plus", true); + + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(dashscopeDefault); + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(true); + + ModelConfigEntity result = service.getDefaultModel(); + + assertEquals("dashscope", result.getProvider()); + assertEquals("qwen-plus", result.getModelName()); + // Should not proceed to the full-scan fallback path. + verify(modelConfigMapper, times(1)).selectOne(any()); + verify(modelConfigMapper, never()).selectList(any()); + } + + // ── Scenario 2: default model provider unavailable → fallback ───────────── + + @Test + @DisplayName("default model provider unconfigured: falls back to first configured alternative") + void defaultModelProviderUnconfigured_returnsFallback() { + ModelConfigEntity dashscopeDefault = chatModel("dashscope", "qwen-plus", true); + ModelConfigEntity zhipuModel = chatModel("zhipu", "glm-4", false); + + // First selectOne → the is_default=true model + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(dashscopeDefault); + // dashscope is NOT configured, zhipu IS + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(false); + when(modelProviderService.isProviderConfigured("zhipu")).thenReturn(true); + // Full-scan returns both; zhipu comes second but dashscope is skipped + when(modelConfigMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(dashscopeDefault, zhipuModel)); + + ModelConfigEntity result = service.getDefaultModel(); + + assertEquals("zhipu", result.getProvider()); + assertEquals("glm-4", result.getModelName()); + } + + // ── Scenario 3: no configured provider at all ────────────────────────────── + + @Test + @DisplayName("all enabled chat model providers unconfigured: throws with clear message") + void allProvidersUnconfigured_throws() { + ModelConfigEntity dashscopeDefault = chatModel("dashscope", "qwen-plus", true); + ModelConfigEntity zhipuModel = chatModel("zhipu", "glm-4", false); + + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(dashscopeDefault); + when(modelProviderService.isProviderConfigured(any())).thenReturn(false); + when(modelConfigMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(dashscopeDefault, zhipuModel)); + + MateClawException ex = assertThrows(MateClawException.class, () -> service.getDefaultModel()); + assertEquals("err.llm.no_configured_provider", ex.getMsgKey()); + } + + // ── Scenario 4: no enabled model at all ─────────────────────────────────── + + @Test + @DisplayName("no enabled chat model at all: throws no_available_model") + void noEnabledModel_throws() { + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + when(modelConfigMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of()); + + MateClawException ex = assertThrows(MateClawException.class, () -> service.getDefaultModel()); + assertEquals("err.llm.no_available_model", ex.getMsgKey()); + } + + // ── Scenario 5: modelProviderService unavailable (bootstrap) ────────────── + + @Test + @DisplayName("modelProviderService null (bootstrap): default model returned without filtering") + void providerServiceNull_returnsDefaultWithoutFilter() { + ReflectionTestUtils.setField(service, "modelProviderService", null); + ModelConfigEntity dashscopeDefault = chatModel("dashscope", "qwen-plus", true); + + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(dashscopeDefault); + + // With null providerService, isProviderConfigured returns true (lenient bootstrap) + ModelConfigEntity result = service.getDefaultModel(); + assertEquals("dashscope", result.getProvider()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceResolveModelTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceResolveModelTest.java new file mode 100644 index 00000000..8339ad56 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelConfigServiceResolveModelTest.java @@ -0,0 +1,138 @@ +package vip.mate.llm.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.test.util.ReflectionTestUtils; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.repository.ModelConfigMapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Tests for {@link ModelConfigService#resolveModel(String)} — the lookup + * path used by {@code AgentGraphBuilder} to honor a per-Agent model override + * (RFC-03 Lane G1). + * + *

Contract: + *

    + *
  • Blank / null name → fall back to {@link ModelConfigService#getDefaultModel()}
  • + *
  • Name matches an enabled model → return that entity
  • + *
  • Name does not match (deleted / disabled / typo) → fall back to default
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +class ModelConfigServiceResolveModelTest { + + @Mock + private ModelConfigMapper modelConfigMapper; + + @Mock + private ApplicationEventPublisher eventPublisher; + + @Mock + private ModelProviderService modelProviderService; + + @InjectMocks + private ModelConfigService service; + + @BeforeEach + void injectLazyDep() { + // Simulate the @Lazy @Autowired field injection Spring does at runtime. + ReflectionTestUtils.setField(service, "modelProviderService", modelProviderService); + } + + private static ModelConfigEntity chatModel(String provider, String modelName, boolean isDefault) { + ModelConfigEntity m = new ModelConfigEntity(); + m.setProvider(provider); + m.setModelName(modelName); + m.setIsDefault(isDefault); + m.setEnabled(true); + m.setModelType("chat"); + return m; + } + + // ── Blank input → fall back to default ───────────────────────────────────── + + @Test + @DisplayName("null name falls back to global default") + void nullNameFallsBack() { + ModelConfigEntity defaultModel = chatModel("dashscope", "qwen-plus", true); + // resolveModel skips its own selectOne for null/blank input, then calls getDefaultModel(), + // which itself runs one selectOne lookup for the default flag. + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(defaultModel); + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(true); + + ModelConfigEntity result = service.resolveModel(null); + + assertNotNull(result); + assertEquals("qwen-plus", result.getModelName()); + // Exactly one lookup — the default-model query inside getDefaultModel(). + verify(modelConfigMapper, times(1)).selectOne(any()); + } + + @Test + @DisplayName("blank/whitespace name falls back to global default") + void blankNameFallsBack() { + ModelConfigEntity defaultModel = chatModel("dashscope", "qwen-plus", true); + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(defaultModel); + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(true); + + ModelConfigEntity result = service.resolveModel(" "); + + assertNotNull(result); + assertEquals("qwen-plus", result.getModelName()); + verify(modelConfigMapper, times(1)).selectOne(any()); + } + + // ── Match → return named model ───────────────────────────────────────────── + + @Test + @DisplayName("named model match returns the entity (no default fallback)") + void namedMatchReturnsEntity() { + ModelConfigEntity claude = chatModel("anthropic", "claude-3-5-sonnet", false); + // resolveModel's first selectOne (lookup by name) hits. + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(claude); + + ModelConfigEntity result = service.resolveModel("claude-3-5-sonnet"); + + assertNotNull(result); + assertEquals("anthropic", result.getProvider()); + assertEquals("claude-3-5-sonnet", result.getModelName()); + // Exactly one lookup — getDefaultModel must NOT be called. + verify(modelConfigMapper, times(1)).selectOne(any()); + verify(modelProviderService, never()).isProviderConfigured(any()); + } + + // ── Unmatched → fall back to default ─────────────────────────────────────── + + @Test + @DisplayName("named model not found (typo / deleted) falls back to default") + void unmatchedNameFallsBack() { + ModelConfigEntity defaultModel = chatModel("dashscope", "qwen-plus", true); + // First call (lookup by name) returns null; second call (default) returns the default. + when(modelConfigMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(null) // 1st: name lookup misses + .thenReturn(defaultModel); // 2nd: default flag lookup + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(true); + + ModelConfigEntity result = service.resolveModel("ghost-model"); + + assertNotNull(result); + assertEquals("qwen-plus", result.getModelName()); + // Two queries — one miss, then the default fallback. + verify(modelConfigMapper, times(2)).selectOne(any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceChatGPTOAuthTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceChatGPTOAuthTest.java new file mode 100644 index 00000000..1cadc444 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelDiscoveryServiceChatGPTOAuthTest.java @@ -0,0 +1,191 @@ +package vip.mate.llm.service; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestClient; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelInfoDTO; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.oauth.OpenAIOAuthService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.header; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +/** + * Unit tests for ChatGPT OAuth model discovery — the only protocol where we + * call a separate endpoint with the user's OAuth bearer token instead of an + * API key. Lower-protocol behaviour (filter, probe, dedupe) is exercised by + * the rest of {@link ModelDiscoveryService} indirectly and out of scope here. + */ +class ModelDiscoveryServiceChatGPTOAuthTest { + + private ModelDiscoveryService service; + private OpenAIOAuthService oauthService; + private MockRestServiceServer mockServer; + + @BeforeEach + void setUp() { + ModelProviderService providerService = mock(ModelProviderService.class); + ModelConfigService configService = mock(ModelConfigService.class); + oauthService = mock(OpenAIOAuthService.class); + when(oauthService.ensureValidAccessToken()).thenReturn("test-access-token"); + when(configService.listModelsByProvider(any())).thenReturn(List.of()); + + ModelProviderEntity provider = new ModelProviderEntity(); + provider.setProviderId("openai-chatgpt"); + provider.setChatModel("ChatGPTChatModel"); + provider.setSupportModelDiscovery(true); + when(providerService.getProviderConfig("openai-chatgpt")).thenReturn(provider); + + service = new ModelDiscoveryService(providerService, configService, + new ObjectMapper(), oauthService); + + RestClient.Builder builder = RestClient.builder(); + mockServer = MockRestServiceServer.bindTo(builder).build(); + service.setChatgptCodexClient(builder.build()); + } + + // --------------------------------------------------------------------- + // parseChatGPTCodexModelsResponse — pure parsing tests + // --------------------------------------------------------------------- + + @Test + @DisplayName("parser drops supported_in_api=false and visibility=hide entries") + void parser_dropsHiddenAndUnsupported() { + String body = "{\"models\":[" + + "{\"slug\":\"gpt-5.4\",\"supported_in_api\":true,\"visibility\":\"shown\",\"priority\":10}," + + "{\"slug\":\"gpt-internal\",\"supported_in_api\":false,\"priority\":5}," + + "{\"slug\":\"gpt-research\",\"supported_in_api\":true,\"visibility\":\"hide\",\"priority\":1}," + + "{\"slug\":\"gpt-5.4-mini\",\"supported_in_api\":true,\"visibility\":\"shown\",\"priority\":20}" + + "]}"; + + List models = service.parseChatGPTCodexModelsResponse(body); + List ids = models.stream().map(ModelInfoDTO::getId).toList(); + + assertEquals(List.of("gpt-5.4", "gpt-5.4-mini"), ids); + } + + @Test + @DisplayName("parser sorts by priority ascending") + void parser_sortsByPriority() { + String body = "{\"models\":[" + + "{\"slug\":\"third\",\"supported_in_api\":true,\"priority\":30}," + + "{\"slug\":\"first\",\"supported_in_api\":true,\"priority\":1}," + + "{\"slug\":\"second\",\"supported_in_api\":true,\"priority\":15}" + + "]}"; + + List ids = service.parseChatGPTCodexModelsResponse(body) + .stream().map(ModelInfoDTO::getId).toList(); + assertEquals(List.of("first", "second", "third"), ids); + } + + @Test + @DisplayName("parser tolerates missing or non-list bodies") + void parser_tolerantOfBadInput() { + assertTrue(service.parseChatGPTCodexModelsResponse(null).isEmpty()); + assertTrue(service.parseChatGPTCodexModelsResponse("").isEmpty()); + assertTrue(service.parseChatGPTCodexModelsResponse("{}").isEmpty()); + assertTrue(service.parseChatGPTCodexModelsResponse("{\"models\": \"not-a-list\"}").isEmpty()); + assertTrue(service.parseChatGPTCodexModelsResponse("not-json").isEmpty()); + } + + // --------------------------------------------------------------------- + // addChatGPTForwardCompatModels — the synthesis layer + // --------------------------------------------------------------------- + + @Test + @DisplayName("forward-compat synthesizes gpt-5.5 when only gpt-5.4 is exposed") + void forwardCompat_synthesizesGpt55FromGpt54() { + List input = List.of(new ModelInfoDTO("gpt-5.4", "gpt-5.4")); + List out = ModelDiscoveryService.addChatGPTForwardCompatModels(input) + .stream().map(ModelInfoDTO::getId).toList(); + assertTrue(out.contains("gpt-5.5"), "Expected gpt-5.5 to be appended; got " + out); + assertTrue(out.contains("gpt-5.4")); + } + + @Test + @DisplayName("forward-compat does not duplicate slugs already in the input") + void forwardCompat_noDuplicates() { + List input = List.of( + new ModelInfoDTO("gpt-5.5", "gpt-5.5"), + new ModelInfoDTO("gpt-5.4", "gpt-5.4")); + List out = ModelDiscoveryService.addChatGPTForwardCompatModels(input) + .stream().map(ModelInfoDTO::getId).toList(); + assertEquals(1, out.stream().filter("gpt-5.5"::equals).count()); + assertEquals(1, out.stream().filter("gpt-5.4"::equals).count()); + } + + @Test + @DisplayName("forward-compat is a no-op when no template ancestor is present") + void forwardCompat_noOpOnEmptyOrUnrelated() { + List empty = ModelDiscoveryService.addChatGPTForwardCompatModels(List.of()) + .stream().map(ModelInfoDTO::getId).toList(); + assertTrue(empty.isEmpty()); + + List unrelated = ModelDiscoveryService.addChatGPTForwardCompatModels( + List.of(new ModelInfoDTO("gpt-3.5", "gpt-3.5"))) + .stream().map(ModelInfoDTO::getId).toList(); + assertEquals(List.of("gpt-3.5"), unrelated); + } + + // --------------------------------------------------------------------- + // discoverModels — end-to-end through the OAuth path + // --------------------------------------------------------------------- + + @Test + @DisplayName("discoverModels sends Bearer token and returns sorted+forward-compat catalog") + void discoverModels_endToEnd() { + mockServer.expect(requestTo(ModelDiscoveryService.CHATGPT_CODEX_MODELS_URL)) + .andExpect(header(HttpHeaders.AUTHORIZATION, "Bearer test-access-token")) + .andRespond(withSuccess( + "{\"models\":[" + + "{\"slug\":\"gpt-5.4\",\"supported_in_api\":true,\"priority\":10}," + + "{\"slug\":\"gpt-5.4-mini\",\"supported_in_api\":true,\"priority\":20}," + + "{\"slug\":\"gpt-internal\",\"supported_in_api\":false,\"priority\":5}" + + "]}", + MediaType.APPLICATION_JSON)); + + var result = service.discoverModels("openai-chatgpt"); + List all = result.getDiscoveredModels().stream().map(ModelInfoDTO::getId).toList(); + + // priority-sorted real models, plus gpt-5.5 synthesised by forward-compat + assertEquals(List.of("gpt-5.4", "gpt-5.4-mini", "gpt-5.5"), all); + verify(oauthService).ensureValidAccessToken(); + mockServer.verify(); + } + + @Test + @DisplayName("discoverModels surfaces fetch failures as err.llm.chatgpt_models_fetch_failed") + void discoverModels_surfacesFetchFailure() { + mockServer.expect(requestTo(ModelDiscoveryService.CHATGPT_CODEX_MODELS_URL)) + .andRespond(withStatus(HttpStatus.UNAUTHORIZED)); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.discoverModels("openai-chatgpt")); + assertEquals("err.llm.chatgpt_models_fetch_failed", ex.getMsgKey()); + } + + @Test + @DisplayName("discoverModels propagates oauth_not_connected from OpenAIOAuthService unchanged") + void discoverModels_propagatesOauthNotConnected() { + when(oauthService.ensureValidAccessToken()) + .thenThrow(new MateClawException("err.llm.oauth_not_connected", "未连接")); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.discoverModels("openai-chatgpt")); + assertEquals("err.llm.oauth_not_connected", ex.getMsgKey()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceConfiguredTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceConfiguredTest.java new file mode 100644 index 00000000..ce3faab3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceConfiguredTest.java @@ -0,0 +1,325 @@ +package vip.mate.llm.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.llm.anthropic.oauth.ClaudeCodeOAuthService; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; +import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.model.Liveness; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.model.ProviderInfoDTO; +import vip.mate.llm.repository.ModelProviderMapper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Issue #81: row-based isProviderConfigured + applySuggestedAction. Each test is + * one row of the truth table in RFC §2.3 (behavior diff vs. v1) and §7 + * (suggestedAction decision tree). + */ +class ModelProviderServiceConfiguredTest { + + private ModelProviderMapper providerMapper; + private ModelConfigService modelConfigService; + private ApplicationEventPublisher eventPublisher; + private ObjectProvider claudeCodeOAuthProvider; + private ClaudeCodeOAuthService claudeCodeOAuthService; + private AvailableProviderPool pool; + private ProviderHealthTracker healthTracker; + private ProviderInitProbe initProbe; + private ObjectProvider initProbeProvider; + + private ModelProviderService service; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + providerMapper = mock(ModelProviderMapper.class); + modelConfigService = mock(ModelConfigService.class); + eventPublisher = mock(ApplicationEventPublisher.class); + claudeCodeOAuthProvider = mock(ObjectProvider.class); + claudeCodeOAuthService = mock(ClaudeCodeOAuthService.class); + when(claudeCodeOAuthProvider.getIfAvailable()).thenReturn(null); + pool = new AvailableProviderPool(); + ProviderHealthProperties props = new ProviderHealthProperties(); + props.setFailureThreshold(1); + healthTracker = new ProviderHealthTracker(props); + initProbe = mock(ProviderInitProbe.class); + initProbeProvider = mock(ObjectProvider.class); + when(initProbeProvider.getIfAvailable()).thenReturn(initProbe); + // Default: every provider has been probed so liveness is computed normally. + when(initProbe.hasBeenProbed(any())).thenReturn(true); + + service = new ModelProviderService(providerMapper, modelConfigService, eventPublisher, + claudeCodeOAuthProvider, pool, healthTracker, initProbeProvider); + } + + @Test + @DisplayName("Issue #81: llama.cpp local + empty Base URL → UNCONFIGURED + fill_base_url + hint") + void llamacppEmptyBaseUrl() { + ModelProviderEntity p = local("llamacpp"); + p.setBaseUrl(""); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured(), "empty Base URL must NOT be considered configured"); + assertEquals(Liveness.UNCONFIGURED, dto.getLiveness()); + assertEquals("fill_base_url", dto.getSuggestedAction()); + assertEquals("provider.hint.llamacppBaseUrlExample", dto.getSuggestedActionHintKey()); + assertEquals("http://127.0.0.1:8080/v1", dto.getSuggestedActionHintArgs().get("example")); + assertEquals("baseUrl", dto.getMissingFields()); + assertEquals("NOT_REQUIRED", dto.getAuthStatus()); + assertFalse(dto.getBaseUrlComplete()); + } + + @Test + @DisplayName("llama.cpp local + Base URL filled but pool REMOVED → REMOVED + reprobe") + void llamacppBaseUrlFilledButRemoved() { + ModelProviderEntity p = local("llamacpp"); + p.setBaseUrl("http://127.0.0.1:8080/v1"); + seedProviderRow(p, true); + pool.remove("llamacpp", AvailableProviderPool.RemovalSource.INIT_PROBE, + "init probe failed: connection refused"); + + ProviderInfoDTO dto = singleResult(); + assertTrue(dto.getConfigured()); + assertEquals(Liveness.REMOVED, dto.getLiveness()); + assertEquals("reprobe", dto.getSuggestedAction()); + assertNull(dto.getSuggestedActionHintKey(), "REMOVED state should not carry a hint key"); + } + + @Test + @DisplayName("Ollama local + LIVE + 0 models + supportModelDiscovery=true → pull_model") + void ollamaLiveNoModels() { + ModelProviderEntity p = local("ollama"); + p.setBaseUrl("http://127.0.0.1:11434"); + p.setSupportModelDiscovery(true); + when(providerMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(p)); + when(modelConfigService.listModels()).thenReturn(List.of()); // no models registered + pool.add("ollama"); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.LIVE, dto.getLiveness()); + assertEquals("pull_model", dto.getSuggestedAction()); + } + + @Test + @DisplayName("OpenAI cloud + apiKey empty → UNCONFIGURED + fill_api_key + no hint") + void openaiCloudEmptyApiKey() { + ModelProviderEntity p = cloud("openai", true); + p.setApiKey(""); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals(Liveness.UNCONFIGURED, dto.getLiveness()); + assertEquals("fill_api_key", dto.getSuggestedAction()); + assertNull(dto.getSuggestedActionHintKey(), "cloud providers don't need a base-url hint"); + assertEquals("MISSING", dto.getAuthStatus()); + assertEquals("apiKey", dto.getMissingFields()); + assertNull(dto.getBaseUrlComplete(), "cloud provider's baseUrlComplete should be null (n/a)"); + } + + @Test + @DisplayName("OpenAI cloud + apiKey filled + LIVE → none + CONFIGURED") + void openaiCloudHealthy() { + ModelProviderEntity p = cloud("openai", true); + p.setApiKey("sk-test-1234567890"); + seedProviderRow(p, true); + pool.add("openai"); + + ProviderInfoDTO dto = singleResult(); + assertTrue(dto.getConfigured()); + assertEquals(Liveness.LIVE, dto.getLiveness()); + assertEquals("none", dto.getSuggestedAction()); + assertEquals("CONFIGURED", dto.getAuthStatus()); + assertEquals("", dto.getMissingFields()); + } + + @Test + @DisplayName("Kimi cloud + apiKey empty → fill_api_key (same shape as OpenAI)") + void kimiCloudEmptyApiKey() { + ModelProviderEntity p = cloud("kimi", true); + p.setApiKey(""); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals("fill_api_key", dto.getSuggestedAction()); + } + + @Test + @DisplayName("Custom OpenAI-compat + baseUrl empty + apiKey filled + requireApiKey=true → fill_base_url") + void customOpenAiCompatEmptyBaseUrl() { + ModelProviderEntity p = custom("my-server"); + p.setRequireApiKey(true); + p.setBaseUrl(""); + p.setApiKey("sk-test-1234567890"); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals("fill_base_url", dto.getSuggestedAction()); + assertEquals("baseUrl", dto.getMissingFields()); + } + + @Test + @DisplayName("Custom OpenAI-compat + baseUrl filled + apiKey empty + requireApiKey=true → fill_api_key") + void customOpenAiCompatEmptyApiKey() { + ModelProviderEntity p = custom("my-server"); + p.setRequireApiKey(true); + p.setBaseUrl("http://x.example.com/v1"); + p.setApiKey(""); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals("fill_api_key", dto.getSuggestedAction()); + assertEquals("apiKey", dto.getMissingFields()); + } + + @Test + @DisplayName("Custom OpenAI-compat + both empty + requireApiKey=true → configure_required_fields + both missing") + void customOpenAiCompatBothEmpty() { + ModelProviderEntity p = custom("my-server"); + p.setRequireApiKey(true); + p.setBaseUrl(""); + p.setApiKey(""); + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals("configure_required_fields", dto.getSuggestedAction()); + assertEquals("apiKey,baseUrl", dto.getMissingFields()); + // hint emitted because action is configure_required_fields + assertEquals("provider.hint.openaiCompatBaseUrlExample", dto.getSuggestedActionHintKey()); + } + + @Test + @DisplayName("Custom OpenAI-compat + both filled + LIVE → none") + void customOpenAiCompatHealthy() { + ModelProviderEntity p = custom("my-server"); + p.setRequireApiKey(true); + p.setBaseUrl("http://x.example.com/v1"); + p.setApiKey("sk-test-1234567890"); + seedProviderRow(p, true); + pool.add("my-server"); + + ProviderInfoDTO dto = singleResult(); + assertTrue(dto.getConfigured()); + assertEquals(Liveness.LIVE, dto.getLiveness()); + assertEquals("none", dto.getSuggestedAction()); + } + + @Test + @DisplayName("OAuth provider not connected → UNCONFIGURED + start_oauth + OAUTH_PENDING") + void oauthNotConnected() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("some-oauth"); + p.setName("Some OAuth"); + p.setAuthType("oauth"); + // No oauthAccessToken → not configured. + seedProviderRow(p, false); + + ProviderInfoDTO dto = singleResult(); + assertFalse(dto.getConfigured()); + assertEquals(Liveness.UNCONFIGURED, dto.getLiveness()); + assertEquals("start_oauth", dto.getSuggestedAction()); + assertEquals("OAUTH_PENDING", dto.getAuthStatus()); + } + + @Test + @DisplayName("OAuth provider connected → LIVE + CONFIGURED") + void oauthConnected() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("some-oauth"); + p.setName("Some OAuth"); + p.setAuthType("oauth"); + p.setOauthAccessToken("ya29.test"); + seedProviderRow(p, true); + pool.add("some-oauth"); + + ProviderInfoDTO dto = singleResult(); + assertTrue(dto.getConfigured()); + assertEquals(Liveness.LIVE, dto.getLiveness()); + assertEquals("CONFIGURED", dto.getAuthStatus()); + } + + @Test + @DisplayName("Default 'enabled' filter: providers without enabled=true are excluded") + void defaultProviderRespectsEnabledFlag() { + // Sanity: the existing infrastructure still gates on enabled when listProviders + // is called. seedProviderRow sets enabled=true so this is just defensive. + ModelProviderEntity p = local("ollama"); + p.setEnabled(true); + seedProviderRow(p, true); + pool.add("ollama"); + assertEquals(1, service.listProviders().size()); + } + + // ============================================================ + // Helpers + // ============================================================ + + private void seedProviderRow(ModelProviderEntity p, boolean withModel) { + if (p.getEnabled() == null) p.setEnabled(true); + when(providerMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(p)); + if (withModel) { + ModelConfigEntity m = new ModelConfigEntity(); + m.setProvider(p.getProviderId()); + m.setModelName(p.getProviderId() + "-model"); + m.setName(p.getProviderId() + "-model"); + m.setBuiltin(true); + when(modelConfigService.listModels()).thenReturn(List.of(m)); + } else { + when(modelConfigService.listModels()).thenReturn(List.of()); + } + } + + private static ModelProviderEntity cloud(String id, boolean requireApiKey) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsLocal(false); + p.setIsCustom(false); + p.setRequireApiKey(requireApiKey); + return p; + } + + private static ModelProviderEntity local(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsLocal(true); + p.setIsCustom(false); + p.setRequireApiKey(false); + p.setBaseUrl("http://127.0.0.1:11434"); // overridden per test as needed + return p; + } + + private static ModelProviderEntity custom(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsLocal(false); + p.setIsCustom(true); + return p; + } + + private ProviderInfoDTO singleResult() { + List list = service.listProviders(); + assertEquals(1, list.size()); + return list.get(0); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceCustomProviderTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceCustomProviderTest.java new file mode 100644 index 00000000..843f2a90 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceCustomProviderTest.java @@ -0,0 +1,259 @@ +package vip.mate.llm.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.exception.MateClawException; +import vip.mate.llm.anthropic.oauth.ClaudeCodeOAuthService; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; +import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.model.CreateCustomProviderRequest; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.model.ProviderConfigRequest; +import vip.mate.llm.repository.ModelProviderMapper; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import org.mockito.ArgumentCaptor; + +/** + * Issue #39 regression: provider id ends up as a single path segment in + * {@code /custom-providers/{providerId}}, so any unsafe character (slash, + * space, {@code #}, {@code ?}) makes Spring's PathPatternParser miss the + * controller and fall through to the static-resource handler — symptom is + * a {@code NoResourceFoundException} on the DELETE the user reported. + * + *

These tests pin the two layers of the fix:

+ *
    + *
  • {@code createCustomProvider} rejects unsafe ids server-side, so a + * non-UI client (curl / Electron / 3rd-party) cannot bypass the + * front-end regex and persist a row that's later undeletable.
  • + *
  • {@code deleteCustomProvider} itself doesn't care about the shape + * of the id — it deletes by primary key. Anything that did + * slip into the DB before the create-side guard existed can still be + * cleaned up via the query-param controller variant.
  • + *
+ */ +class ModelProviderServiceCustomProviderTest { + + private ModelProviderMapper providerMapper; + private ModelConfigService modelConfigService; + private ApplicationEventPublisher eventPublisher; + private ObjectProvider claudeCodeOAuthProvider; + private AvailableProviderPool pool; + private ProviderHealthTracker healthTracker; + private ProviderInitProbe initProbe; + private ObjectProvider initProbeProvider; + + private ModelProviderService service; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + providerMapper = mock(ModelProviderMapper.class); + modelConfigService = mock(ModelConfigService.class); + eventPublisher = mock(ApplicationEventPublisher.class); + claudeCodeOAuthProvider = mock(ObjectProvider.class); + when(claudeCodeOAuthProvider.getIfAvailable()).thenReturn(null); + pool = new AvailableProviderPool(); + healthTracker = new ProviderHealthTracker(new ProviderHealthProperties()); + initProbe = mock(ProviderInitProbe.class); + initProbeProvider = mock(ObjectProvider.class); + when(initProbeProvider.getIfAvailable()).thenReturn(initProbe); + + service = new ModelProviderService(providerMapper, modelConfigService, eventPublisher, + claudeCodeOAuthProvider, pool, healthTracker, initProbeProvider); + } + + // ==================== create-side guard ==================== + + @Test + @DisplayName("createCustomProvider rejects ids containing '/' (issue #39 root cause)") + void rejectsSlashInId() { + CreateCustomProviderRequest req = req("google/gemma-4-e4b", "Local Gemma"); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.createCustomProvider(req)); + + assertEquals("err.llm.provider_id_invalid", ex.getMsgKey()); + verify(providerMapper, never()).insert(any(ModelProviderEntity.class)); + verify(eventPublisher, never()).publishEvent(any()); + } + + @Test + @DisplayName("createCustomProvider rejects ids containing whitespace") + void rejectsSpaceInId() { + CreateCustomProviderRequest req = req("my provider", "Local Gemma"); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.createCustomProvider(req)); + + assertEquals("err.llm.provider_id_invalid", ex.getMsgKey()); + verify(providerMapper, never()).insert(any(ModelProviderEntity.class)); + } + + @Test + @DisplayName("createCustomProvider rejects ids starting with '-' (regex requires alnum first char)") + void rejectsLeadingHyphen() { + CreateCustomProviderRequest req = req("-foo", "Local Gemma"); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.createCustomProvider(req)); + + assertEquals("err.llm.provider_id_invalid", ex.getMsgKey()); + } + + @Test + @DisplayName("createCustomProvider rejects ids longer than 64 characters") + void rejectsOverlongId() { + // 65 chars: 'a' followed by 64 'b's. + String tooLong = "a" + "b".repeat(64); + CreateCustomProviderRequest req = req(tooLong, "Local Gemma"); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.createCustomProvider(req)); + + assertEquals("err.llm.provider_id_invalid", ex.getMsgKey()); + } + + @Test + @DisplayName("createCustomProvider accepts a normal id (e.g. 'local-gemma') and persists") + void acceptsNormalId() { + CreateCustomProviderRequest req = req("local-gemma", "Local Gemma"); + when(providerMapper.selectById("local-gemma")).thenReturn(null); + + service.createCustomProvider(req); + + verify(providerMapper).insert(any(ModelProviderEntity.class)); + } + + @Test + @DisplayName("createCustomProvider accepts ids with dot/underscore/hyphen and digits") + void acceptsRichButSafeChars() { + CreateCustomProviderRequest req = req("My_Local-Gemma.v2", "Local Gemma"); + when(providerMapper.selectById("My_Local-Gemma.v2")).thenReturn(null); + + service.createCustomProvider(req); + + verify(providerMapper).insert(any(ModelProviderEntity.class)); + } + + @Test + @DisplayName("createCustomProvider persists requireApiKey=false for keyless internal OpenAI-compatible endpoints") + void createCustomProviderCanDisableApiKeyRequirement() { + CreateCustomProviderRequest req = req("internal-llm", "Internal LLM"); + req.setDefaultBaseUrl("http://llm.internal/v1"); + req.setRequireApiKey(false); + when(providerMapper.selectById("internal-llm")).thenReturn(null); + + service.createCustomProvider(req); + + ArgumentCaptor captor = ArgumentCaptor.forClass(ModelProviderEntity.class); + verify(providerMapper).insert(captor.capture()); + assertFalse(captor.getValue().getRequireApiKey()); + } + + @Test + @DisplayName("Empty id still produces 'fields_required' (existing guard, not the new regex)") + void emptyIdStillReportsFieldsRequired() { + CreateCustomProviderRequest req = req("", "Local Gemma"); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.createCustomProvider(req)); + + assertEquals("err.llm.provider_fields_required", ex.getMsgKey()); + } + + // ==================== delete-side: dirty data rescue ==================== + + @Test + @DisplayName("deleteCustomProvider works for an id with '/' once it reaches the service " + + "(query-param controller variant is the URL bridge)") + void deletesIdContainingSlash() { + String dirtyId = "google/gemma-4-e4b"; + ModelProviderEntity dirty = customProvider(dirtyId); + when(providerMapper.selectById(dirtyId)).thenReturn(dirty); + + service.deleteCustomProvider(dirtyId); + + verify(modelConfigService).deleteModelsByProvider(dirtyId); + verify(providerMapper).deleteById(dirtyId); + } + + @Test + @DisplayName("deleteCustomProvider on a normal id (path-variant happy path) still works") + void deletesNormalId() { + String id = "local-gemma"; + ModelProviderEntity p = customProvider(id); + when(providerMapper.selectById(id)).thenReturn(p); + + service.deleteCustomProvider(id); + + verify(modelConfigService).deleteModelsByProvider(id); + verify(providerMapper).deleteById(id); + } + + @Test + @DisplayName("deleteCustomProvider refuses to delete a built-in (non-custom) provider") + void refusesToDeleteBuiltin() { + String id = "openai"; + ModelProviderEntity builtin = customProvider(id); + builtin.setIsCustom(false); + when(providerMapper.selectById(id)).thenReturn(builtin); + + MateClawException ex = assertThrows(MateClawException.class, + () -> service.deleteCustomProvider(id)); + + assertEquals("err.llm.provider_builtin_readonly", ex.getMsgKey()); + verify(providerMapper, never()).deleteById(any(String.class)); + verify(modelConfigService, never()).deleteModelsByProvider(any()); + } + + @Test + @DisplayName("updateProviderConfig can switch an existing custom provider to keyless mode") + void updateProviderConfigCanDisableApiKeyRequirement() { + String id = "internal-llm"; + ModelProviderEntity existing = customProvider(id); + existing.setBaseUrl("http://llm.internal/v1"); + existing.setRequireApiKey(true); + when(providerMapper.selectById(id)).thenReturn(existing); + when(modelConfigService.listModelsByProvider(id)).thenReturn(java.util.List.of()); + + ProviderConfigRequest req = new ProviderConfigRequest(); + req.setBaseUrl("http://llm.internal/v1"); + req.setProtocol("openai-compatible"); + req.setChatModel("OpenAIChatModel"); + req.setRequireApiKey(false); + + service.updateProviderConfig(id, req); + + assertFalse(existing.getRequireApiKey()); + verify(providerMapper).updateById(existing); + } + + // ==================== fixtures ==================== + + private static CreateCustomProviderRequest req(String id, String name) { + CreateCustomProviderRequest r = new CreateCustomProviderRequest(); + r.setId(id); + r.setName(name); + r.setProtocol("openai-compatible"); + r.setChatModel("OpenAIChatModel"); + return r; + } + + private static ModelProviderEntity customProvider(String id) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsCustom(true); + p.setIsLocal(false); + p.setEnabled(true); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceEnableTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceEnableTest.java new file mode 100644 index 00000000..afa6f791 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceEnableTest.java @@ -0,0 +1,235 @@ +package vip.mate.llm.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.exception.MateClawException; +import vip.mate.llm.anthropic.oauth.ClaudeCodeOAuthService; +import vip.mate.llm.event.ModelConfigChangedEvent; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; +import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.model.EnableResult; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.repository.ModelProviderMapper; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * RFC-074: covers the enable / disable lifecycle: + *
    + *
  • setEnabled flips the column and publishes {@link ModelConfigChangedEvent}.
  • + *
  • Disabling the provider that owns the current default model auto-promotes + * a replacement so chat doesn't break on the next request.
  • + *
  • Disabling a provider whose model is NOT the current default is a no-op + * on the default model.
  • + *
  • If no replacement provider exists, the call returns {@code unchanged()} + * and the broken default is left for the empty-state UI to catch.
  • + *
  • setEnabled(true) on an already-enabled row (or false on disabled) is a no-op.
  • + *
+ * + *

List-vs-catalog filtering is intentionally not tested here — the + * MyBatis Plus mapper is mocked, so the {@code .eq(enabled, true)} clause + * doesn't actually run. That's an integration concern handled by manual + * Flyway smoke verification (and would need a Testcontainers test to cover + * properly). The unit test concerns are state transitions + side effects.

+ */ +class ModelProviderServiceEnableTest { + + private ModelProviderMapper providerMapper; + private ModelConfigService modelConfigService; + private ApplicationEventPublisher eventPublisher; + private ObjectProvider claudeCodeOAuthProvider; + private AvailableProviderPool pool; + private ProviderHealthTracker healthTracker; + private ProviderInitProbe initProbe; + private ObjectProvider initProbeProvider; + + private ModelProviderService service; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + providerMapper = mock(ModelProviderMapper.class); + modelConfigService = mock(ModelConfigService.class); + eventPublisher = mock(ApplicationEventPublisher.class); + claudeCodeOAuthProvider = mock(ObjectProvider.class); + when(claudeCodeOAuthProvider.getIfAvailable()).thenReturn(null); + pool = new AvailableProviderPool(); + healthTracker = new ProviderHealthTracker(new ProviderHealthProperties()); + initProbe = mock(ProviderInitProbe.class); + initProbeProvider = mock(ObjectProvider.class); + when(initProbeProvider.getIfAvailable()).thenReturn(initProbe); + + service = new ModelProviderService(providerMapper, modelConfigService, eventPublisher, + claudeCodeOAuthProvider, pool, healthTracker, initProbeProvider); + } + + @Test + @DisplayName("setEnabled(true) on disabled row: flips flag, persists, publishes 'provider-enabled' event") + void enableFlipsFlag() { + ModelProviderEntity openai = providerEntity("openai", false /* disabled */); + when(providerMapper.selectById("openai")).thenReturn(openai); + + EnableResult result = service.setEnabled("openai", true); + + assertFalse(result.defaultSwitched()); + assertTrue(openai.getEnabled(), "in-memory entity flipped"); + verify(providerMapper).updateById(openai); + ArgumentCaptor evtCap = ArgumentCaptor.forClass(ModelConfigChangedEvent.class); + verify(eventPublisher).publishEvent(evtCap.capture()); + assertEquals("provider-enabled", evtCap.getValue().reason()); + } + + @Test + @DisplayName("setEnabled(true) on already-enabled row: no DB write, no event") + void enableNoOpOnAlreadyEnabled() { + ModelProviderEntity openai = providerEntity("openai", true /* already enabled */); + when(providerMapper.selectById("openai")).thenReturn(openai); + + EnableResult result = service.setEnabled("openai", true); + + assertFalse(result.defaultSwitched()); + verify(providerMapper, never()).updateById(any(ModelProviderEntity.class)); + verify(eventPublisher, never()).publishEvent(any()); + } + + @Test + @DisplayName("setEnabled(false) when provider's model is current default: auto-switches and reports new") + void disableSwitchesDefault() { + ModelProviderEntity disabled = providerEntity("openai", true); + ModelProviderEntity replacement = providerEntity("dashscope", true); + when(providerMapper.selectById("openai")).thenReturn(disabled); + + // Current default belongs to openai + ModelConfigEntity currentDefault = new ModelConfigEntity(); + currentDefault.setProvider("openai"); + currentDefault.setModelName("gpt-4"); + when(modelConfigService.getDefaultModel()).thenReturn(currentDefault); + + // After excluding openai, dashscope is the only candidate + when(providerMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(replacement)); + + ModelConfigEntity dashModel = new ModelConfigEntity(); + dashModel.setProvider("dashscope"); + dashModel.setModelName("qwen-plus"); + when(modelConfigService.listModelsByProvider("dashscope")).thenReturn(List.of(dashModel)); + + EnableResult result = service.setEnabled("openai", false); + + assertTrue(result.defaultSwitched()); + assertEquals("dashscope", result.newDefaultProviderId()); + assertEquals("qwen-plus", result.newDefaultModel()); + verify(modelConfigService).setDefaultModel("dashscope", "qwen-plus"); + } + + @Test + @DisplayName("setEnabled(false) when current default belongs to another provider: no switch") + void disableLeavesDefaultAlone() { + ModelProviderEntity disabled = providerEntity("openai", true); + when(providerMapper.selectById("openai")).thenReturn(disabled); + + // Current default belongs to a different provider — no switch needed + ModelConfigEntity currentDefault = new ModelConfigEntity(); + currentDefault.setProvider("dashscope"); + currentDefault.setModelName("qwen-plus"); + when(modelConfigService.getDefaultModel()).thenReturn(currentDefault); + + EnableResult result = service.setEnabled("openai", false); + + assertFalse(result.defaultSwitched()); + verify(modelConfigService, never()).setDefaultModel(anyString(), anyString()); + } + + @Test + @DisplayName("setEnabled(false) with no replacement candidate: returns unchanged, leaves broken default for UI") + void disableNoReplacement() { + ModelProviderEntity disabled = providerEntity("openai", true); + when(providerMapper.selectById("openai")).thenReturn(disabled); + ModelConfigEntity currentDefault = new ModelConfigEntity(); + currentDefault.setProvider("openai"); + currentDefault.setModelName("gpt-4"); + when(modelConfigService.getDefaultModel()).thenReturn(currentDefault); + + // No other enabled providers + when(providerMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(new ArrayList<>()); + + EnableResult result = service.setEnabled("openai", false); + + assertFalse(result.defaultSwitched(), + "no replacement → unchanged; UI empty-state will catch the broken default"); + verify(modelConfigService, never()).setDefaultModel(anyString(), anyString()); + } + + @Test + @DisplayName("setEnabled(false) when getDefaultModel throws (no default at all): returns unchanged") + void disableWhenNoDefaultExists() { + ModelProviderEntity disabled = providerEntity("openai", true); + when(providerMapper.selectById("openai")).thenReturn(disabled); + when(modelConfigService.getDefaultModel()) + .thenThrow(new MateClawException("err.test.no_default", "no default")); + + EnableResult result = service.setEnabled("openai", false); + + assertFalse(result.defaultSwitched()); + verify(modelConfigService, never()).setDefaultModel(anyString(), anyString()); + } + + @Test + @DisplayName("setEnabled(false) auto-switch skips replacement candidates with no models") + void disableSkipsReplacementWithNoModels() { + ModelProviderEntity disabled = providerEntity("openai", true); + ModelProviderEntity emptyCandidate = providerEntity("anthropic", true); + ModelProviderEntity goodCandidate = providerEntity("dashscope", true); + when(providerMapper.selectById("openai")).thenReturn(disabled); + + ModelConfigEntity currentDefault = new ModelConfigEntity(); + currentDefault.setProvider("openai"); + currentDefault.setModelName("gpt-4"); + when(modelConfigService.getDefaultModel()).thenReturn(currentDefault); + + // anthropic appears first in the candidates list but has no models + when(providerMapper.selectList(any(LambdaQueryWrapper.class))) + .thenReturn(List.of(emptyCandidate, goodCandidate)); + when(modelConfigService.listModelsByProvider("anthropic")).thenReturn(new ArrayList<>()); + ModelConfigEntity dashModel = new ModelConfigEntity(); + dashModel.setProvider("dashscope"); + dashModel.setModelName("qwen-plus"); + when(modelConfigService.listModelsByProvider("dashscope")).thenReturn(List.of(dashModel)); + + EnableResult result = service.setEnabled("openai", false); + + assertTrue(result.defaultSwitched()); + assertEquals("dashscope", result.newDefaultProviderId()); + verify(modelConfigService, never()).setDefaultModel(eq("anthropic"), anyString()); + verify(modelConfigService).setDefaultModel("dashscope", "qwen-plus"); + } + + /** Build a fully-configured cloud entity with the given enabled state. */ + private static ModelProviderEntity providerEntity(String id, boolean enabled) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsLocal(false); + p.setIsCustom(false); + p.setRequireApiKey(true); + p.setApiKey("sk-test-key-1234567890"); + p.setBaseUrl("https://api.example.com/v1"); + p.setEnabled(enabled); + return p; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceLivenessTest.java b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceLivenessTest.java new file mode 100644 index 00000000..7dc467df --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/llm/service/ModelProviderServiceLivenessTest.java @@ -0,0 +1,188 @@ +package vip.mate.llm.service; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.llm.anthropic.oauth.ClaudeCodeOAuthService; +import vip.mate.llm.failover.AvailableProviderPool; +import vip.mate.llm.failover.ProviderHealthProperties; +import vip.mate.llm.failover.ProviderHealthTracker; +import vip.mate.llm.failover.ProviderInitProbe; +import vip.mate.llm.model.Liveness; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.model.ProviderInfoDTO; +import vip.mate.llm.repository.ModelProviderMapper; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * RFC-073: covers the five {@link Liveness} states surfaced through + * {@code listProviders()}. The five branches must remain orthogonal and + * mutually exclusive — the UI relies on it as a state machine. + * + *

Real {@link AvailableProviderPool} and {@link ProviderHealthTracker} + * (no Spring deps); {@link ProviderInitProbe} is a Mockito mock since its + * own constructor pulls the Spring context.

+ */ +class ModelProviderServiceLivenessTest { + + private ModelProviderMapper providerMapper; + private ModelConfigService modelConfigService; + private ApplicationEventPublisher eventPublisher; + private ObjectProvider claudeCodeOAuthProvider; + private AvailableProviderPool pool; + private ProviderHealthTracker healthTracker; + private ProviderInitProbe initProbe; + private ObjectProvider initProbeProvider; + + private ModelProviderService service; + + @BeforeEach + @SuppressWarnings("unchecked") + void setUp() { + providerMapper = mock(ModelProviderMapper.class); + modelConfigService = mock(ModelConfigService.class); + eventPublisher = mock(ApplicationEventPublisher.class); + claudeCodeOAuthProvider = mock(ObjectProvider.class); + when(claudeCodeOAuthProvider.getIfAvailable()).thenReturn(null); + pool = new AvailableProviderPool(); + // failure-threshold = 1 so a single recordFailure() trips cooldown deterministically. + ProviderHealthProperties props = new ProviderHealthProperties(); + props.setFailureThreshold(1); + healthTracker = new ProviderHealthTracker(props); + initProbe = mock(ProviderInitProbe.class); + initProbeProvider = mock(ObjectProvider.class); + when(initProbeProvider.getIfAvailable()).thenReturn(initProbe); + + service = new ModelProviderService(providerMapper, modelConfigService, eventPublisher, + claudeCodeOAuthProvider, pool, healthTracker, initProbeProvider); + } + + @Test + @DisplayName("LIVE: configured + probed + in pool + not in cooldown") + void liveProvider() { + seedProvider("openai", false); + when(initProbe.hasBeenProbed("openai")).thenReturn(true); + pool.add("openai"); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.LIVE, dto.getLiveness()); + assertNull(dto.getUnavailableReason()); + assertNull(dto.getCooldownRemainingMs()); + assertTrue(dto.getAvailable(), "available must be true when LIVE and has models"); + } + + @Test + @DisplayName("UNCONFIGURED: cloud provider with no api key — short-circuit before pool / probe checks") + void unconfiguredProvider() { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId("openai"); + p.setName("OpenAI"); + p.setIsLocal(false); + p.setIsCustom(false); + p.setRequireApiKey(true); + p.setApiKey(""); + p.setBaseUrl("https://api.openai.com/v1"); + when(providerMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(p)); + when(modelConfigService.listModels()).thenReturn(List.of()); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.UNCONFIGURED, dto.getLiveness()); + // Probe should not even be consulted for unconfigured providers. + verify(initProbe, never()).hasBeenProbed("openai"); + assertFalse(dto.getAvailable()); + } + + @Test + @DisplayName("UNPROBED: configured but probe hasn't fired yet (startup window)") + void unprobedProvider() { + seedProvider("ollama", true); + when(initProbe.hasBeenProbed("ollama")).thenReturn(false); + // pool intentionally empty — UNPROBED takes precedence over REMOVED so the UI + // can render skeletons during the startup window instead of false negatives. + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.UNPROBED, dto.getLiveness()); + assertFalse(dto.getAvailable()); + } + + @Test + @DisplayName("REMOVED: probed and HARD-removed — reason + lastProbedAtMs populated") + void removedProvider() { + seedProvider("openai", false); + when(initProbe.hasBeenProbed("openai")).thenReturn(true); + pool.remove("openai", AvailableProviderPool.RemovalSource.AUTH_ERROR, "401 Unauthorized"); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.REMOVED, dto.getLiveness()); + assertEquals("401 Unauthorized", dto.getUnavailableReason()); + assertNotNull(dto.getLastProbedAtMs()); + assertFalse(dto.getAvailable()); + } + + @Test + @DisplayName("COOLDOWN: in pool but tracker reports cooldown remaining") + void cooldownProvider() { + seedProvider("openai", false); + when(initProbe.hasBeenProbed("openai")).thenReturn(true); + pool.add("openai"); + // failure-threshold = 1 → one recorded failure trips cooldown immediately. + healthTracker.recordFailure("openai"); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.COOLDOWN, dto.getLiveness()); + assertNotNull(dto.getCooldownRemainingMs()); + assertTrue(dto.getCooldownRemainingMs() > 0); + assertFalse(dto.getAvailable(), "cooldown is not LIVE so available must be false"); + } + + @Test + @DisplayName("Probe-bean absent (test context with no init probe) → fall back to LIVE not UNPROBED") + void noProbeBeanFallsOpen() { + when(initProbeProvider.getIfAvailable()).thenReturn(null); + seedProvider("openai", false); + pool.add("openai"); + + ProviderInfoDTO dto = singleResult(); + assertEquals(Liveness.LIVE, dto.getLiveness(), + "no probe bean must not strand all providers in UNPROBED forever"); + } + + // ============================================================ + // Helpers + // ============================================================ + + /** Wire mapper / model service to return a single configured provider with one model. */ + private void seedProvider(String id, boolean local) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setIsLocal(local); + p.setIsCustom(false); + p.setRequireApiKey(!local); + p.setApiKey(local ? "" : "sk-test-key-1234567890"); + p.setBaseUrl(local ? "http://127.0.0.1:11434" : "https://api.example.com/v1"); + when(providerMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(List.of(p)); + + ModelConfigEntity m = new ModelConfigEntity(); + m.setProvider(id); + m.setModelName(id + "-model"); + m.setName(id + "-model"); + m.setBuiltin(true); + when(modelConfigService.listModels()).thenReturn(List.of(m)); + } + + private ProviderInfoDTO singleResult() { + List list = service.listProviders(); + assertEquals(1, list.size()); + return list.get(0); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/archive/MemoryArchiveServiceTest.java b/mateclaw-server/src/test/java/vip/mate/memory/archive/MemoryArchiveServiceTest.java new file mode 100644 index 00000000..73163686 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/archive/MemoryArchiveServiceTest.java @@ -0,0 +1,107 @@ +package vip.mate.memory.archive; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.memory.MemoryProperties; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * B.13 — MemoryArchiveService tests. + */ +@ExtendWith(MockitoExtension.class) +class MemoryArchiveServiceTest { + + @Mock private WorkspaceFileService workspaceFileService; + + private MemoryProperties props; + private MemoryArchiveService archiveService; + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + props.getDream().setArchiveEnabled(true); + props.getDream().setArchiveKeepDays(30); + archiveService = new MemoryArchiveService(workspaceFileService, props); + } + + @Test + @DisplayName("Flag off: archiveOldDreams is a no-op") + void flagOff_noOp() { + props.getDream().setArchiveEnabled(false); + archiveService.archiveOldDreams(1L); + verify(workspaceFileService, never()).saveFile(any(), any(), any()); + } + + @Test + @DisplayName("Empty DREAMS.md: nothing to archive") + void emptyDreams_noArchive() { + when(workspaceFileService.getFile(1L, "DREAMS.md")).thenReturn(null); + archiveService.archiveOldDreams(1L); + verify(workspaceFileService, never()).saveFile(any(), any(), any()); + } + + @Test + @DisplayName("All entries recent: nothing archived, DREAMS.md unchanged") + void allRecent_noArchive() { + String content = "# Dreaming 整合日记\n\n## 2099-01-01 03:00 Dreaming\n\nSome content\n"; + WorkspaceFileEntity file = new WorkspaceFileEntity(); + file.setContent(content); + when(workspaceFileService.getFile(1L, "DREAMS.md")).thenReturn(file); + + archiveService.archiveOldDreams(1L); + + // Only the DREAMS.md save should NOT happen since nothing was archived + verify(workspaceFileService, never()).saveFile(any(), any(), any()); + } + + @Test + @DisplayName("Old entries moved to monthly archive file") + void oldEntries_archived() { + String content = "# Dreaming 整合日记\n\n" + + "## 2020-01-15 03:00 Dreaming\n\nOld entry content\n\n" + + "## 2099-12-01 03:00 Dreaming\n\nRecent entry\n"; + WorkspaceFileEntity file = new WorkspaceFileEntity(); + file.setContent(content); + when(workspaceFileService.getFile(1L, "DREAMS.md")).thenReturn(file); + when(workspaceFileService.getFile(1L, "memory/dreams/2020-01.md")).thenReturn(null); + + archiveService.archiveOldDreams(1L); + + // Should save the archive file + ArgumentCaptor contentCaptor = ArgumentCaptor.forClass(String.class); + verify(workspaceFileService).saveFile(eq(1L), eq("memory/dreams/2020-01.md"), contentCaptor.capture()); + assertTrue(contentCaptor.getValue().contains("Old entry content")); + + // Should save updated DREAMS.md (only recent entry) + verify(workspaceFileService).saveFile(eq(1L), eq("DREAMS.md"), contentCaptor.capture()); + String updatedDreams = contentCaptor.getValue(); + assertTrue(updatedDreams.contains("Recent entry")); + assertFalse(updatedDreams.contains("Old entry content")); + } + + @Test + @DisplayName("Idempotent: second archive call on same content does not duplicate") + void idempotent_noDuplicate() { + // After first archive, DREAMS.md only has recent entries + String content = "# Dreaming 整合日记\n\n## 2099-12-01 03:00 Dreaming\n\nRecent\n"; + WorkspaceFileEntity file = new WorkspaceFileEntity(); + file.setContent(content); + when(workspaceFileService.getFile(1L, "DREAMS.md")).thenReturn(file); + + archiveService.archiveOldDreams(1L); + + // Nothing old to archive + verify(workspaceFileService, never()).saveFile(any(), any(), any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/controller/HilEditValidationTest.java b/mateclaw-server/src/test/java/vip/mate/memory/controller/HilEditValidationTest.java new file mode 100644 index 00000000..7d9f4967 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/controller/HilEditValidationTest.java @@ -0,0 +1,149 @@ +package vip.mate.memory.controller; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.memory.model.DreamReportEntity; +import vip.mate.memory.model.MemoryRecallEntity; +import vip.mate.memory.repository.DreamReportMapper; +import vip.mate.memory.repository.MemoryRecallMapper; +import vip.mate.memory.service.MemoryHilService; +import vip.mate.memory.service.MorningCardService; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * Tests for HiL edit API contract: + * - Report-scoped edit: key must belong to that report's entry set + * - Direct edit (reportId=0): key must be an existing MEMORY.md section + */ +@ExtendWith(MockitoExtension.class) +class HilEditValidationTest { + + @Mock private DreamReportMapper dreamReportMapper; + @Mock private MemoryRecallMapper recallMapper; + @Mock private MorningCardService morningCardService; + @Mock private MemoryHilService hilService; + @Mock private DreamEventBroadcaster eventBroadcaster; + + private DreamController controller; + + @BeforeEach + void setUp() { + controller = new DreamController(dreamReportMapper, recallMapper, + morningCardService, hilService, eventBroadcaster); + } + + @Test + @DisplayName("Report-scoped edit: key not in report's candidates → rejected") + void reportScopedEdit_keyNotInReport_rejected() { + // Setup: report exists and belongs to agent + DreamReportEntity report = new DreamReportEntity(); + report.setId(100L); + report.setAgentId(1L); + report.setStartedAt(LocalDateTime.of(2026, 4, 20, 3, 0)); + report.setFinishedAt(LocalDateTime.of(2026, 4, 20, 3, 5)); + report.setDeleted(0); + lenient().when(dreamReportMapper.selectOne(any())).thenReturn(report); + + // No recall entries match the key "unrelated_section" + MemoryRecallEntity candidate = new MemoryRecallEntity(); + candidate.setFilename("memory/2026-04-19.md#deployment_info"); + candidate.setLastRecalledAt(LocalDateTime.of(2026, 4, 20, 3, 2)); + candidate.setDeleted(0); + lenient().when(recallMapper.selectList(any())).thenReturn(List.of(candidate)); + + var result = controller.editEntry(1L, 100L, "unrelated_section", + Map.of("content", "hacked content")); + + // Should fail — key doesn't belong to this report + assertNotEquals(200, result.getCode()); + verify(hilService, never()).editMemoryEntry(any(), any(), any()); + } + + @Test + @DisplayName("Report-scoped edit: key matches report candidate → allowed") + void reportScopedEdit_keyInReport_allowed() { + DreamReportEntity report = new DreamReportEntity(); + report.setId(100L); + report.setAgentId(1L); + report.setStartedAt(LocalDateTime.of(2026, 4, 20, 3, 0)); + report.setFinishedAt(LocalDateTime.of(2026, 4, 20, 3, 5)); + report.setDeleted(0); + lenient().when(dreamReportMapper.selectOne(any())).thenReturn(report); + + // Recall entry filename contains the key + MemoryRecallEntity candidate = new MemoryRecallEntity(); + candidate.setFilename("MEMORY.md#deployment_info"); + candidate.setLastRecalledAt(LocalDateTime.of(2026, 4, 20, 3, 2)); + candidate.setDeleted(0); + lenient().when(recallMapper.selectList(any())).thenReturn(List.of(candidate)); + + var result = controller.editEntry(1L, 100L, "deployment_info", + Map.of("content", "updated content")); + + // Should succeed + assertEquals(200, result.getCode()); + verify(hilService).editMemoryEntry(eq(1L), eq("deployment_info"), eq("updated content")); + } + + @Test + @DisplayName("Report-scoped edit: substring of candidate key → rejected (exact match required)") + void reportScopedEdit_substringKey_rejected() { + DreamReportEntity report = new DreamReportEntity(); + report.setId(100L); + report.setAgentId(1L); + report.setStartedAt(LocalDateTime.of(2026, 4, 20, 3, 0)); + report.setFinishedAt(LocalDateTime.of(2026, 4, 20, 3, 5)); + report.setDeleted(0); + lenient().when(dreamReportMapper.selectOne(any())).thenReturn(report); + + MemoryRecallEntity candidate = new MemoryRecallEntity(); + candidate.setFilename("MEMORY.md#deployment_info"); + candidate.setLastRecalledAt(LocalDateTime.of(2026, 4, 20, 3, 2)); + candidate.setDeleted(0); + lenient().when(recallMapper.selectList(any())).thenReturn(List.of(candidate)); + + // "deployment" is a substring of "deployment_info" — must be rejected + var result = controller.editEntry(1L, 100L, "deployment", + Map.of("content", "content")); + + assertNotEquals(200, result.getCode()); + verify(hilService, never()).editMemoryEntry(any(), any(), any()); + } + + @Test + @DisplayName("Direct edit (reportId=0): existing section → allowed") + void directEdit_existingSection_allowed() { + when(hilService.sectionExists(1L, "stable_facts")).thenReturn(true); + + var result = controller.editEntry(1L, 0L, "stable_facts", + Map.of("content", "new content")); + + assertEquals(200, result.getCode()); + verify(hilService).editMemoryEntry(eq(1L), eq("stable_facts"), eq("new content")); + } + + @Test + @DisplayName("Direct edit (reportId=0): non-existing section → rejected") + void directEdit_nonExistingSection_rejected() { + when(hilService.sectionExists(1L, "ghost_section")).thenReturn(false); + + var result = controller.editEntry(1L, 0L, "ghost_section", + Map.of("content", "content")); + + assertNotEquals(200, result.getCode()); + verify(hilService, never()).editMemoryEntry(any(), any(), any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/fact/FactProjectionInvariantTest.java b/mateclaw-server/src/test/java/vip/mate/memory/fact/FactProjectionInvariantTest.java new file mode 100644 index 00000000..4776e0be --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/fact/FactProjectionInvariantTest.java @@ -0,0 +1,131 @@ +package vip.mate.memory.fact; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.memory.fact.extraction.ExtractedFact; +import vip.mate.memory.fact.extraction.PatternEntityExtractor; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * E1.6-E1.7: Core invariant guard tests for fact projection. + */ +class FactProjectionInvariantTest { + + private final PatternEntityExtractor extractor = new PatternEntityExtractor(); + + @Test + @DisplayName("Pattern extractor: KV bullet format → subject/predicate/object") + void patternExtractor_kvBullet() { + String content = """ + ## User Profile + - **user_name**: User's name is Xu Zhanfu. + - **role**: User works as a backend developer. + """; + List facts = extractor.extract(1L, "structured/user.md", content); + + assertTrue(facts.size() >= 2); + ExtractedFact nameFact = facts.stream() + .filter(f -> f.subject().equals("user_name")) + .findFirst().orElse(null); + assertNotNull(nameFact); + assertEquals("is", nameFact.predicate()); + assertTrue(nameFact.objectValue().contains("Xu Zhanfu")); + assertEquals("user_pref", nameFact.category()); + assertEquals("pattern", nameFact.extractedBy()); + } + + @Test + @DisplayName("Pattern extractor: sourceRef includes filename#slug") + void patternExtractor_sourceRef() { + String content = "- **preferred_language**: Chinese\n"; + List facts = extractor.extract(1L, "structured/user.md", content); + + assertFalse(facts.isEmpty()); + assertTrue(facts.get(0).sourceRef().startsWith("structured/user.md#")); + } + + @Test + @DisplayName("Pattern extractor: empty content returns empty list") + void patternExtractor_emptyContent() { + assertEquals(List.of(), extractor.extract(1L, "MEMORY.md", "")); + assertEquals(List.of(), extractor.extract(1L, "MEMORY.md", null)); + } + + @Test + @DisplayName("Pattern extractor: MEMORY.md general category") + void patternExtractor_memoryCategory() { + String content = "- **project_fact**: We use PostgreSQL 15\n"; + List facts = extractor.extract(1L, "MEMORY.md", content); + assertFalse(facts.isEmpty()); + assertEquals("general", facts.get(0).category()); + } + + @Test + @DisplayName("Pattern extractor: section heading extraction from structured files") + void patternExtractor_sectionHeading() { + String content = "## deployment_env\nProduction runs on Kubernetes with 3 replicas.\n\n## tech_stack\nSpring Boot 3.5 + Vue 3 + PostgreSQL 15\n"; + List facts = extractor.extract(1L, "structured/project.md", content); + assertTrue(facts.size() >= 1, "Should extract at least one section fact, got: " + facts); + } + + @Test + @DisplayName("Core invariant: extractedBy is always 'pattern' for PatternExtractor") + void coreInvariant_extractedByPattern() { + String content = "- **key**: value\n## section\ncontent here\n"; + List facts = extractor.extract(1L, "structured/user.md", content); + for (ExtractedFact f : facts) { + assertEquals("pattern", f.extractedBy(), + "PatternEntityExtractor must always set extractedBy='pattern'"); + } + } + + @Test + @DisplayName("Core invariant: confidence is in [0, 1] range") + void coreInvariant_confidenceRange() { + String content = "- **name**: test value\n## heading\nbody text content\n"; + List facts = extractor.extract(1L, "structured/user.md", content); + for (ExtractedFact f : facts) { + assertTrue(f.confidence() >= 0 && f.confidence() <= 1, + "Confidence must be in [0,1]: " + f.confidence()); + } + } + + @Test + @DisplayName("E1.6: rebuild after bumpUseCount preserves accumulated columns") + void rebuildAfterBumpUseCount_preservesAccumulatedColumns() { + // Invariant: FactProjectionBuilder.upsertDerived only writes derived columns. + // Accumulated columns (use_count, last_used_at) are set by bumpUseCount only. + // Verify: a new FactEntity from upsertDerived has useCount=0 (not overwritten). + var fact = new vip.mate.memory.fact.model.FactEntity(); + fact.setUseCount(42); + fact.setLastUsedAt(java.time.LocalDateTime.now()); + // After a hypothetical rebuild, derived columns change but accumulated must not + // This structural test verifies the entity has separate fields + fact.setSubject("new_subject"); + fact.setObjectValue("new_value"); + assertEquals(42, fact.getUseCount(), + "Accumulated column use_count must not be reset by derived column updates"); + assertNotNull(fact.getLastUsedAt(), + "Accumulated column last_used_at must not be nulled by derived column updates"); + } + + @Test + @DisplayName("E1.7: FactMapper has no direct insert/update for accumulated columns") + void factMapper_noDirectAccumulatedColumnWrite() { + // Structural: FactMapper should only expose bumpUseCount for accumulated writes. + // Check that the mapper interface has bumpUseCount method. + boolean hasBumpUseCount = false; + for (var method : vip.mate.memory.fact.repository.FactMapper.class.getDeclaredMethods()) { + if (method.getName().equals("bumpUseCount")) { + hasBumpUseCount = true; + } + // No method named "updateUseCount" or "setUseCount" should exist + assertFalse(method.getName().matches("updateUseCount|setUseCount|incrementUseCount"), + "FactMapper must not have direct accumulated column setter: " + method.getName()); + } + assertTrue(hasBumpUseCount, "FactMapper must have bumpUseCount method"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/integration/DreamV2AcceptanceIT.java b/mateclaw-server/src/test/java/vip/mate/memory/integration/DreamV2AcceptanceIT.java new file mode 100644 index 00000000..edeaa1be --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/integration/DreamV2AcceptanceIT.java @@ -0,0 +1,287 @@ +package vip.mate.memory.integration; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.archive.MemoryArchiveService; +import vip.mate.memory.model.DreamReportEntity; +import vip.mate.memory.model.MemoryRecallEntity; +import vip.mate.memory.repository.DreamReportMapper; +import vip.mate.memory.service.*; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import java.time.LocalDateTime; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * Dream v2 acceptance test — verifies the full consolidate pipeline + * with mocked LLM responses, covering: + * - DreamReport is returned with correct structure + * - DreamReport entity is persisted to DB (via mock mapper) + * - review_count is incremented for rejected candidates + * - FOCUSED mode uses topic-biased prompt + * - NIGHTLY mode produces report even with no candidates + * - Archive is triggered when flag is on + * + *

Uses mock LLM to avoid real API calls and token costs. + */ +@ExtendWith(MockitoExtension.class) +class DreamV2AcceptanceIT { + + @Mock private WorkspaceFileService workspaceFileService; + @Mock private ModelConfigService modelConfigService; + @Mock private AgentGraphBuilder agentGraphBuilder; + @Mock private MemoryRecallService recallService; + @Mock private DreamReportMapper dreamReportMapper; + @Mock private MemoryArchiveService archiveService; + @Mock private org.springframework.context.ApplicationEventPublisher eventPublisher; + @Mock private org.springframework.ai.chat.model.ChatModel chatModel; + + private MemoryProperties props; + private MemoryEmergenceService emergenceService; + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + props.setEmergenceEnabled(true); + props.setEmergenceDayRange(7); + props.setEmergenceScoreThreshold(0.4); + props.getDream().setFocusedEnabled(true); + props.getDream().setArchiveEnabled(false); + + emergenceService = new MemoryEmergenceService( + workspaceFileService, modelConfigService, agentGraphBuilder, + props, objectMapper, recallService, dreamReportMapper, archiveService, eventPublisher, null); + + // Mock model resolution + ModelConfigEntity modelConfig = new ModelConfigEntity(); + modelConfig.setProvider("mock"); + modelConfig.setModelName("mock-model"); + lenient().when(modelConfigService.getDefaultModel()).thenReturn(modelConfig); + lenient().when(agentGraphBuilder.buildRuntimeChatModel(any())).thenReturn(chatModel); + } + + private void setupDailyNotes(Long agentId) { + WorkspaceFileEntity note = new WorkspaceFileEntity(); + note.setFilename("memory/2026-04-19.md"); + note.setContent("## 工作记录\n- 讨论了国企信创选型\n- 等保三级对微服务架构的要求\n- CI/CD 推进受阻"); + when(workspaceFileService.listFiles(agentId)).thenReturn(List.of(note)); + when(workspaceFileService.getFile(agentId, "memory/2026-04-19.md")).thenReturn(note); + + WorkspaceFileEntity memoryFile = new WorkspaceFileEntity(); + memoryFile.setContent("## 长期记忆\n\n- 用户是央企开发工程师"); + lenient().when(workspaceFileService.getFile(agentId, "MEMORY.md")).thenReturn(memoryFile); + lenient().when(workspaceFileService.getFile(agentId, "DREAMS.md")).thenReturn(null); + } + + private void setupLlmResponse(String jsonResponse) { + var chatResponse = mock(org.springframework.ai.chat.model.ChatResponse.class); + var generation = mock(org.springframework.ai.chat.model.Generation.class); + var output = mock(org.springframework.ai.chat.messages.AssistantMessage.class); + when(chatModel.call(any(org.springframework.ai.chat.prompt.Prompt.class))).thenReturn(chatResponse); + when(chatResponse.getResult()).thenReturn(generation); + when(generation.getOutput()).thenReturn(output); + when(output.getText()).thenReturn(jsonResponse); + } + + private MemoryRecallEntity makeCandidate(Long id, String filename, double score) { + MemoryRecallEntity e = new MemoryRecallEntity(); + e.setId(id); + e.setAgentId(1L); + e.setFilename(filename); + e.setSnippetPreview("国企信创选型要求使用自主可控技术栈"); + e.setRecallCount(5); + e.setDailyCount(2); + e.setScore(score); + e.setReviewCount(0); + e.setLastRecalledAt(LocalDateTime.now()); + e.setPromoted(false); + return e; + } + + // ==================== Tests ==================== + + @Test + @DisplayName("NIGHTLY dream: returns SUCCESS report with promoted/rejected candidates") + void nightlyDream_successReport() { + setupDailyNotes(1L); + List candidates = List.of( + makeCandidate(100L, "memory/2026-04-19.md#信创", 0.85), + makeCandidate(101L, "memory/2026-04-19.md#CI/CD", 0.72) + ); + when(recallService.computeScores(1L)).thenReturn(candidates); + + // LLM adopts the first candidate content + setupLlmResponse(""" + {"should_update": true, "reason": "整合信创选型信息", + "memory_content": "## 长期记忆\\n\\n- 用户是央企开发工程师\\n- 国企信创选型要求使用自主可控技术栈"} + """); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + assertEquals(DreamStatus.SUCCESS, report.status()); + assertEquals(DreamMode.NIGHTLY, report.mode()); + assertNull(report.topic()); + assertEquals(2, report.candidateCount()); + assertTrue(report.promotedCount() >= 1); + assertNotNull(report.memoryDiff()); + + // DreamReport should be persisted + verify(dreamReportMapper).insert(any(DreamReportEntity.class)); + + // Rejected candidates should have review_count incremented + if (report.rejectedCount() > 0) { + verify(recallService).incrementReviewCounts(any()); + } + } + + @Test + @DisplayName("FOCUSED dream: topic appears in report and uses focused prompt") + void focusedDream_topicInReport() { + setupDailyNotes(1L); + when(recallService.computeScores(1L)).thenReturn(List.of( + makeCandidate(200L, "memory/2026-04-19.md#等保", 0.9) + )); + + setupLlmResponse(""" + {"should_update": true, "reason": "围绕等保合规整合", + "memory_content": "## 长期记忆\\n\\n- 等保三级要求加密传输、审计日志"} + """); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.FOCUSED, "等保合规要求"); + + assertEquals(DreamMode.FOCUSED, report.mode()); + assertEquals("等保合规要求", report.topic()); + assertEquals(DreamStatus.SUCCESS, report.status()); + verify(dreamReportMapper).insert(any(DreamReportEntity.class)); + } + + @Test + @DisplayName("LLM failure: returns FAILED report, persisted") + void llmFailure_failedReport() { + setupDailyNotes(1L); + when(recallService.computeScores(1L)).thenReturn(List.of()); + + doThrow(new RuntimeException("API timeout")) + .when(chatModel).call(any(org.springframework.ai.chat.prompt.Prompt.class)); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + assertEquals(DreamStatus.FAILED, report.status()); + assertNotNull(report.errorMessage()); + assertTrue(report.errorMessage().contains("API timeout")); + verify(dreamReportMapper).insert(any(DreamReportEntity.class)); + } + + @Test + @DisplayName("No daily notes: returns SKIPPED report") + void noDailyNotes_skippedReport() { + when(workspaceFileService.listFiles(1L)).thenReturn(List.of()); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.FOCUSED, "测试"); + + assertEquals(DreamStatus.SKIPPED, report.status()); + assertEquals("no daily notes", report.llmReason()); + verify(dreamReportMapper).insert(any(DreamReportEntity.class)); + } + + @Test + @DisplayName("Archive flag ON: archiveService called after dream diary") + void archiveOn_archiveCalled() { + props.getDream().setArchiveEnabled(true); + setupDailyNotes(1L); + + List candidates = List.of( + makeCandidate(300L, "memory/2026-04-19.md#总结", 0.8) + ); + when(recallService.computeScores(1L)).thenReturn(candidates); + + setupLlmResponse(""" + {"should_update": true, "reason": "ok", + "memory_content": "## 记忆\\n\\n- 国企信创选型要求使用自主可控技术栈"} + """); + + emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + verify(archiveService).archiveOldDreams(1L); + } + + @Test + @DisplayName("Archive flag OFF: archiveService NOT called, 20KB truncation preserved") + void archiveOff_noArchive() { + props.getDream().setArchiveEnabled(false); + setupDailyNotes(1L); + + List candidates = List.of( + makeCandidate(400L, "memory/2026-04-19.md#总结", 0.8) + ); + when(recallService.computeScores(1L)).thenReturn(candidates); + + setupLlmResponse(""" + {"should_update": true, "reason": "ok", + "memory_content": "## 记忆\\n\\n- 国企信创选型要求使用自主可控技术栈"} + """); + + emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + verify(archiveService, never()).archiveOldDreams(any()); + } + + @Test + @DisplayName("review_count: rejected candidates get incremented") + void reviewCount_rejected() { + setupDailyNotes(1L); + + // Two candidates: one will be adopted (content matches), one won't + MemoryRecallEntity adopted = makeCandidate(500L, "file-a.md", 0.9); + adopted.setSnippetPreview("信创选型要求使用自主可控"); + + MemoryRecallEntity rejected = makeCandidate(501L, "file-b.md", 0.7); + rejected.setSnippetPreview("完全不相关的内容xyz123"); + + when(recallService.computeScores(1L)).thenReturn(List.of(adopted, rejected)); + + // LLM output contains adopted candidate's key phrase + setupLlmResponse(""" + {"should_update": true, "reason": "整合", + "memory_content": "## 记忆\\n\\n- 信创选型要求使用自主可控技术栈"} + """); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + assertEquals(1, report.promotedCount()); + assertEquals(1, report.rejectedCount()); + + // Verify promoted was marked + verify(recallService).markPromoted(List.of(500L)); + // Verify rejected had review_count incremented + verify(recallService).incrementReviewCounts(List.of(501L)); + } + + @Test + @DisplayName("Emergence disabled: SKIPPED without LLM call") + void emergenceDisabled_skipped() { + props.setEmergenceEnabled(false); + + DreamReport report = emergenceService.consolidate(1L, DreamMode.NIGHTLY, null); + + assertEquals(DreamStatus.SKIPPED, report.status()); + verify(chatModel, never()).call(any(org.springframework.ai.chat.prompt.Prompt.class)); + verify(dreamReportMapper).insert(any(DreamReportEntity.class)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleFlagGuardTest.java b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleFlagGuardTest.java new file mode 100644 index 00000000..df3a354d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleFlagGuardTest.java @@ -0,0 +1,161 @@ +package vip.mate.memory.lifecycle; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.event.ConversationCompletedEvent; +import vip.mate.memory.spi.MemoryManager; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A.8 — Flag guard test: verifies that lifecycleMediatorEnabled=false + * means zero calls to prefetchAll / syncAll / onSessionEnd, and that + * enabling the flag activates all three. + * + *

Covers both AgentService helper paths (via Mediator) and + * MemoryLifecycleEventListener (via onConversationCompleted). + */ +@ExtendWith(MockitoExtension.class) +class LifecycleFlagGuardTest { + + @Mock private MemoryManager memoryManager; + @Mock private ApplicationEventPublisher eventPublisher; + + private MemoryProperties props; + private MemoryLifecycleMediator mediator; + private MemoryLifecycleEventListener listener; + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + mediator = new MemoryLifecycleMediator(memoryManager, eventPublisher); + listener = new MemoryLifecycleEventListener(mediator, props); + } + + // ==================== Flag OFF ==================== + + @Test + @DisplayName("Flag OFF: MemoryLifecycleEventListener.onConversationCompleted is a no-op") + void flagOff_listenerNoOp() { + props.setLifecycleMediatorEnabled(false); + + for (int i = 0; i < 10; i++) { + listener.onConversationCompleted( + new ConversationCompletedEvent(1L, "conv-" + i, "hello", "reply", 5, "web")); + } + + verify(memoryManager, never()).prefetchAll(any(), any()); + verify(memoryManager, never()).syncAll(any(), any(), any(), any()); + verify(memoryManager, never()).onSessionEnd(any(), any()); + } + + @Test + @DisplayName("Flag OFF: Mediator methods still work (called by AgentService helpers only when flag is on)") + void flagOff_mediatorDirectCallsStillWork() { + // Mediator itself has no flag check — that's AgentService's job. + // But MemoryLifecycleEventListener guards onSessionEnd. + props.setLifecycleMediatorEnabled(false); + + when(memoryManager.prefetchAll(eq(1L), eq("q"))).thenReturn(""); + + // Direct mediator call works (AgentService would not call this when flag is off) + mediator.beforeLlmCall(new TurnContext(1L, "c1", "s1", 1, "q")); + verify(memoryManager, times(1)).prefetchAll(1L, "q"); + } + + // ==================== Flag ON ==================== + + @Test + @DisplayName("Flag ON: beforeLlmCall invokes prefetchAll") + void flagOn_prefetchAll() { + props.setLifecycleMediatorEnabled(true); + when(memoryManager.prefetchAll(eq(1L), eq("hello"))).thenReturn(""); + + for (int i = 0; i < 10; i++) { + mediator.beforeLlmCall(new TurnContext(1L, "c1", "s1", i, "hello")); + } + + verify(memoryManager, times(10)).prefetchAll(1L, "hello"); + } + + @Test + @DisplayName("Flag ON: afterLlmCall invokes syncAll") + void flagOn_syncAll() { + props.setLifecycleMediatorEnabled(true); + + for (int i = 0; i < 10; i++) { + mediator.afterLlmCall(new TurnContext(1L, "c1", "s1", i, "hello"), "reply-" + i); + } + + verify(memoryManager, times(10)).syncAll(eq(1L), eq("c1"), eq("hello"), anyString()); + } + + @Test + @DisplayName("Flag ON: onConversationCompleted invokes onSessionEnd") + void flagOn_onSessionEnd() { + props.setLifecycleMediatorEnabled(true); + + for (int i = 0; i < 10; i++) { + listener.onConversationCompleted( + new ConversationCompletedEvent(1L, "conv-" + i, "hello", "reply", 5, "web")); + } + + verify(memoryManager, times(10)).onSessionEnd(eq(1L), anyString()); + } + + @Test + @DisplayName("Flag ON: cron conversations also trigger onSessionEnd") + void flagOn_cronConversation() { + props.setLifecycleMediatorEnabled(true); + + listener.onConversationCompleted( + new ConversationCompletedEvent(1L, "cron-conv", "task", "done", 2, "cron")); + + verify(memoryManager, times(1)).onSessionEnd(1L, "cron-conv"); + } + + // ==================== Provider exception degradation ==================== + + @Test + @DisplayName("Provider exception in prefetchAll degrades gracefully (returns empty)") + void prefetchException_graceful() { + when(memoryManager.prefetchAll(any(), any())).thenThrow(new RuntimeException("boom")); + + String result = mediator.beforeLlmCall(new TurnContext(1L, "c1", "s1", 1, "q")); + + // Should return empty string, not throw + assert result.isEmpty(); + } + + @Test + @DisplayName("Provider exception in syncAll degrades gracefully (no throw)") + void syncException_graceful() { + org.mockito.Mockito.doThrow(new RuntimeException("boom")) + .when(memoryManager).syncAll(any(), any(), any(), any()); + + // Should not throw + mediator.afterLlmCall(new TurnContext(1L, "c1", "s1", 1, "q"), "reply"); + } + + @Test + @DisplayName("Provider exception in onSessionEnd degrades gracefully (no throw)") + void sessionEndException_graceful() { + org.mockito.Mockito.doThrow(new RuntimeException("boom")) + .when(memoryManager).onSessionEnd(any(), any()); + + // Should not throw + mediator.onSessionEnd(1L, "c1"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleRecallCountIT.java b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleRecallCountIT.java new file mode 100644 index 00000000..f6f4e2bd --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/LifecycleRecallCountIT.java @@ -0,0 +1,141 @@ +package vip.mate.memory.lifecycle; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.agent.AgentService; +import vip.mate.agent.BaseAgent; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.service.MemoryRecallTracker; +import vip.mate.memory.spi.MemoryManager; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A.10 — F4 regression test: recall_count / daily_count must remain + * identical whether lifecycleMediatorEnabled is on or off. + * + *

Verifies that MemoryLifecycleMediator never calls trackRecalls, + * and AgentService calls trackRecalls exactly once per chat entry + * regardless of the flag state. + */ +@ExtendWith(MockitoExtension.class) +class LifecycleRecallCountIT { + + @Mock private AgentMapper agentMapper; + @Mock private AgentGraphBuilder agentGraphBuilder; + @Mock private MemoryRecallTracker memoryRecallTracker; + @Mock private MemoryManager memoryManager; + @Mock private ApplicationEventPublisher eventPublisher; + @Mock private BaseAgent mockAgent; + + private MemoryProperties props; + private AgentService agentService; + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + MemoryLifecycleMediator mediator = new MemoryLifecycleMediator(memoryManager, eventPublisher); + agentService = new AgentService(agentMapper, agentGraphBuilder, + memoryRecallTracker, mediator, props); + + // Stub agent resolution (lenient for structural-only tests) + AgentEntity entity = new AgentEntity(); + entity.setId(1L); + entity.setEnabled(true); + lenient().when(agentMapper.selectById(1L)).thenReturn(entity); + lenient().when(agentGraphBuilder.build(any(AgentEntity.class))).thenReturn(mockAgent); + lenient().when(mockAgent.chat(any(), any())).thenReturn("reply"); + } + + @Test + @DisplayName("F4 regression: flag OFF — trackRecalls called once per chat, mediator is silent") + void flagOff_trackRecallsOncePerChat() { + props.setLifecycleMediatorEnabled(false); + + for (int i = 0; i < 10; i++) { + agentService.chat(1L, "msg-" + i, "conv-1"); + } + + // trackRecalls: exactly 10 times (once per chat call) + verify(memoryRecallTracker, times(10)).trackRecalls(eq(1L), any()); + + // Mediator is not invoked when flag is off + verify(memoryManager, never()).prefetchAll(any(), any()); + verify(memoryManager, never()).syncAll(any(), any(), any(), any()); + } + + @Test + @DisplayName("F4 regression: flag ON — trackRecalls still called exactly once per chat (not doubled)") + void flagOn_trackRecallsStillOncePerChat() { + props.setLifecycleMediatorEnabled(true); + when(memoryManager.prefetchAll(any(), any())).thenReturn(""); + + for (int i = 0; i < 10; i++) { + agentService.chat(1L, "msg-" + i, "conv-1"); + } + + // trackRecalls: still exactly 10 times — NOT 20 (D4: mediator does not call trackRecalls) + verify(memoryRecallTracker, times(10)).trackRecalls(eq(1L), any()); + + // Mediator IS invoked + verify(memoryManager, times(10)).prefetchAll(eq(1L), any()); + verify(memoryManager, times(10)).syncAll(eq(1L), eq("conv-1"), any(), any()); + } + + @Test + @DisplayName("F4 regression: flag toggle does not change trackRecalls count") + void flagToggle_sameTrackRecallsCount() { + // 5 rounds with flag OFF + props.setLifecycleMediatorEnabled(false); + for (int i = 0; i < 5; i++) { + agentService.chat(1L, "off-" + i, "conv-1"); + } + + // 5 rounds with flag ON + props.setLifecycleMediatorEnabled(true); + when(memoryManager.prefetchAll(any(), any())).thenReturn(""); + for (int i = 0; i < 5; i++) { + agentService.chat(1L, "on-" + i, "conv-1"); + } + + // Total: 10 trackRecalls calls regardless of flag state + verify(memoryRecallTracker, times(10)).trackRecalls(eq(1L), any()); + + // Mediator only called for the ON rounds + verify(memoryManager, times(5)).prefetchAll(eq(1L), any()); + } + + @Test + @DisplayName("Mediator source code does not reference trackRecalls (structural guard)") + void mediator_noTrackRecallsReference() throws Exception { + // Structural assertion: MemoryLifecycleMediator has no field or method + // that references MemoryRecallTracker + var mediatorClass = MemoryLifecycleMediator.class; + for (var field : mediatorClass.getDeclaredFields()) { + if (field.getType().getSimpleName().contains("RecallTracker")) { + throw new AssertionError("Mediator must not depend on MemoryRecallTracker (D4)"); + } + } + // Also verify via declared constructor params + var ctorParams = mediatorClass.getDeclaredConstructors()[0].getParameterTypes(); + for (var param : ctorParams) { + if (param.getSimpleName().contains("RecallTracker")) { + throw new AssertionError("Mediator constructor must not accept MemoryRecallTracker (D4)"); + } + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/MemoryLifecycleMediatorTest.java b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/MemoryLifecycleMediatorTest.java new file mode 100644 index 00000000..db7a07c7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/lifecycle/MemoryLifecycleMediatorTest.java @@ -0,0 +1,155 @@ +package vip.mate.memory.lifecycle; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.memory.spi.MemoryManager; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * A.9 — Unit tests for MemoryLifecycleMediator covering: + * normal path, provider exception degradation, and onSessionEnd for cron conversations. + */ +@ExtendWith(MockitoExtension.class) +class MemoryLifecycleMediatorTest { + + @Mock private MemoryManager memoryManager; + @Mock private ApplicationEventPublisher eventPublisher; + + private MemoryLifecycleMediator mediator; + + @BeforeEach + void setUp() { + mediator = new MemoryLifecycleMediator(memoryManager, eventPublisher); + } + + // ==================== Normal path ==================== + + @Test + @DisplayName("beforeLlmCall returns prefetchAll result and publishes TurnStartedEvent") + void beforeLlmCall_normalPath() { + when(memoryManager.prefetchAll(eq(1L), eq("hello"))) + .thenReturn("some context"); + + TurnContext ctx = new TurnContext(1L, "c1", "s1", 1, "hello"); + String result = mediator.beforeLlmCall(ctx); + + assertEquals("some context", result); + verify(memoryManager).prefetchAll(1L, "hello"); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(Object.class); + verify(eventPublisher).publishEvent(eventCaptor.capture()); + assertTrue(eventCaptor.getValue() instanceof TurnStartedEvent); + assertEquals(ctx, ((TurnStartedEvent) eventCaptor.getValue()).context()); + } + + @Test + @DisplayName("beforeLlmCall returns empty string when prefetchAll returns empty") + void beforeLlmCall_emptyPrefetch() { + when(memoryManager.prefetchAll(any(), any())).thenReturn(""); + + String result = mediator.beforeLlmCall(new TurnContext(1L, "c1", "s1", 1, "q")); + + assertEquals("", result); + } + + @Test + @DisplayName("afterLlmCall calls syncAll and publishes TurnCompletedEvent") + void afterLlmCall_normalPath() { + TurnContext ctx = new TurnContext(1L, "c1", "s1", 1, "hello"); + mediator.afterLlmCall(ctx, "reply text"); + + verify(memoryManager).syncAll(1L, "c1", "hello", "reply text"); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(Object.class); + verify(eventPublisher).publishEvent(eventCaptor.capture()); + assertTrue(eventCaptor.getValue() instanceof TurnCompletedEvent); + TurnCompletedEvent event = (TurnCompletedEvent) eventCaptor.getValue(); + assertEquals(ctx, event.context()); + assertEquals("reply text", event.assistantReply()); + } + + @Test + @DisplayName("onSessionEnd delegates to memoryManager.onSessionEnd") + void onSessionEnd_normalPath() { + mediator.onSessionEnd(1L, "conv-123"); + + verify(memoryManager).onSessionEnd(1L, "conv-123"); + } + + // ==================== Provider exception degradation ==================== + + @Test + @DisplayName("beforeLlmCall degrades to empty string when prefetchAll throws") + void beforeLlmCall_exceptionDegrades() { + when(memoryManager.prefetchAll(any(), any())) + .thenThrow(new RuntimeException("provider down")); + + String result = mediator.beforeLlmCall(new TurnContext(1L, "c1", "s1", 1, "q")); + + assertEquals("", result); + } + + @Test + @DisplayName("afterLlmCall swallows syncAll exceptions") + void afterLlmCall_exceptionSwallowed() { + doThrow(new RuntimeException("sync failed")) + .when(memoryManager).syncAll(any(), any(), any(), any()); + + // Should not throw + mediator.afterLlmCall(new TurnContext(1L, "c1", "s1", 1, "q"), "reply"); + } + + @Test + @DisplayName("onSessionEnd swallows exceptions") + void onSessionEnd_exceptionSwallowed() { + doThrow(new RuntimeException("session end failed")) + .when(memoryManager).onSessionEnd(any(), any()); + + // Should not throw + mediator.onSessionEnd(1L, "c1"); + } + + // ==================== Cron conversations ==================== + + @Test + @DisplayName("onSessionEnd works the same for cron-triggered conversations") + void onSessionEnd_cronConversation() { + // onSessionEnd has no special handling for trigger source; + // that distinction only matters in PostConversationMemoryListener. + // The mediator processes all conversations equally. + mediator.onSessionEnd(42L, "cron-conv-001"); + + verify(memoryManager, times(1)).onSessionEnd(42L, "cron-conv-001"); + } + + // ==================== Reentrant / multi-turn ==================== + + @Test + @DisplayName("Multiple sequential turns do not interfere (Mediator is stateless)") + void multipleTurns_noInterference() { + when(memoryManager.prefetchAll(any(), any())).thenReturn(""); + + for (int i = 0; i < 5; i++) { + TurnContext ctx = new TurnContext(1L, "c1", "s1", i, "msg-" + i); + mediator.beforeLlmCall(ctx); + mediator.afterLlmCall(ctx, "reply-" + i); + } + + verify(memoryManager, times(5)).prefetchAll(eq(1L), any()); + verify(memoryManager, times(5)).syncAll(eq(1L), eq("c1"), any(), any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/service/DreamFlagGuardTest.java b/mateclaw-server/src/test/java/vip/mate/memory/service/DreamFlagGuardTest.java new file mode 100644 index 00000000..70325a4d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/service/DreamFlagGuardTest.java @@ -0,0 +1,111 @@ +package vip.mate.memory.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.archive.MemoryArchiveService; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * B.14 — Dream flag guard tests: verifies flag on/off behavior for + * focused-enabled and archive-enabled flags. + */ +@ExtendWith(MockitoExtension.class) +class DreamFlagGuardTest { + + @Mock private WorkspaceFileService workspaceFileService; + @Mock private MemoryArchiveService archiveService; + + private MemoryProperties props; + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + } + + @Test + @DisplayName("archive-enabled=false: archiveService.archiveOldDreams never called") + void archiveOff_noArchive() { + props.getDream().setArchiveEnabled(false); + MemoryArchiveService service = new MemoryArchiveService(workspaceFileService, props); + service.archiveOldDreams(1L); + verify(workspaceFileService, never()).saveFile(any(), any(), any()); + } + + @Test + @DisplayName("archive-enabled=true: archiveService.archiveOldDreams runs") + void archiveOn_runs() { + props.getDream().setArchiveEnabled(true); + props.getDream().setArchiveKeepDays(30); + MemoryArchiveService service = new MemoryArchiveService(workspaceFileService, props); + + // Set up old content + String content = "# Dreaming\n\n## 2020-01-01 03:00 Dreaming\n\nOld\n"; + WorkspaceFileEntity file = new WorkspaceFileEntity(); + file.setContent(content); + when(workspaceFileService.getFile(1L, "DREAMS.md")).thenReturn(file); + lenient().when(workspaceFileService.getFile(eq(1L), argThat(s -> s != null && s.startsWith("memory/dreams/")))).thenReturn(null); + + service.archiveOldDreams(1L); + + // Archive file should be written + verify(workspaceFileService, atLeastOnce()).saveFile(eq(1L), argThat(s -> s != null && s.contains("memory/dreams/")), any()); + } + + @Test + @DisplayName("focused-enabled flag is correctly read from DreamProperties") + void focusedEnabledFlag() { + props.getDream().setFocusedEnabled(false); + assertFalse(props.getDream().isFocusedEnabled()); + + props.getDream().setFocusedEnabled(true); + assertTrue(props.getDream().isFocusedEnabled()); + } + + @Test + @DisplayName("archive-enabled flag is correctly read from DreamProperties") + void archiveEnabledFlag() { + props.getDream().setArchiveEnabled(false); + assertFalse(props.getDream().isArchiveEnabled()); + + props.getDream().setArchiveEnabled(true); + assertTrue(props.getDream().isArchiveEnabled()); + } + + @Test + @DisplayName("DreamReport SKIPPED when emergence is disabled") + void emergenceDisabled_skipped() { + props.setEmergenceEnabled(false); + // Create a minimal service to test skipped report + MemoryEmergenceService service = new MemoryEmergenceService( + workspaceFileService, null, null, props, null, null, null, archiveService, null, null); + + DreamReport report = service.consolidate(1L, DreamMode.NIGHTLY, null); + assertEquals(DreamStatus.SKIPPED, report.status()); + assertEquals("emergence disabled", report.llmReason()); + } + + @Test + @DisplayName("DreamReport SKIPPED when no daily notes found") + void noDailyNotes_skipped() { + props.setEmergenceEnabled(true); + when(workspaceFileService.listFiles(1L)).thenReturn(List.of()); + + MemoryEmergenceService service = new MemoryEmergenceService( + workspaceFileService, null, null, props, null, null, null, archiveService, null, null); + + DreamReport report = service.consolidate(1L, DreamMode.FOCUSED, "test"); + assertEquals(DreamStatus.SKIPPED, report.status()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/service/MemorySummarizationGateTest.java b/mateclaw-server/src/test/java/vip/mate/memory/service/MemorySummarizationGateTest.java new file mode 100644 index 00000000..a0bab816 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/service/MemorySummarizationGateTest.java @@ -0,0 +1,139 @@ +package vip.mate.memory.service; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class MemorySummarizationGateTest { + + @Test + @DisplayName("skips conversations whose final assistant message is evidence_insufficient") + void skipsEvidenceInsufficientTurns() { + MessageEntity user = message("user", "分析 MateClaw 技能系统源码", null); + MessageEntity assistant = message("assistant", "SkillServiceImpl.java 负责业务。", + "{\"finishReason\":\"evidence_insufficient\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + assertTrue(decision.reason().contains("finishReason")); + } + + @Test + @DisplayName("skips evidence warning answers even when metadata does not carry finishReason") + void skipsEvidenceWarningContent() { + MessageEntity user = message("user", "分析系统设计", null); + MessageEntity assistant = message("assistant", + "结论如下。\n\n[证据不足] 以下源码引用未出现在已读取/搜索到的工具证据中:SkillServiceImpl.java。", + "{}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + assertTrue(decision.reason().contains("assistant content")); + } + + @Test + @DisplayName("skips one-off source analysis even when the assistant message is completed") + void skipsSourceAnalysisTasks() { + MessageEntity user = message("user", "请全面 review skill 技能功能源码,看看有哪些待修复内容", null); + MessageEntity assistant = message("assistant", "已分析 SkillController.java。", + "{\"finishReason\":\"normal\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + assertTrue(decision.reason().contains("source-analysis")); + } + + @Test + @DisplayName("allows explicit remember requests") + void allowsExplicitRememberRequests() { + MessageEntity user = message("user", "记住:这个项目后端默认用 MyBatis Plus 分页", null); + MessageEntity assistant = message("assistant", "已记录。", "{\"finishReason\":\"normal\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertTrue(decision.shouldAnalyze()); + } + + @Test + @DisplayName("skips incomplete turns once finishReason rides in metadata (regression for the lifecycle sink)") + void skipsIncompleteFinishReason() { + // Critical regression: the new INCOMPLETE fallback texts produced by the + // repetition / thinking-only soft caps do NOT match the text heuristic + // ("自动截断" is not in the heuristic list). Without finishReason in + // metadata they would silently leak into long-term memory. After the + // ReActLifecycleListener finishReasonSink wiring, INCOMPLETE rides in + // metadata and the gate skips on it. + MessageEntity user = message("user", "分析这段代码", null); + MessageEntity assistant = message("assistant", + "(模型输出被自动截断且未产出可见内容,请重试。)", + "{\"finishReason\":\"incomplete\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + assertTrue(decision.reason().contains("incomplete"), + "reason must surface the actual finishReason for log/debug"); + } + + @Test + @DisplayName("skips stopped turns based on finishReason metadata") + void skipsStoppedFinishReason() { + MessageEntity user = message("user", "做一个表格", null); + MessageEntity assistant = message("assistant", "已停止生成的部分内容…", + "{\"finishReason\":\"stopped\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + } + + @Test + @DisplayName("skips error_fallback turns based on finishReason metadata") + void skipsErrorFallbackFinishReason() { + // Even when the visible content does not include "error_fallback" verbatim, + // metadata-based detection short-circuits the text heuristic. + MessageEntity user = message("user", "做点事", null); + MessageEntity assistant = message("assistant", "[错误] 认证失败: Invalid API Key", + "{\"finishReason\":\"error_fallback\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertFalse(decision.shouldAnalyze()); + } + + @Test + @DisplayName("return_direct turns are eligible (tool-direct outputs are durable)") + void allowsReturnDirectFinishReason() { + MessageEntity user = message("user", "随便聊聊", null); + MessageEntity assistant = message("assistant", "工具直接返回的内容。", + "{\"finishReason\":\"return_direct\"}"); + + MemorySummarizationGate.Decision decision = + MemorySummarizationGate.evaluate(List.of(user, assistant)); + + assertTrue(decision.shouldAnalyze(), + "return_direct represents a successful tool-driven answer; should reach analysis"); + } + + private static MessageEntity message(String role, String content, String metadata) { + MessageEntity entity = new MessageEntity(); + entity.setRole(role); + entity.setContent(content); + entity.setMetadata(metadata); + return entity; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/memory/service/SoulSummarizerServiceTest.java b/mateclaw-server/src/test/java/vip/mate/memory/service/SoulSummarizerServiceTest.java new file mode 100644 index 00000000..fcab98e0 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/memory/service/SoulSummarizerServiceTest.java @@ -0,0 +1,126 @@ +package vip.mate.memory.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.memory.MemoryProperties; +import vip.mate.memory.event.MemoryWriteEvent; +import vip.mate.workspace.document.WorkspaceFileService; +import vip.mate.workspace.document.model.WorkspaceFileEntity; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * C.8 — Tests for SoulSummarizerService: K-accumulate trigger + SOUL update. + */ +@ExtendWith(MockitoExtension.class) +class SoulSummarizerServiceTest { + + @Mock private WorkspaceFileService workspaceFileService; + @Mock private ModelConfigService modelConfigService; + @Mock private AgentGraphBuilder agentGraphBuilder; + @Mock private org.springframework.ai.chat.model.ChatModel chatModel; + + private MemoryProperties props; + private SoulSummarizerService service; + + @BeforeEach + void setUp() { + props = new MemoryProperties(); + service = new SoulSummarizerService(workspaceFileService, modelConfigService, + agentGraphBuilder, props); + } + + @Test + @DisplayName("soulUpdateInterval=0: no SOUL update triggered") + void intervalZero_noUpdate() { + props.setSoulUpdateInterval(0); + for (int i = 0; i < 100; i++) { + service.onMemoryWrite(new MemoryWriteEvent(1L, "MEMORY.md", "consolidate", "content")); + } + verify(workspaceFileService, never()).saveFile(eq(1L), eq("SOUL.md"), any()); + } + + @Test + @DisplayName("soulUpdateInterval=5: first 4 writes are no-op, 5th triggers update") + void interval5_triggersOn5th() { + props.setSoulUpdateInterval(5); + + // Mock LLM for when it triggers + ModelConfigEntity model = new ModelConfigEntity(); + model.setProvider("mock"); + lenient().when(modelConfigService.getDefaultModel()).thenReturn(model); + lenient().when(agentGraphBuilder.buildRuntimeChatModel(any())).thenReturn(chatModel); + + var chatResponse = mock(org.springframework.ai.chat.model.ChatResponse.class); + var generation = mock(org.springframework.ai.chat.model.Generation.class); + var output = mock(org.springframework.ai.chat.messages.AssistantMessage.class); + lenient().when(chatModel.call(any(org.springframework.ai.chat.prompt.Prompt.class))).thenReturn(chatResponse); + lenient().when(chatResponse.getResult()).thenReturn(generation); + lenient().when(generation.getOutput()).thenReturn(output); + lenient().when(output.getText()).thenReturn("_Updated SOUL content that is longer than 50 chars to pass the length check._"); + + // Mock file reads + WorkspaceFileEntity soulFile = new WorkspaceFileEntity(); + soulFile.setContent("old soul"); + lenient().when(workspaceFileService.getFile(1L, "SOUL.md")).thenReturn(soulFile); + lenient().when(workspaceFileService.getFile(1L, "MEMORY.md")).thenReturn(soulFile); + lenient().when(workspaceFileService.getFile(1L, "PROFILE.md")).thenReturn(soulFile); + + // First 4 writes: no SOUL update + for (int i = 0; i < 4; i++) { + service.onMemoryWrite(new MemoryWriteEvent(1L, "MEMORY.md", "remember", "c" + i)); + } + verify(workspaceFileService, never()).saveFile(eq(1L), eq("SOUL.md"), any()); + + // 5th write: triggers SOUL update + service.onMemoryWrite(new MemoryWriteEvent(1L, "structured/user.md", "remember", "c4")); + verify(workspaceFileService, times(1)).saveFile(eq(1L), eq("SOUL.md"), any()); + } + + @Test + @DisplayName("Counter resets after trigger: needs another K writes for next update") + void counterResets_afterTrigger() { + props.setSoulUpdateInterval(3); + + ModelConfigEntity model = new ModelConfigEntity(); + lenient().when(modelConfigService.getDefaultModel()).thenReturn(model); + lenient().when(agentGraphBuilder.buildRuntimeChatModel(any())).thenReturn(chatModel); + + var chatResponse = mock(org.springframework.ai.chat.model.ChatResponse.class); + var generation = mock(org.springframework.ai.chat.model.Generation.class); + var output = mock(org.springframework.ai.chat.messages.AssistantMessage.class); + lenient().when(chatModel.call(any(org.springframework.ai.chat.prompt.Prompt.class))).thenReturn(chatResponse); + lenient().when(chatResponse.getResult()).thenReturn(generation); + lenient().when(generation.getOutput()).thenReturn(output); + lenient().when(output.getText()).thenReturn("New SOUL content with enough length to pass the fifty character minimum threshold check."); + + WorkspaceFileEntity file = new WorkspaceFileEntity(); + file.setContent("content"); + lenient().when(workspaceFileService.getFile(eq(1L), any())).thenReturn(file); + + // Trigger 1st update at write #3 + for (int i = 0; i < 3; i++) { + service.onMemoryWrite(new MemoryWriteEvent(1L, "MEMORY.md", "remember", "x")); + } + verify(workspaceFileService, times(1)).saveFile(eq(1L), eq("SOUL.md"), any()); + + // Next 2 writes: no update yet + for (int i = 0; i < 2; i++) { + service.onMemoryWrite(new MemoryWriteEvent(1L, "MEMORY.md", "remember", "y")); + } + verify(workspaceFileService, times(1)).saveFile(eq(1L), eq("SOUL.md"), any()); + + // 3rd write after reset: triggers 2nd update + service.onMemoryWrite(new MemoryWriteEvent(1L, "MEMORY.md", "remember", "z")); + verify(workspaceFileService, times(2)).saveFile(eq(1L), eq("SOUL.md"), any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerListEnabledTest.java b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerListEnabledTest.java new file mode 100644 index 00000000..76f87522 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerListEnabledTest.java @@ -0,0 +1,145 @@ +package vip.mate.skill.controller; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.common.result.R; +import vip.mate.skill.acp.AcpSkillBridge; +import vip.mate.skill.mcp.McpSkillBridge; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.service.SkillService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Focused unit coverage for the bridge merge in + * {@link SkillController#listEnabled()} — ensures the agent picker's + * skill-list endpoint shows MCP/ACP virtual skills, mirrors the shadow + * rule used by the paginated {@code /skills} endpoint, and never 500s + * when a bridge throws. + */ +class SkillControllerListEnabledTest { + + private SkillService skillService; + private McpSkillBridge mcpSkillBridge; + private AcpSkillBridge acpSkillBridge; + private SkillController controller; + + @BeforeEach + void setUp() { + // Only the four collaborators reachable from listEnabled() need real + // mocks; the rest are nulls because the method never touches them. + skillService = mock(SkillService.class); + mcpSkillBridge = mock(McpSkillBridge.class); + acpSkillBridge = mock(AcpSkillBridge.class); + controller = new SkillController( + skillService, + /* skillRuntimeService */ null, + /* workspaceManager */ null, + /* bundledSkillSyncer */ null, + /* skillFileSyncer */ null, + /* synthesisService */ null, + /* dependencyChecker */ null, + /* lessonsService */ null, + /* agentSkillBindingMapper */ null, + /* agentService */ null, + /* agentBindingService */ null, + mcpSkillBridge, + acpSkillBridge); + // listSkills() supplies realSkillNames() for shadow base — default + // to empty so each test can override. + when(skillService.listSkills()).thenReturn(List.of()); + when(skillService.listEnabledSkills()).thenReturn(List.of()); + when(mcpSkillBridge.listMcpDerivedSkillEntities()).thenReturn(List.of()); + when(acpSkillBridge.listAcpDerivedSkillEntities()).thenReturn(List.of()); + } + + @Test + @DisplayName("listEnabled merges MCP virtual skills into the response") + void includesMcpVirtualSkills() { + SkillEntity mcp = skill("github", "mcp"); + when(mcpSkillBridge.listMcpDerivedSkillEntities()).thenReturn(List.of(mcp)); + + R> response = controller.listEnabled(); + + assertNotNull(response.getData()); + assertTrue(response.getData().stream().anyMatch(s -> "github".equals(s.getName())), + "expected the MCP virtual skill 'github' in the response"); + } + + @Test + @DisplayName("listEnabled merges ACP virtual skills into the response") + void includesAcpVirtualSkills() { + SkillEntity acp = skill("claude-code", "acp"); + when(acpSkillBridge.listAcpDerivedSkillEntities()).thenReturn(List.of(acp)); + + R> response = controller.listEnabled(); + + assertTrue(response.getData().stream().anyMatch(s -> "claude-code".equals(s.getName()))); + } + + @Test + @DisplayName("a same-name real skill that is DISABLED still shadows the virtual MCP twin") + void disabledRealSkillShadowsVirtualTwin() { + // realSkillNames() pulls from listSkills() (all rows, regardless of + // enabled). If listEnabled() derived its shadow base from listEnabledSkills() + // (enabled-only) by mistake, the virtual would slip through here. + SkillEntity disabledReal = skill("github", "custom"); + disabledReal.setEnabled(false); + when(skillService.listSkills()).thenReturn(List.of(disabledReal)); + when(skillService.listEnabledSkills()).thenReturn(List.of()); + + SkillEntity virtualMcp = skill("github", "mcp"); + when(mcpSkillBridge.listMcpDerivedSkillEntities()).thenReturn(List.of(virtualMcp)); + + R> response = controller.listEnabled(); + + // The real skill is disabled, so listEnabledSkills() returns nothing; + // the virtual MCP must also be filtered to keep this endpoint in step + // with the management page. + assertEquals(0, response.getData().size(), + "disabled real skill should still suppress the virtual twin in /enabled"); + } + + @Test + @DisplayName("MCP bridge failure does not 500 the response") + void mcpBridgeFailureSwallowed() { + SkillEntity enabled = skill("web_search", "builtin"); + enabled.setEnabled(true); + when(skillService.listEnabledSkills()).thenReturn(List.of(enabled)); + when(mcpSkillBridge.listMcpDerivedSkillEntities()) + .thenThrow(new RuntimeException("MCP bridge offline")); + + R> response = controller.listEnabled(); + + assertEquals(1, response.getData().size()); + assertEquals("web_search", response.getData().get(0).getName()); + } + + @Test + @DisplayName("ACP bridge failure does not 500 the response and MCP results still merge") + void acpBridgeFailureSwallowedMcpStillMerged() { + when(acpSkillBridge.listAcpDerivedSkillEntities()) + .thenThrow(new RuntimeException("ACP discovery failed")); + SkillEntity mcp = skill("github", "mcp"); + when(mcpSkillBridge.listMcpDerivedSkillEntities()).thenReturn(List.of(mcp)); + + R> response = controller.listEnabled(); + + assertTrue(response.getData().stream().anyMatch(s -> "github".equals(s.getName()))); + } + + private static SkillEntity skill(String name, String type) { + SkillEntity s = new SkillEntity(); + s.setName(name); + s.setSkillType(type); + s.setEnabled(true); + return s; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualGuardTest.java b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualGuardTest.java new file mode 100644 index 00000000..4366574b --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualGuardTest.java @@ -0,0 +1,82 @@ +package vip.mate.skill.controller; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.exception.MateClawException; +import vip.mate.skill.acp.AcpSkillBridge; +import vip.mate.skill.mcp.McpSkillBridge; +import vip.mate.skill.model.SkillEntity; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Mutation paths refuse virtual MCP/ACP skill ids upfront so the user + * gets a clear redirect to the connection page instead of the previous + * "技能不存在" 500 surfacing from a doomed mate_skill lookup. + */ +class SkillControllerVirtualGuardTest { + + private final SkillController controller = new SkillController( + null, null, null, null, null, null, null, null, null, null, null, null, null); + + @Test + @DisplayName("update on a virtual MCP skill id is rejected before hitting the service") + void updateRejectsVirtualMcpId() { + long virtualId = McpSkillBridge.VIRTUAL_ID_BASE + 42L; + MateClawException ex = assertThrows(MateClawException.class, + () -> controller.update(virtualId, new SkillEntity())); + assertTrue(ex.getMessage().contains("MCP/ACP"), + "expected redirect-to-connection-page hint, got: " + ex.getMessage()); + } + + @Test + @DisplayName("update on a virtual ACP skill id is rejected before hitting the service") + void updateRejectsVirtualAcpId() { + long virtualAcpId = AcpSkillBridge.VIRTUAL_ID_BASE + 7L; + // Sanity guard against the test's own arithmetic — any drift in + // bridge layout should fail the test loudly here, not silently + // pass elsewhere. + assertTrue(AcpSkillBridge.isVirtualAcpSkillId(virtualAcpId), + "test fixture id is not in ACP virtual range; ACP base layout changed?"); + assertThrows(MateClawException.class, + () -> controller.update(virtualAcpId, new SkillEntity())); + } + + @Test + @DisplayName("delete / toggle / rescan all reject virtual ids the same way") + void mutationFamilyAllGuarded() { + long virtualId = McpSkillBridge.VIRTUAL_ID_BASE + 42L; + assertThrows(MateClawException.class, () -> controller.delete(virtualId)); + assertThrows(MateClawException.class, () -> controller.toggle(virtualId, true)); + assertThrows(MateClawException.class, () -> controller.rescan(virtualId)); + } + + @Test + @DisplayName("real skill ids fall through to the service (no false-positive guard)") + void realIdNotGuarded() { + // A Snowflake-shaped id below VIRTUAL_ID_BASE — should pass the + // guard. The downstream service call will fail because we're + // passing nulls, but the failure must be from the service layer, + // not the guard. + SkillController real = new SkillController( + mock(vip.mate.skill.service.SkillService.class), + null, null, null, null, null, null, null, null, null, null, null, null); + long snowflakeId = 1_900_000_001_000_000_902L; + // updateSkill on a mocked SkillService returns null without throwing, + // which is fine — we just need to confirm the guard didn't fire. + // A virtual-id call would have thrown MateClawException before + // reaching the service. + try { + real.update(snowflakeId, new SkillEntity()); + } catch (MateClawException e) { + // The guard message contains "MCP/ACP"; any other MateClawException + // (e.g. from the service layer) is acceptable. + org.junit.jupiter.api.Assertions.assertFalse(e.getMessage().contains("MCP/ACP"), + "real id incorrectly treated as virtual: " + e.getMessage()); + } catch (Exception ignored) { + // Service-layer failures are out of scope for this test. + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualMergeTest.java b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualMergeTest.java new file mode 100644 index 00000000..cd26956e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerVirtualMergeTest.java @@ -0,0 +1,74 @@ +package vip.mate.skill.controller; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.model.SkillEntity; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SkillControllerVirtualMergeTest { + + @Test + @DisplayName("virtual MCP rows shadowed by real skills are not merged into list") + void virtualRowsShadowedByRealSkillAreFiltered() { + SkillEntity virtualMcp = skill("ckjia-shopping", "mcp"); + SkillEntity github = skill("github", "mcp"); + + List filtered = SkillController.filterShadowedVirtualSkills( + List.of(virtualMcp, github), + Set.of("ckjia-shopping")); + + assertEquals(List.of(github), filtered); + } + + @Test + @DisplayName("virtual count excludes rows shadowed by real skills") + void virtualCountExcludesShadowedRows() { + SkillEntity virtualMcp = skill("ckjia-shopping", "mcp"); + SkillEntity github = skill("github", "mcp"); + + long count = SkillController.countUnshadowedVirtualSkills( + List.of(virtualMcp, github), + Set.of("ckjia-shopping")); + + assertEquals(1L, count); + } + + @Test + @DisplayName("virtual rows are appended after the DB page window") + void virtualRowsDoNotDisplaceFirstDbPage() { + List dbRecords = List.of( + skill("apple-notes", "builtin"), + skill("arxiv", "builtin")); + SkillEntity claudeCode = skill("claude-code", "acp"); + + SkillController.VirtualPageMergeResult merged = SkillController.mergeVirtualTailPageRecords( + dbRecords, List.of(claudeCode), 50, 1, 10); + + assertEquals(51L, merged.total()); + assertEquals(dbRecords, merged.records()); + } + + @Test + @DisplayName("virtual rows fill the tail page after DB records are exhausted") + void virtualRowsFillTailPage() { + SkillEntity claudeCode = skill("claude-code", "acp"); + + SkillController.VirtualPageMergeResult merged = SkillController.mergeVirtualTailPageRecords( + List.of(), List.of(claudeCode), 50, 6, 10); + + assertEquals(51L, merged.total()); + assertEquals(List.of(claudeCode), merged.records()); + } + + private static SkillEntity skill(String name, String type) { + SkillEntity s = new SkillEntity(); + s.setName(name); + s.setSkillType(type); + s.setEnabled(true); + return s; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/installer/BuiltinSkillSeedServiceTest.java b/mateclaw-server/src/test/java/vip/mate/skill/installer/BuiltinSkillSeedServiceTest.java new file mode 100644 index 00000000..721e9b02 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/installer/BuiltinSkillSeedServiceTest.java @@ -0,0 +1,252 @@ +package vip.mate.skill.installer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.runtime.SkillFrontmatterParser; + +import java.lang.reflect.Method; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@link BuiltinSkillSeedService}. Deliberately avoids Mockito + * so the suite runs on every JDK/OS combination (Windows + JDK 21 + inline + * byte-buddy self-attach is flaky). The merge / build helpers are exercised + * directly via reflection with parsed frontmatter; the mapper is never + * touched, so {@code null} is safe. + */ +class BuiltinSkillSeedServiceTest { + + private BuiltinSkillSeedService service; + private SkillFrontmatterParser parser; + + @BeforeEach + void setUp() { + parser = new SkillFrontmatterParser(); + // Mapper stays null: none of the tests below go through syncBuiltinSkills() + // — they drive the private buildNew / mergeIntoExisting helpers directly. + service = new BuiltinSkillSeedService(null, parser, new ObjectMapper()); + } + + @Test + @DisplayName("New skill: insert with frontmatter values + sensible defaults") + void insertsNewSkillWithDefaults() throws Exception { + String md = """ + --- + name: my_skill + version: "2.1.0" + description: "Pretend skill for testing." + dependencies: + tools: + - read_file + --- + # body + """; + + SkillEntity built = invokeBuildNew(md); + + assertEquals("my_skill", built.getName()); + assertEquals("2.1.0", built.getVersion()); + assertEquals("Pretend skill for testing.", built.getDescription()); + assertEquals("builtin", built.getSkillType()); + assertEquals(Boolean.TRUE, built.getBuiltin()); + assertEquals(Boolean.TRUE, built.getEnabled()); + assertEquals("MateClaw", built.getAuthor(), "default author"); + assertEquals("🛠️", built.getIcon(), "default icon"); + assertEquals("my_skill", built.getTags(), "default tag = name"); + assertNotNull(built.getSkillContent()); + assertTrue(built.getSkillContent().contains("# body")); + assertTrue(built.getConfigJson().contains("\"requiredTools\""), "tools deps should land in configJson"); + } + + @Test + @DisplayName("Existing skill: frontmatter wins for declared fields, DB values preserved otherwise") + void mergeKeepsDbFieldsWhenFrontmatterSilent() throws Exception { + SkillEntity existing = new SkillEntity(); + existing.setId(1000000001L); + existing.setName("cron"); + existing.setDescription("OLD"); + existing.setVersion("1.0.0"); + existing.setIcon("⏰"); + existing.setTags("cron,schedule"); + existing.setAuthor("MateClaw"); + existing.setSkillType("builtin"); + existing.setBuiltin(true); + existing.setSkillContent("OLD CONTENT"); + existing.setConfigJson("{\"upstream\":\"mateclaw\",\"entryFile\":\"SKILL.md\"}"); + + String md = """ + --- + name: cron + version: "1.4.0" + description: "NEW description" + --- + # cron body + """; + + boolean dirty = invokeMerge(existing, md); + + assertTrue(dirty); + assertEquals("1.4.0", existing.getVersion(), "version updated from frontmatter"); + assertEquals("NEW description", existing.getDescription(), "description updated"); + // Frontmatter omitted these — DB values preserved: + assertEquals("⏰", existing.getIcon(), "icon preserved when frontmatter silent"); + assertEquals("cron,schedule", existing.getTags(), "tags preserved when frontmatter silent"); + assertEquals("MateClaw", existing.getAuthor(), "author preserved when frontmatter silent"); + // skill_content always re-syncs from bundled SKILL.md: + assertTrue(existing.getSkillContent().contains("# cron body")); + } + + @Test + @DisplayName("Existing skill: idempotent — second pass with identical frontmatter is a no-op") + void mergeIsIdempotent() throws Exception { + SkillEntity existing = new SkillEntity(); + existing.setName("cron"); + existing.setVersion("1.4.0"); + existing.setDescription("Same desc."); + existing.setSkillType("builtin"); + existing.setBuiltin(true); + existing.setIcon("⏰"); + existing.setTags("cron"); + existing.setAuthor("MateClaw"); + // The configJson the service produces for this frontmatter (no tools, no platforms) + existing.setConfigJson("{\"upstream\":\"mateclaw\",\"entryFile\":\"SKILL.md\"}"); + String md = """ + --- + name: cron + version: "1.4.0" + description: "Same desc." + --- + # body + """; + existing.setSkillContent(md); + + assertFalse(invokeMerge(existing, md), "no fields should change on second pass"); + } + + @Test + @DisplayName("Frontmatter tags as YAML list serialize to CSV") + void tagsListSerializesToCsv() throws Exception { + String md = """ + --- + name: my_skill + tags: + - alpha + - beta + - gamma + --- + """; + SkillEntity built = invokeBuildNew(md); + assertEquals("alpha,beta,gamma", built.getTags()); + } + + @Test + @DisplayName("Frontmatter `optional: true` seeds the row as enabled=false") + void optionalFrontmatterSeedsAsDisabled() throws Exception { + String md = """ + --- + name: heavy_skill + description: "Needs paid API + manual OAuth — ship dark." + optional: true + --- + # body + """; + + SkillEntity built = invokeBuildNew(md); + + assertEquals("heavy_skill", built.getName()); + assertEquals(Boolean.TRUE, built.getBuiltin(), "still a builtin row"); + assertEquals(Boolean.FALSE, built.getEnabled(), + "optional: true must flip the initial enabled to false"); + } + + @Test + @DisplayName("Frontmatter absent / false defaults to enabled=true (back-compat)") + void defaultRemainsEnabled() throws Exception { + // Frontmatter doesn't mention `optional` → current behavior preserved. + SkillEntity defaultCase = invokeBuildNew(""" + --- + name: lightweight_skill + --- + # body + """); + assertEquals(Boolean.TRUE, defaultCase.getEnabled()); + + // Explicit `optional: false` is equivalent. + SkillEntity explicitFalse = invokeBuildNew(""" + --- + name: lightweight_too + optional: false + --- + # body + """); + assertEquals(Boolean.TRUE, explicitFalse.getEnabled()); + } + + @Test + @DisplayName("mergeIntoExisting leaves `enabled` alone so user toggles aren't clobbered by frontmatter") + void mergeNeverFlipsEnabled() throws Exception { + // User installed an optional skill (enabled=false at seed time), then + // turned it on from the UI. Subsequent boots must not silently turn + // it back off just because the frontmatter still says optional: true. + SkillEntity existing = new SkillEntity(); + existing.setName("heavy_skill"); + existing.setDescription("Needs paid API + manual OAuth — ship dark."); + existing.setSkillType("builtin"); + existing.setBuiltin(true); + existing.setIcon("🛠️"); + existing.setTags("heavy_skill"); + existing.setAuthor("MateClaw"); + existing.setEnabled(true); // user activated it + existing.setConfigJson("{\"upstream\":\"mateclaw\",\"entryFile\":\"SKILL.md\"}"); + String md = """ + --- + name: heavy_skill + description: "Needs paid API + manual OAuth — ship dark." + optional: true + --- + # body + """; + existing.setSkillContent(md); + + invokeMerge(existing, md); + assertEquals(Boolean.TRUE, existing.getEnabled(), + "merge must never override a user-toggled enabled flag"); + } + + @Test + @DisplayName("Frontmatter without `name` is skipped — never inserts a nameless row") + void skippedWhenNameMissing() { + // Empty frontmatter and a namespace clash both produce an empty `name`. + SkillFrontmatterParser.ParsedSkillMd empty = parser.parse("# only body, no frontmatter"); + assertEquals("", empty.getName()); + // Nothing to assert against the mock — buildNew shouldn't be called when + // the orchestrator sees an empty name. We're just locking the contract + // that getName() returns "" for malformed input so the orchestrator's + // guard works. + } + + // ==================== reflection helpers ==================== + // These two private methods are the load-bearing logic; we test them + // directly to keep the suite fast (no DB) and focused. + + private SkillEntity invokeBuildNew(String md) throws Exception { + SkillFrontmatterParser.ParsedSkillMd parsed = parser.parse(md); + Method m = BuiltinSkillSeedService.class.getDeclaredMethod( + "buildNew", SkillFrontmatterParser.ParsedSkillMd.class, String.class); + m.setAccessible(true); + return (SkillEntity) m.invoke(service, parsed, md); + } + + private boolean invokeMerge(SkillEntity existing, String md) throws Exception { + SkillFrontmatterParser.ParsedSkillMd parsed = parser.parse(md); + Method m = BuiltinSkillSeedService.class.getDeclaredMethod( + "mergeIntoExisting", SkillEntity.class, + SkillFrontmatterParser.ParsedSkillMd.class, String.class); + m.setAccessible(true); + return (boolean) m.invoke(service, existing, parsed, md); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/installer/SkillHubClientTest.java b/mateclaw-server/src/test/java/vip/mate/skill/installer/SkillHubClientTest.java new file mode 100644 index 00000000..223b083c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/installer/SkillHubClientTest.java @@ -0,0 +1,224 @@ +package vip.mate.skill.installer; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.installer.model.HubSkillInfo; +import vip.mate.skill.installer.model.SkillBundle; +import vip.mate.skill.runtime.SkillFrontmatterParser; + +import java.io.ByteArrayOutputStream; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Regression tests for the ClawHub schema mismatch reported in GitHub issue #42. + *

+ * The hub's actual JSON shape uses {@code displayName} / {@code summary} + * and nests skill metadata under a {@code skill} key, with the SKILL.md + * content delivered separately as a ZIP via {@code /api/v1/download}. The + * earlier client expected a flat {@code {name, description, content}} JSON, + * which made search results render blank and every install fail with + * "empty content; treat as failure". These tests pin the parsing. + */ +class SkillHubClientTest { + + private static SkillHubClient newClient() { + SkillHubProperties props = new SkillHubProperties(); + return new SkillHubClient(props, new ObjectMapper(), new SkillFrontmatterParser()); + } + + @Test + @DisplayName("Search: clawhub.ai response shape — displayName→name, summary→description") + void searchMapsHubFieldsToHubSkillInfo() throws Exception { + // Verbatim shape from https://clawhub.ai/api/v1/search?q=feishu-room-booking + String body = """ + { + "results": [ + { + "score": 2.87, + "slug": "feishu-room-booking", + "displayName": "Feishu Room Booking", + "summary": "Book meeting rooms on Feishu/Lark.", + "version": null, + "updatedAt": 1777359717617 + } + ] + } + """; + + @SuppressWarnings("unchecked") + List parsed = (List) invokePrivate( + newClient(), "parseSearchResponse", new Class[]{String.class}, body); + + assertEquals(1, parsed.size()); + HubSkillInfo info = parsed.get(0); + assertEquals("feishu-room-booking", info.getSlug()); + assertEquals("Feishu Room Booking", info.getName(), + "displayName must populate name (was blank in the bug report)"); + assertEquals("Book meeting rooms on Feishu/Lark.", info.getDescription(), + "summary must populate description"); + } + + @Test + @DisplayName("Search: legacy flat shape with name/description still works") + void searchAcceptsLegacyShape() throws Exception { + String body = """ + { + "results": [ + { + "slug": "x", + "name": "Legacy Name", + "description": "Legacy description" + } + ] + } + """; + @SuppressWarnings("unchecked") + List parsed = (List) invokePrivate( + newClient(), "parseSearchResponse", new Class[]{String.class}, body); + assertEquals(1, parsed.size()); + assertEquals("Legacy Name", parsed.get(0).getName()); + assertEquals("Legacy description", parsed.get(0).getDescription()); + } + + @Test + @DisplayName("Metadata: nested {skill, latestVersion, owner} shape extracts all fields") + void metadataExtractsNestedFields() throws Exception { + // Verbatim shape from https://clawhub.ai/api/v1/skills/feishu-room-booking + String body = """ + { + "skill": { + "slug": "feishu-room-booking", + "displayName": "Feishu Room Booking", + "summary": "Book meeting rooms on Feishu." + }, + "latestVersion": { + "version": "2.9.0", + "license": "MIT-0" + }, + "owner": { + "handle": "qiushibang", + "displayName": "qiushibang" + } + } + """; + + Object metadata = invokePrivate(newClient(), "parseMetadataResponse", new Class[]{String.class}, body); + assertNotNull(metadata, "Nested metadata must parse successfully"); + + // Use reflection on the record to verify all four fields land. + assertEquals("Feishu Room Booking", recordField(metadata, "displayName")); + assertEquals("Book meeting rooms on Feishu.", recordField(metadata, "summary")); + assertEquals("2.9.0", recordField(metadata, "version")); + assertEquals("qiushibang", recordField(metadata, "owner")); + } + + @Test + @DisplayName("Bundle ZIP extraction: SKILL.md frontmatter wins, references/scripts have no prefix in keys") + void zipExtractStoresKeysWithoutPrefix() throws Exception { + byte[] zip = buildZipBundle(); + ZipSkillFetcher.ExtractedSkill extracted = ZipSkillFetcher.extract(new java.io.ByteArrayInputStream(zip)); + + assertTrue(extracted.skillMdContent().contains("name: feishu-room-booking")); + // Keys must be relative to references/ and scripts/ — installers prepend the prefix themselves. + assertTrue(extracted.references().containsKey("rooms.json"), + "expected 'rooms.json' (no 'references/' prefix), got: " + extracted.references().keySet()); + assertTrue(extracted.scripts().containsKey("query.py"), + "expected 'query.py' (no 'scripts/' prefix), got: " + extracted.scripts().keySet()); + assertEquals("{\"a\":1}", extracted.references().get("rooms.json")); + assertEquals("print('hi')\n", extracted.scripts().get("query.py")); + } + + @Test + @DisplayName("Bundle ZIP missing SKILL.md throws IllegalArgumentException") + void zipExtractRequiresSkillMd() throws Exception { + byte[] zip; + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(out)) { + zos.putNextEntry(new ZipEntry("scripts/query.py")); + zos.write("print('hi')\n".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + zos.finish(); + zip = out.toByteArray(); + } + assertThrows(IllegalArgumentException.class, + () -> ZipSkillFetcher.extract(new java.io.ByteArrayInputStream(zip))); + } + + // ==================== helpers ==================== + + private static byte[] buildZipBundle() throws Exception { + try (ByteArrayOutputStream out = new ByteArrayOutputStream(); + ZipOutputStream zos = new ZipOutputStream(out)) { + String md = """ + --- + name: feishu-room-booking + description: Book meeting rooms. + version: "2.9.0" + --- + body + """; + zos.putNextEntry(new ZipEntry("SKILL.md")); + zos.write(md.getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + + zos.putNextEntry(new ZipEntry("references/rooms.json")); + zos.write("{\"a\":1}".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + + zos.putNextEntry(new ZipEntry("scripts/query.py")); + zos.write("print('hi')\n".getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + + zos.finish(); + return out.toByteArray(); + } + } + + /** Round-trip a SkillBundle assembly purely via the data we'd get from the hub. */ + @Test + @DisplayName("End-to-end shape: bundle assembled from ZIP + metadata has non-empty content") + void assembledBundleHasNonEmptyContent() throws Exception { + byte[] zip = buildZipBundle(); + ZipSkillFetcher.ExtractedSkill extracted = ZipSkillFetcher.extract(new java.io.ByteArrayInputStream(zip)); + SkillFrontmatterParser parser = new SkillFrontmatterParser(); + var parsed = parser.parse(extracted.skillMdContent()); + + SkillBundle bundle = new SkillBundle( + parsed.getName(), + extracted.skillMdContent(), + extracted.references(), + extracted.scripts(), + "clawhub", + "https://clawhub.ai/skills/feishu-room-booking@2.9.0", + "2.9.0", + parsed.getDescription(), + "qiushibang", + "📦" + ); + + // The original bug rejected bundles with bundle.content().isBlank(). + assertNotNull(bundle.content()); + assertFalse(bundle.content().isBlank(), "content must be non-empty so installer doesn't reject as failure"); + assertEquals("feishu-room-booking", bundle.name()); + assertEquals("2.9.0", bundle.version()); + } + + private static Object invokePrivate(Object target, String name, Class[] sig, Object... args) throws Exception { + Method m = target.getClass().getDeclaredMethod(name, sig); + m.setAccessible(true); + return m.invoke(target, args); + } + + private static Object recordField(Object record, String fieldName) throws Exception { + Method accessor = record.getClass().getDeclaredMethod(fieldName); + accessor.setAccessible(true); + return accessor.invoke(record); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java b/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java new file mode 100644 index 00000000..9f720692 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java @@ -0,0 +1,207 @@ +package vip.mate.skill.installer; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Regression tests for {@link ZipSkillFetcher#extract}. + * + *

The original single-pass extractor depended on SKILL.md being seen + * before any {@code scripts/} or {@code references/} entry, so packaging + * tools that emitted entries in a different order silently dropped scripts. + * Issue #104 hit this with {@code tencent-meeting-mcp.zip}: the zip's + * scripts streamed first and were never persisted, leaving the installed + * skill unable to run. The two-pass extractor must classify entries + * regardless of order. + */ +class ZipSkillFetcherTest { + + private static final String SKILL_MD = """ + --- + name: tencent-meeting + description: Test + version: 1.0.0 + --- + # Test skill + """; + + private record Entry(String name, String content) {} + + private static byte[] zipOf(List entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos, StandardCharsets.UTF_8)) { + for (Entry e : entries) { + zos.putNextEntry(new ZipEntry(e.name())); + zos.write(e.content().getBytes(StandardCharsets.UTF_8)); + zos.closeEntry(); + } + } + return baos.toByteArray(); + } + + @Test + @DisplayName("scripts emitted BEFORE SKILL.md (issue #104) are still classified") + void extractsScriptsEvenWhenTheyComeBeforeSkillMd() throws IOException { + byte[] zip = zipOf(List.of( + new Entry("tencent-meeting-mcp/scripts/run.py", "print('hi')\n"), + new Entry("tencent-meeting-mcp/scripts/helper.py", "x = 1\n"), + new Entry("tencent-meeting-mcp/references/notes.md", "# notes\n"), + new Entry("tencent-meeting-mcp/SKILL.md", SKILL_MD) + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertNotNull(ex.skillMdContent()); + assertEquals(2, ex.scripts().size(), + "Both scripts must survive even though they preceded SKILL.md"); + assertEquals("print('hi')\n", ex.scripts().get("run.py")); + assertEquals("x = 1\n", ex.scripts().get("helper.py")); + assertEquals(1, ex.references().size()); + assertEquals("# notes\n", ex.references().get("notes.md")); + } + + @Test + @DisplayName("scripts emitted AFTER SKILL.md still work (no regression)") + void extractsScriptsWhenSkillMdComesFirst() throws IOException { + byte[] zip = zipOf(List.of( + new Entry("pkg/SKILL.md", SKILL_MD), + new Entry("pkg/scripts/run.py", "print('after')\n"), + new Entry("pkg/references/cfg.md", "cfg\n") + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(1, ex.scripts().size()); + assertEquals("print('after')\n", ex.scripts().get("run.py")); + assertEquals(1, ex.references().size()); + } + + @Test + @DisplayName("SKILL.md at zip root: scripts in same root level still classify correctly") + void extractsWhenSkillMdAtRoot() throws IOException { + byte[] zip = zipOf(List.of( + new Entry("scripts/a.py", "a"), + new Entry("scripts/sub/b.py", "b"), + new Entry("references/r.md", "r"), + new Entry("SKILL.md", SKILL_MD) + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(2, ex.scripts().size()); + assertEquals("a", ex.scripts().get("a.py")); + assertEquals("b", ex.scripts().get("sub/b.py")); + assertEquals(1, ex.references().size()); + } + + @Test + @DisplayName("Missing SKILL.md still throws") + void rejectsZipWithoutSkillMd() throws IOException { + byte[] zip = zipOf(List.of(new Entry("scripts/run.py", "x"))); + assertThrows(IllegalArgumentException.class, + () -> ZipSkillFetcher.extract(new ByteArrayInputStream(zip))); + } + + @Test + @DisplayName("Nested entries outside scripts/ and references/ are dropped (no extension fallback)") + void ignoresNestedNoiseEntries() throws IOException { + // README inside the wrapper dir is unclear (could be docs vs install + // instructions) — strict mode wins here. Only root-level files get + // the extension fallback. + byte[] zip = zipOf(List.of( + new Entry("pkg/SKILL.md", SKILL_MD), + new Entry("pkg/docs/extra.md", "ignored"), + new Entry("pkg/scripts/run.py", "x"), + new Entry("pkg/.git/HEAD", "ref: refs/heads/main") + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(Map.of("run.py", "x"), ex.scripts()); + assertTrue(ex.references().isEmpty()); + } + + @Test + @DisplayName("Real-world tencent layout: setup.sh at zip root → classified as script") + void rootLevelSetupShIsClassifiedAsScript() throws IOException { + // Verbatim shape of the official tencent-meeting-mcp.zip: + // setup.sh + // references/api_references.md + // SKILL.md + // setup.sh sits at the zip root, not under scripts/. Without the + // extension fallback the skill installs with an empty scripts/ + // and SKILL.md's `bash setup.sh` instruction goes nowhere. + byte[] zip = zipOf(List.of( + new Entry("setup.sh", "#!/bin/bash\necho hello\n"), + new Entry("references/api_references.md", "# api docs"), + new Entry("SKILL.md", SKILL_MD) + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(1, ex.scripts().size(), + "setup.sh at zip root should land in scripts via extension fallback"); + assertEquals("#!/bin/bash\necho hello\n", ex.scripts().get("setup.sh")); + assertEquals(1, ex.references().size()); + assertEquals("# api docs", ex.references().get("api_references.md")); + } + + @Test + @DisplayName("Root-level README.md is auto-classified into references/") + void rootLevelMarkdownGoesToReferences() throws IOException { + byte[] zip = zipOf(List.of( + new Entry("SKILL.md", SKILL_MD), + new Entry("README.md", "# top-level readme"), + new Entry("config.yaml", "key: value\n") + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(2, ex.references().size()); + assertEquals("# top-level readme", ex.references().get("README.md")); + assertEquals("key: value\n", ex.references().get("config.yaml")); + assertTrue(ex.scripts().isEmpty()); + } + + @Test + @DisplayName("Root-level file with unknown extension is still dropped (with WARN)") + void rootLevelUnknownExtensionStillDropped() throws IOException { + byte[] zip = zipOf(List.of( + new Entry("SKILL.md", SKILL_MD), + new Entry("mystery.bin", "binary blob") + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertTrue(ex.scripts().isEmpty()); + assertTrue(ex.references().isEmpty()); + } + + @Test + @DisplayName("Root-level fallback also works when SKILL.md is in a wrapper dir") + void rootLevelFallbackWorksAfterPrefixStrip() throws IOException { + // pkg/setup.sh becomes "setup.sh" after prefix strip, so the same + // fallback rules apply — packagers shouldn't have to choose between + // "wrap everything" and "use a sub-script-dir". + byte[] zip = zipOf(List.of( + new Entry("pkg/setup.sh", "#!/bin/sh\n"), + new Entry("pkg/SKILL.md", SKILL_MD) + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + assertEquals(1, ex.scripts().size()); + assertEquals("#!/bin/sh\n", ex.scripts().get("setup.sh")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/knowledge/AcpSkillWrapperToolFactoryTest.java b/mateclaw-server/src/test/java/vip/mate/skill/knowledge/AcpSkillWrapperToolFactoryTest.java new file mode 100644 index 00000000..4d68d78d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/knowledge/AcpSkillWrapperToolFactoryTest.java @@ -0,0 +1,123 @@ +package vip.mate.skill.knowledge; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.ToolCallback; +import vip.mate.acp.model.AcpEndpointEntity; +import vip.mate.acp.service.AcpDelegationService; +import vip.mate.acp.service.AcpEndpointService; +import vip.mate.skill.manifest.SkillManifest; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.*; + +/** + * RFC-090 Phase 7b — locks in the wrapper factory contract: + * + *

    + *
  1. name shape is {@code acp___prompt}
  2. + *
  3. missing endpoint → empty list (resolver downgrades skill to + * SETUP_NEEDED rather than register broken tools)
  4. + *
  5. resolveEndpointId hits {@link AcpEndpointService#findByName} + * and returns the row id when present
  6. + *
  7. callback delegates to {@link AcpDelegationService#prompt} and + * bakes in the manifest's {@code system_prefix}
  8. + *
  9. empty input → JSON error (caller can decide what to do)
  10. + *
+ */ +class AcpSkillWrapperToolFactoryTest { + + private AcpEndpointService endpointService; + private AcpDelegationService delegationService; + private AcpSkillWrapperToolFactory factory; + + @BeforeEach + void setUp() { + endpointService = mock(AcpEndpointService.class); + delegationService = mock(AcpDelegationService.class); + factory = new AcpSkillWrapperToolFactory( + endpointService, delegationService, new ObjectMapper()); + } + + @Test + @DisplayName("wrapperNames returns the canonical acp___prompt shape") + void wrapperNamesShape() { + SkillManifest m = SkillManifest.builder() + .name("Team-Codex Helper") // mixed case + dash + .acp(SkillManifest.AcpBinding.builder().endpoint("codex").build()) + .build(); + List names = factory.wrapperNames(m); + assertEquals(1, names.size()); + assertEquals("acp_codex_team_codex_helper_prompt", names.get(0)); + } + + @Test + @DisplayName("buildWrappers returns empty when no acp binding") + void buildWrappersNoBinding() { + SkillManifest m = SkillManifest.builder().name("foo").build(); + assertTrue(factory.buildWrappers(m).isEmpty()); + } + + @Test + @DisplayName("resolveEndpointId hits findByName and returns id") + void resolveEndpointIdLooksUpName() { + AcpEndpointEntity ep = new AcpEndpointEntity(); + ep.setId(42L); + when(endpointService.findByName("codex")).thenReturn(ep); + assertEquals(42L, factory.resolveEndpointId("codex")); + verify(endpointService).findByName("codex"); + } + + @Test + @DisplayName("resolveEndpointId returns null for missing endpoint") + void resolveEndpointIdMissing() { + when(endpointService.findByName("ghost")).thenReturn(null); + assertNull(factory.resolveEndpointId("ghost")); + } + + @Test + @DisplayName("callback delegates to AcpDelegationService and prepends system_prefix") + void callbackDelegates() { + SkillManifest m = SkillManifest.builder() + .name("codex-helper") + .acp(SkillManifest.AcpBinding.builder() + .endpoint("codex") + .systemPrefix("Be concise.") + .cwd("/tmp/proj") + .build()) + .build(); + when(delegationService.prompt(eq("codex"), any(String.class), eq("/tmp/proj"))) + .thenReturn("DONE"); + + List wrappers = factory.buildWrappers(m); + assertEquals(1, wrappers.size()); + String out = wrappers.get(0).call("{\"prompt\":\"hello\"}"); + assertTrue(out.contains("\"reply\"")); + assertTrue(out.contains("DONE")); + + // Composed prompt should carry system_prefix + blank line + user text. + verify(delegationService).prompt(eq("codex"), + argThat((String s) -> s.contains("Be concise.") && s.contains("hello")), + eq("/tmp/proj")); + } + + @Test + @DisplayName("callback returns JSON error when prompt is empty") + void callbackEmptyPromptError() { + SkillManifest m = SkillManifest.builder() + .name("codex-helper") + .acp(SkillManifest.AcpBinding.builder().endpoint("codex").build()) + .build(); + List wrappers = factory.buildWrappers(m); + String out = wrappers.get(0).call("{}"); + assertTrue(out.contains("\"error\"")); + verifyNoInteractions(delegationService); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/knowledge/WikiSkillWrapperToolFactoryTest.java b/mateclaw-server/src/test/java/vip/mate/skill/knowledge/WikiSkillWrapperToolFactoryTest.java new file mode 100644 index 00000000..95ee9b19 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/knowledge/WikiSkillWrapperToolFactoryTest.java @@ -0,0 +1,187 @@ +package vip.mate.skill.knowledge; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.tool.ToolCallback; +import vip.mate.skill.manifest.SkillManifest; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.service.HybridRetriever; +import vip.mate.wiki.service.WikiKnowledgeBaseService; +import vip.mate.wiki.service.WikiPageService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.*; + +/** + * RFC-090 §14.4 — locks in wrapper factory contract for type=knowledge. + * + *
    + *
  1. wrapperNames produces the canonical {@code kb__*} triple
  2. + *
  3. resolveKbId tries numeric id first, then name match
  4. + *
  5. buildWrappers returns 3 callbacks (search / read / list) with + * a captured kbId — the LLM never sees the kbId in the schema
  6. + *
  7. The search wrapper delegates to {@code HybridRetriever.search} + * and trackReference fires per result
  8. + *
  9. read wrapper truncates content via maxChars
  10. + *
  11. list wrapper hides system pages (RFC-051 PR-2 parity)
  12. + *
+ */ +class WikiSkillWrapperToolFactoryTest { + + private WikiKnowledgeBaseService kbService; + private WikiPageService pageService; + private HybridRetriever retriever; + private WikiSkillWrapperToolFactory factory; + + @BeforeEach + void setUp() { + kbService = mock(WikiKnowledgeBaseService.class); + pageService = mock(WikiPageService.class); + retriever = mock(HybridRetriever.class); + factory = new WikiSkillWrapperToolFactory( + kbService, pageService, retriever, new ObjectMapper()); + } + + @Test + @DisplayName("wrapperNames returns search/read/list triple with sanitized slug") + void wrapperNamesShape() { + SkillManifest m = SkillManifest.builder() + .name("TCM-Classics") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("ignored").build()) + .build(); + List names = factory.wrapperNames(m); + assertEquals(List.of("kb_tcm_classics_search", "kb_tcm_classics_read", "kb_tcm_classics_list"), names); + } + + @Test + @DisplayName("resolveKbId tries numeric id parse first") + void resolveKbIdNumeric() { + assertEquals(42L, factory.resolveKbId("42")); + verifyNoInteractions(kbService); + } + + @Test + @DisplayName("resolveKbId falls back to name match (case-insensitive)") + void resolveKbIdByName() { + WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity(); + kb.setId(7L); + kb.setName("TCM Classics"); + when(kbService.listAll()).thenReturn(List.of(kb)); + assertEquals(7L, factory.resolveKbId("tcm classics")); + } + + @Test + @DisplayName("resolveKbId returns null for missing slug + missing name") + void resolveKbIdMissing() { + when(kbService.listAll()).thenReturn(List.of()); + assertNull(factory.resolveKbId("nope")); + } + + @Test + @DisplayName("buildWrappers returns empty when manifest has no knowledge binding") + void buildWrappersNoBinding() { + SkillManifest m = SkillManifest.builder().name("foo").build(); + assertTrue(factory.buildWrappers(m, 1L).isEmpty()); + } + + @Test + @DisplayName("buildWrappers returns 3 callbacks: search / read / list") + void buildWrappersThreeCallbacks() { + SkillManifest m = SkillManifest.builder() + .name("tcm") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("tcm").build()) + .build(); + List wrappers = factory.buildWrappers(m, 99L); + assertEquals(3, wrappers.size()); + assertEquals("kb_tcm_search", wrappers.get(0).getToolDefinition().name()); + assertEquals("kb_tcm_read", wrappers.get(1).getToolDefinition().name()); + assertEquals("kb_tcm_list", wrappers.get(2).getToolDefinition().name()); + } + + @Test + @DisplayName("search wrapper passes captured kbId to HybridRetriever and tracks references") + void searchDelegatesAndTracks() { + SkillManifest m = SkillManifest.builder() + .name("tcm") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("tcm").build()) + .build(); + when(retriever.search(eq(99L), anyString(), anyString(), anyInt())) + .thenReturn(List.of(vip.mate.wiki.dto.PageSearchResult.of( + "shanghan-lun", "伤寒论", "summary", "snippet", List.of(), "matched", 0.9))); + ToolCallback search = factory.buildWrappers(m, 99L).get(0); + String out = search.call("{\"query\":\"小柴胡\",\"mode\":\"hybrid\",\"topK\":3}"); + assertTrue(out.contains("\"kbId\":99")); + assertTrue(out.contains("shanghan-lun")); + verify(retriever).search(99L, "小柴胡", "hybrid", 3); + verify(pageService).trackReference(99L, "shanghan-lun"); + } + + @Test + @DisplayName("search wrapper rejects empty query with JSON error") + void searchRejectsEmptyQuery() { + SkillManifest m = SkillManifest.builder() + .name("tcm") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("tcm").build()) + .build(); + ToolCallback search = factory.buildWrappers(m, 99L).get(0); + String out = search.call("{}"); + assertTrue(out.contains("\"error\"")); + verifyNoInteractions(retriever); + } + + @Test + @DisplayName("read wrapper truncates content to maxChars") + void readTruncatesContent() { + WikiPageEntity page = new WikiPageEntity(); + page.setSlug("a"); + page.setTitle("A"); + page.setVersion(2); + // Build a long content; the wrapper should chop to maxChars + "...(truncated)" suffix. + StringBuilder body = new StringBuilder(); + for (int i = 0; i < 100; i++) body.append("line ").append(i).append('\n'); + page.setContent(body.toString()); + when(pageService.getBySlug(99L, "a")).thenReturn(page); + + SkillManifest m = SkillManifest.builder() + .name("tcm") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("tcm").build()) + .build(); + ToolCallback read = factory.buildWrappers(m, 99L).get(1); + String out = read.call("{\"slug\":\"a\",\"maxChars\":40}"); + // Truncation suffix is "...(truncated)" appended to the content + // body, then JSON-escaped. Look for the inline marker rather + // than a top-level field — wrapper doesn't surface a flag. + assertTrue(out.contains("(truncated)"), + "expected truncation marker in content; got: " + out); + verify(pageService).trackReference(99L, "a"); + } + + @Test + @DisplayName("list wrapper filters out system pages") + void listFiltersSystemPages() { + WikiPageEntity normal = new WikiPageEntity(); + normal.setSlug("a"); normal.setTitle("A"); normal.setSummary("aa"); + normal.setPageType("page"); + WikiPageEntity system = new WikiPageEntity(); + system.setSlug("overview"); system.setTitle("Overview"); system.setSummary("ov"); + system.setPageType("system"); + when(pageService.listSummaries(99L)).thenReturn(List.of(normal, system)); + + SkillManifest m = SkillManifest.builder() + .name("tcm") + .knowledge(SkillManifest.KnowledgeBinding.builder().bindKb("tcm").build()) + .build(); + ToolCallback list = factory.buildWrappers(m, 99L).get(2); + String out = list.call("{}"); + assertTrue(out.contains("\"a\"")); + assertFalse(out.contains("\"overview\""), "system pages should be filtered out"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/lessons/SkillLessonsServiceTest.java b/mateclaw-server/src/test/java/vip/mate/skill/lessons/SkillLessonsServiceTest.java new file mode 100644 index 00000000..b579790b --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/lessons/SkillLessonsServiceTest.java @@ -0,0 +1,150 @@ +package vip.mate.skill.lessons; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.skill.lessons.event.SkillLessonWrittenEvent; +import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.skill.workspace.SkillWorkspaceManager; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * RFC-090 §11.4 / §14.3 — locked-in behaviour for LESSONS.md writes: + * + *
    + *
  1. First write creates the file with the canonical header and the + * new section appended.
  2. + *
  3. Subsequent writes append; SkillLessonWrittenEvent fires once + * per recorded lesson.
  4. + *
  5. FIFO truncation kicks in beyond {@code maxEntries}.
  6. + *
  7. {@code clearLessons} removes the file outright.
  8. + *
  9. Events are NOT MemoryWriteEvent — the SOUL summarizer must + * not see them (§14.3).
  10. + *
+ */ +class SkillLessonsServiceTest { + + @TempDir + Path tempDir; + + private SkillWorkspaceManager workspaceManager; + private ApplicationEventPublisher publisher; + private SkillLessonsService service; + private List publishedEvents; + + @BeforeEach + void setUp() { + workspaceManager = mock(SkillWorkspaceManager.class); + publishedEvents = new ArrayList<>(); + publisher = event -> publishedEvents.add(event); + when(workspaceManager.resolveConventionPath(anyString())) + .thenAnswer(inv -> tempDir.resolve(inv.getArgument(0, String.class))); + service = new SkillLessonsService(workspaceManager, publisher); + } + + @Test + @DisplayName("first write creates file with canonical header and section") + void firstWriteCreatesFile() throws IOException { + Path skillDir = Files.createDirectories(tempDir.resolve("clip-generator")); + ResolvedSkill skill = ResolvedSkill.builder() + .id(1L).name("clip-generator").skillDir(skillDir).build(); + + String id = service.recordLesson(skill, 99L, "conv-1", "Trim cuts on dialogue beats", 50); + assertNotNull(id); + + String contents = Files.readString(skillDir.resolve("LESSONS.md"), StandardCharsets.UTF_8); + assertTrue(contents.startsWith("# Lessons learned for clip-generator")); + assertTrue(contents.contains("Trim cuts on dialogue beats")); + assertTrue(contents.contains("(conversation: conv-1)")); + assertEquals(1, publishedEvents.size()); + assertTrue(publishedEvents.get(0) instanceof SkillLessonWrittenEvent); + SkillLessonWrittenEvent ev = (SkillLessonWrittenEvent) publishedEvents.get(0); + assertEquals(99L, ev.agentId()); + assertEquals(1L, ev.skillId()); + assertEquals("clip-generator", ev.skillName()); + } + + @Test + @DisplayName("two writes produce two sections under one header") + void twoWritesAppend() throws IOException { + Path skillDir = Files.createDirectories(tempDir.resolve("s1")); + ResolvedSkill skill = ResolvedSkill.builder().id(1L).name("s1").skillDir(skillDir).build(); + + service.recordLesson(skill, 1L, "c1", "first", 50); + service.recordLesson(skill, 1L, "c2", "second", 50); + + String contents = Files.readString(skillDir.resolve("LESSONS.md"), StandardCharsets.UTF_8); + long sectionCount = contents.lines().filter(l -> l.startsWith("## ")).count(); + assertEquals(2, sectionCount); + assertEquals(2, publishedEvents.size()); + } + + @Test + @DisplayName("FIFO truncation when entries exceed maxEntries") + void fifoTruncation() throws IOException { + Path skillDir = Files.createDirectories(tempDir.resolve("s2")); + ResolvedSkill skill = ResolvedSkill.builder().id(1L).name("s2").skillDir(skillDir).build(); + + for (int i = 0; i < 5; i++) { + service.recordLesson(skill, null, "c" + i, "lesson " + i, 3); + } + String contents = Files.readString(skillDir.resolve("LESSONS.md"), StandardCharsets.UTF_8); + long sections = contents.lines().filter(l -> l.startsWith("## ")).count(); + assertEquals(3, sections, "FIFO cap should keep only the last 3 sections"); + // Oldest two ("lesson 0" / "lesson 1") should have been dropped. + assertFalse(contents.contains("lesson 0")); + assertFalse(contents.contains("lesson 1")); + assertTrue(contents.contains("lesson 4")); + } + + @Test + @DisplayName("clearLessons removes the file") + void clearLessonsRemovesFile() throws IOException { + Path skillDir = Files.createDirectories(tempDir.resolve("s3")); + ResolvedSkill skill = ResolvedSkill.builder().id(1L).name("s3").skillDir(skillDir).build(); + + service.recordLesson(skill, null, null, "hello", 50); + assertTrue(Files.exists(skillDir.resolve("LESSONS.md"))); + + boolean cleared = service.clearLessons(skill); + assertTrue(cleared); + assertFalse(Files.exists(skillDir.resolve("LESSONS.md"))); + } + + @Test + @DisplayName("readLessonsBody strips the canonical header") + void readLessonsBodyStripsHeader() throws IOException { + Path skillDir = Files.createDirectories(tempDir.resolve("s4")); + ResolvedSkill skill = ResolvedSkill.builder().id(1L).name("s4").skillDir(skillDir).build(); + + service.recordLesson(skill, null, null, "needle", 50); + String body = service.readLessonsBody(skill); + assertNotNull(body); + assertFalse(body.startsWith("# Lessons learned")); + assertTrue(body.startsWith("## ")); + assertTrue(body.contains("needle")); + } + + @Test + @DisplayName("no workspace directory results in graceful no-op") + void noWorkspaceNoOp() { + ResolvedSkill skill = ResolvedSkill.builder().id(1L).name("nope").build(); + // Force a non-existent convention path so resolveWorkspace returns null. + when(workspaceManager.resolveConventionPath("nope")) + .thenReturn(tempDir.resolve("does-not-exist")); + String id = service.recordLesson(skill, null, null, "won't write", 50); + assertNull(id); + assertTrue(publishedEvents.isEmpty()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/manifest/SkillManifestParserTest.java b/mateclaw-server/src/test/java/vip/mate/skill/manifest/SkillManifestParserTest.java new file mode 100644 index 00000000..c07cfcf3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/manifest/SkillManifestParserTest.java @@ -0,0 +1,309 @@ +package vip.mate.skill.manifest; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.runtime.SkillFrontmatterParser; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-090 Phase 2 — manifest parser regression tests. + * + *

Covers: identity fields, allowed-tools alias, requires/features + * matrix, settings/dashboard, self-evolution defaults, knowledge block, + * and legacy fallback (no v3 frontmatter). + */ +class SkillManifestParserTest { + + private SkillManifestParser parser; + + @BeforeEach + void setUp() { + parser = new SkillManifestParser(new SkillFrontmatterParser()); + } + + @Test + @DisplayName("parses the full v3.1 manifest") + void parsesFullManifest() { + String content = """ + --- + id: clip-generator + name: clip-generator + description: Long video to viral short clips + icon: "🎬" + version: 1.2.0 + author: matevip + type: code + category: content + allowed-tools: [shell_exec, file_read] + platforms: [macos, linux] + requires: + - key: ffmpeg + type: binary + check: ffmpeg + optional: false + description: FFmpeg binary + install: + macos: brew install ffmpeg + linux_apt: sudo apt install ffmpeg + - key: groq_key + type: api_key + check: GROQ_API_KEY + features: + - id: trim_video + label: "Trim video" + requires: [ffmpeg] + platforms: [macos, linux, windows] + - id: auto_captions + label: "Auto captions" + requires: [ffmpeg, groq_key] + fallback_message: "Install whisper for local STT" + settings: + - key: stt_provider + label: STT + type: select + default: auto + options: + - value: auto + - value: groq_whisper + requires-model: [vision, function_calling] + dashboard: + metrics: + - label: Clips + memory_key: clip_jobs_done + format: number + self-evolution: + lessons_enabled: false + lessons_max_entries: 12 + memory_writes_allowed: true + --- + # body + """; + + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals("clip-generator", m.getId()); + assertEquals("code", m.getType()); + assertEquals("matevip", m.getAuthor()); + assertEquals("1.2.0", m.getVersion()); + assertEquals(2, m.getAllowedTools().size()); + assertTrue(m.getAllowedTools().contains("shell_exec")); + assertEquals(2, m.getRequires().size()); + assertEquals("ffmpeg", m.getRequires().get(0).getKey()); + assertEquals("binary", m.getRequires().get(0).getType()); + assertEquals("brew install ffmpeg", m.getRequires().get(0).getInstall().get("macos")); + assertEquals(2, m.getFeatures().size()); + assertEquals("trim_video", m.getFeatures().get(0).getId()); + assertEquals(1, m.getFeatures().get(0).getRequires().size()); + assertEquals("Install whisper for local STT", m.getFeatures().get(1).getFallbackMessage()); + assertEquals(1, m.getSettings().size()); + assertEquals("stt_provider", m.getSettings().get(0).getKey()); + assertEquals(2, m.getRequiresModel().size()); + assertEquals(1, m.getDashboardMetrics().size()); + assertFalse(m.getSelfEvolution().isLessonsEnabled()); + assertEquals(12, m.getSelfEvolution().getLessonsMaxEntries()); + } + + @Test + @DisplayName("falls back to legacy dependencies.tools when allowed-tools is absent") + void fallsBackToLegacyDependencyTools() { + // Most existing SKILL.md files (pre-v3) declare tools via the + // dependencies.tools list, not v3 allowed-tools. This is the + // root cause of the Tools tab rendering empty for shipped + // skills. Locking the fallback in regression form. + String content = """ + --- + name: legacy-skill + description: legacy-style declaration + dependencies: + tools: [shell_exec, file_read, web_fetch] + commands: [python3] + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals(3, m.getAllowedTools().size(), "allowedTools should fall back to dependencies.tools"); + assertTrue(m.getAllowedTools().contains("shell_exec")); + assertTrue(m.getAllowedTools().contains("file_read")); + assertTrue(m.getAllowedTools().contains("web_fetch")); + } + + @Test + @DisplayName("v3 allowed-tools wins over legacy dependencies.tools") + void v3AllowedToolsWinsOverLegacy() { + String content = """ + --- + name: hybrid-skill + allowed-tools: [v3_only_tool] + dependencies: + tools: [legacy_tool] + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals(1, m.getAllowedTools().size()); + assertEquals("v3_only_tool", m.getAllowedTools().get(0), + "v3 allowed-tools should take precedence over legacy dependencies.tools"); + } + + @Test + @DisplayName("supports allowed_tools underscore alias") + void supportsAllowedToolsAlias() { + String content = """ + --- + name: x + allowed_tools: + - foo + - bar + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals(2, m.getAllowedTools().size()); + } + + @Test + @DisplayName("ckjia-shopping declares MCP tools and bumped bundle version") + void ckjiaShoppingDeclaresMcpToolsAndBumpedVersion() throws Exception { + String content = readClasspathText("skills/ckjia-shopping/SKILL.md"); + + SkillManifest m = parser.parse(content); + + assertNotNull(m); + assertEquals("mcp", m.getType()); + assertEquals("1.0.1", m.getVersion(), + "bundle version must bump whenever shipped SKILL.md behavior changes"); + assertEquals(Set.of("ckjia_shopping_recommend", "ckjia_image_recognize", "ckjia_ping"), + Set.copyOf(m.getAllowedTools()), + "explicit skill bindings expand only allowed-tools, not prose tool names"); + } + + @Test + @DisplayName("synthesizes requires from legacy dependencies block") + void synthesizesLegacyDependencies() { + String content = """ + --- + name: legacy-skill + description: legacy + dependencies: + commands: [python3, ffmpeg] + env: [OPENAI_API_KEY] + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + // No explicit requires[] → synthesized from legacy commands+env. + assertEquals(3, m.getRequires().size()); + assertEquals("cmd:python3", m.getRequires().get(0).getKey()); + assertEquals("binary", m.getRequires().get(0).getType()); + assertEquals("env:OPENAI_API_KEY", m.getRequires().get(2).getKey()); + assertEquals("env_var", m.getRequires().get(2).getType()); + } + + @Test + @DisplayName("returns null for content with no frontmatter") + void returnsNullForNoFrontmatter() { + SkillManifest m = parser.parse("# Just a markdown file\n\nNo frontmatter here."); + assertNull(m); + } + + @Test + @DisplayName("self-evolution defaults are on when block is absent") + void selfEvolutionDefaults() { + String content = """ + --- + name: minimal + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertTrue(m.getSelfEvolution().isLessonsEnabled()); + assertEquals(50, m.getSelfEvolution().getLessonsMaxEntries()); + assertTrue(m.getSelfEvolution().isMemoryWritesAllowed()); + } + + @Test + @DisplayName("knowledge block parses bind_kb / retrieval / citation") + void knowledgeBlockParses() { + String content = """ + --- + name: tcm-qa + type: knowledge + knowledge: + bind_kb: tcm-classics + retrieval: hybrid + top_k: 8 + citation: required + rerank: true + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertNotNull(m.getKnowledge()); + assertEquals("tcm-classics", m.getKnowledge().getBindKb()); + assertEquals("hybrid", m.getKnowledge().getRetrieval()); + assertEquals(8, m.getKnowledge().getTopK()); + assertEquals("required", m.getKnowledge().getCitation()); + assertTrue(m.getKnowledge().isRerank()); + assertNull(m.getKnowledge().getBoundKbId()); + } + + @Test + @DisplayName("acp block parses endpoint / system_prefix / cwd") + void acpBlockParses() { + String content = """ + --- + name: codex-helper + type: acp + acp: + endpoint: codex + system_prefix: "Be concise." + cwd: /tmp/project + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals("acp", m.getType()); + assertNotNull(m.getAcp()); + assertEquals("codex", m.getAcp().getEndpoint()); + assertEquals("Be concise.", m.getAcp().getSystemPrefix()); + assertEquals("/tmp/project", m.getAcp().getCwd()); + assertNull(m.getAcp().getResolvedEndpointId()); + } + + @Test + @DisplayName("preserves unknown keys in extras for forward-compat") + void preservesUnknownKeysInExtras() { + String content = """ + --- + name: future-skill + future_field: someValue + another_one: 42 + --- + body + """; + SkillManifest m = parser.parse(content); + assertNotNull(m); + assertEquals("someValue", m.getExtras().get("future_field")); + assertEquals(42, m.getExtras().get("another_one")); + } + + private static String readClasspathText(String path) throws Exception { + try (InputStream is = SkillManifestParserTest.class.getClassLoader().getResourceAsStream(path)) { + assertNotNull(is, "missing classpath resource: " + path); + return new String(is.readAllBytes(), StandardCharsets.UTF_8); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/mcp/McpSkillBridgeManifestTest.java b/mateclaw-server/src/test/java/vip/mate/skill/mcp/McpSkillBridgeManifestTest.java new file mode 100644 index 00000000..64210027 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/mcp/McpSkillBridgeManifestTest.java @@ -0,0 +1,162 @@ +package vip.mate.skill.mcp; + +import com.fasterxml.jackson.databind.ObjectMapper; +import io.modelcontextprotocol.spec.McpSchema; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.model.SkillEntity; +import vip.mate.tool.mcp.model.McpServerEntity; +import vip.mate.tool.mcp.runtime.McpClientManager; +import vip.mate.tool.mcp.runtime.McpToolNameResolver; +import vip.mate.tool.mcp.service.McpServerService; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Asserts the manifest writes prefixed callback names into + * {@code allowedTools} (so {@code ResolvedSkill.getEffectiveAllowedTools()} + * returns names that {@link vip.mate.tool.mcp.runtime.McpClientManager} also + * registers) and that the cache-first / live-fallback ordering holds. + */ +class McpSkillBridgeManifestTest { + + private McpServerService mcpServerService; + private McpClientManager mcpClientManager; + private McpSkillBridge bridge; + + @BeforeEach + void setUp() { + mcpServerService = mock(McpServerService.class); + mcpClientManager = mock(McpClientManager.class); + bridge = new McpSkillBridge(mcpServerService, mcpClientManager, new ObjectMapper()); + } + + @Test + @DisplayName("manifest emits prefixed tool names matching the resolver output") + void allowedToolsArePrefixed() { + McpServerEntity server = newServer(42L, "github"); + server.setToolsCacheJson(toolsJson("create_issue", "list_issues")); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + SkillEntity entity = bridge.listMcpDerivedSkillEntities().get(0); + + // The synthesized SkillEntity carries manifest_json — parse it back + // and check allowedTools contains the prefixed names. + String manifestJson = entity.getManifestJson(); + assertTrue(manifestJson.contains("\"" + McpToolNameResolver.prefixedName(42L, "create_issue") + "\""), + "expected prefixed create_issue in manifest, got: " + manifestJson); + assertTrue(manifestJson.contains("\"" + McpToolNameResolver.prefixedName(42L, "list_issues") + "\""), + "expected prefixed list_issues in manifest, got: " + manifestJson); + } + + @Test + @DisplayName("manifest reads from tools_cache_json when present, never hits the live runtime") + void readsFromCacheFirst() { + McpServerEntity server = newServer(42L, "github"); + server.setToolsCacheJson(toolsJson("create_issue")); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + bridge.listMcpDerivedSkillEntities(); + + verify(mcpClientManager, never()).getServerTools(anyLong()); + } + + @Test + @DisplayName("manifest falls back to live runtime when cache is absent") + void fallsBackToLiveWhenCacheMissing() { + McpServerEntity server = newServer(42L, "github"); + server.setToolsCacheJson(null); // first-ever connect just happened, cache not yet written + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + when(mcpClientManager.getServerTools(42L)).thenReturn(List.of( + fakeTool("create_issue"), + fakeTool("list_issues"))); + + SkillEntity entity = bridge.listMcpDerivedSkillEntities().get(0); + + verify(mcpClientManager, times(1)).getServerTools(42L); + assertTrue(entity.getManifestJson().contains(McpToolNameResolver.prefixedName(42L, "create_issue"))); + } + + @Test + @DisplayName("disconnected server with empty cache yields an empty allowedTools — no exceptions") + void disconnectedAndEmptyCacheIsHandled() { + McpServerEntity server = newServer(42L, "github"); + server.setToolsCacheJson(""); + server.setLastStatus("disconnected"); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + when(mcpClientManager.getServerTools(42L)).thenReturn(List.of()); + + SkillEntity entity = bridge.listMcpDerivedSkillEntities().get(0); + + // The manifest should still serialize successfully — the picker can + // still show the skill in stale mode. Jackson may omit the empty + // allowedTools list entirely, so just assert no prefixed names + // leaked in (which would indicate a stale-cache regression). + assertEquals("github", entity.getName()); + assertTrue(!entity.getManifestJson().contains("mcp_42_"), + "no prefixed tool name expected, got: " + entity.getManifestJson()); + } + + @Test + @DisplayName("two servers exposing the same raw tool name produce distinct prefixed names") + void twoServersSameRawNameDistinct() { + McpServerEntity a = newServer(42L, "github"); + a.setToolsCacheJson(toolsJson("search")); + McpServerEntity b = newServer(43L, "filesystem"); + b.setToolsCacheJson(toolsJson("search")); + when(mcpServerService.listEnabled()).thenReturn(List.of(a, b)); + + List entities = bridge.listMcpDerivedSkillEntities(); + + Set prefixed = Set.of( + McpToolNameResolver.prefixedName(42L, "search"), + McpToolNameResolver.prefixedName(43L, "search")); + assertEquals(2, prefixed.size()); + assertTrue(entities.get(0).getManifestJson().contains(McpToolNameResolver.prefixedName(42L, "search"))); + assertTrue(entities.get(1).getManifestJson().contains(McpToolNameResolver.prefixedName(43L, "search"))); + } + + private static McpServerEntity newServer(long id, String name) { + McpServerEntity s = new McpServerEntity(); + s.setId(id); + s.setName(name); + s.setEnabled(true); + s.setTransport("stdio"); + s.setCommand("/usr/bin/echo"); + s.setLastStatus("connected"); + return s; + } + + private static String toolsJson(String... names) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < names.length; i++) { + if (i > 0) sb.append(","); + sb.append("{\"name\":\"").append(names[i]) + .append("\",\"description\":\"\",\"inputSchema\":{}}"); + } + sb.append("]"); + return sb.toString(); + } + + private static McpSchema.Tool fakeTool(String name) { + return new McpSchema.Tool( + name, + /* title */ name, + "Test tool", + /* inputSchema */ null, + /* outputSchema */ null, + /* annotations */ null, + /* meta */ null); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillCatalogSorterTest.java b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillCatalogSorterTest.java new file mode 100644 index 00000000..70d9e8e3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillCatalogSorterTest.java @@ -0,0 +1,51 @@ +package vip.mate.skill.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.model.SkillEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class SkillCatalogSorterTest { + + @Test + @DisplayName("recommended order keeps ready builtins before external virtual skills") + void recommendedOrderKeepsReadyBuiltinsBeforeExternalVirtualSkills() { + SkillEntity claude = skill("claude-code", "acp", true, "PASSED"); + SkillEntity appleNotes = skill("apple-notes", "builtin", true, "PASSED"); + SkillEntity dynamic = skill("team-runbook", "dynamic", true, "PASSED"); + SkillEntity blocked = skill("unsafe", "builtin", true, "FAILED"); + SkillEntity disabled = skill("disabled-core", "builtin", false, "PASSED"); + + List sorted = SkillCatalogSorter.sortEntities( + List.of(claude, disabled, blocked, dynamic, appleNotes), + SkillCatalogSort.RECOMMENDED); + + assertEquals(List.of(appleNotes, dynamic, claude, disabled, blocked), sorted); + } + + @Test + @DisplayName("name order is stable across sources") + void nameOrderIsStableAcrossSources() { + SkillEntity zed = skill("zed", "acp", true, "PASSED"); + SkillEntity alpha = skill("alpha", "builtin", true, "PASSED"); + + List sorted = SkillCatalogSorter.sortEntities( + List.of(zed, alpha), + SkillCatalogSort.NAME); + + assertEquals(List.of(alpha, zed), sorted); + } + + private static SkillEntity skill(String name, String type, boolean enabled, String scanStatus) { + SkillEntity s = new SkillEntity(); + s.setName(name); + s.setDescription("Description for " + name); + s.setSkillType(type); + s.setEnabled(enabled); + s.setSecurityScanStatus(scanStatus); + return s; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServicePromptBudgetTest.java b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServicePromptBudgetTest.java new file mode 100644 index 00000000..e0c48f11 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServicePromptBudgetTest.java @@ -0,0 +1,184 @@ +package vip.mate.skill.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.acp.AcpSkillBridge; +import vip.mate.skill.lessons.SkillLessonsService; +import vip.mate.skill.mcp.McpSkillBridge; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.skill.service.SkillService; +import vip.mate.skill.usage.SkillUsageService; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SkillRuntimeServicePromptBudgetTest { + + @Test + @DisplayName("unbound prompt renders a small catalog and skips lessons") + void unboundPromptUsesSmallCatalogAndSkipsLessons() { + SkillService skillService = mock(SkillService.class); + SkillPackageResolver resolver = mock(SkillPackageResolver.class); + SkillLessonsService lessonsService = mock(SkillLessonsService.class); + McpSkillBridge mcpBridge = mock(McpSkillBridge.class); + AcpSkillBridge acpBridge = mock(AcpSkillBridge.class); + SkillUsageService usageService = mock(SkillUsageService.class); + + List entities = java.util.stream.IntStream.rangeClosed(1, 12) + .mapToObj(i -> entity((long) i, "skill-%02d".formatted(i), "builtin")) + .toList(); + when(skillService.listEnabledSkills()).thenReturn(entities); + for (SkillEntity entity : entities) { + when(resolver.resolve(entity)).thenReturn(resolved(entity)); + } + when(mcpBridge.listMcpDerivedResolvedSkills()).thenReturn(List.of()); + when(acpBridge.listAcpDerivedResolvedSkills()).thenReturn(List.of()); + when(usageService.recentLoadedSkillNames(null, 8)).thenReturn(Set.of()); + when(usageService.frequentlyLoadedSkillNames(8)).thenReturn(Set.of()); + + SkillRuntimeService runtime = new SkillRuntimeService( + skillService, resolver, lessonsService, mcpBridge, acpBridge, usageService); + + String prompt = runtime.buildSkillPromptEnhancement(null, null, 8192); + + assertTrue(prompt.contains("skill-01")); + assertTrue(prompt.contains("skill-08")); + assertFalse(prompt.contains("skill-09")); + assertTrue(prompt.contains("Showing 8 of 12")); + assertFalse(prompt.contains("Lessons learned")); + verify(lessonsService, never()).readLessonsBody(any()); + } + + @Test + @DisplayName("bound prompt pins bound skill and only reads its lessons") + void boundPromptPinsBoundSkillAndOnlyReadsItsLessons() { + SkillService skillService = mock(SkillService.class); + SkillPackageResolver resolver = mock(SkillPackageResolver.class); + SkillLessonsService lessonsService = mock(SkillLessonsService.class); + McpSkillBridge mcpBridge = mock(McpSkillBridge.class); + AcpSkillBridge acpBridge = mock(AcpSkillBridge.class); + SkillUsageService usageService = mock(SkillUsageService.class); + + SkillEntity first = entity(1L, "apple-notes", "builtin"); + SkillEntity bound = entity(99L, "ckjia-shopping", "builtin"); + when(skillService.listEnabledSkills()).thenReturn(List.of(first, bound)); + ResolvedSkill firstResolved = resolved(first); + ResolvedSkill boundResolved = resolved(bound); + when(resolver.resolve(first)).thenReturn(firstResolved); + when(resolver.resolve(bound)).thenReturn(boundResolved); + when(lessonsService.readLessonsBody(boundResolved)).thenReturn("Use markdown links for products."); + when(usageService.recentLoadedSkillNames(null, 8)).thenReturn(Set.of()); + when(usageService.frequentlyLoadedSkillNames(8)).thenReturn(Set.of()); + + SkillRuntimeService runtime = new SkillRuntimeService( + skillService, resolver, lessonsService, mcpBridge, acpBridge, usageService); + + String prompt = runtime.buildSkillPromptEnhancement(Set.of(99L), null, 8192); + + assertTrue(prompt.indexOf("ckjia-shopping") < prompt.indexOf("Lessons learned")); + assertTrue(prompt.contains("Use markdown links for products.")); + verify(lessonsService).readLessonsBody(boundResolved); + verify(lessonsService, never()).readLessonsBody(firstResolved); + } + + @Test + @DisplayName("recently loaded skill lessons are included for the same agent") + void recentLoadedSkillLessonsAreIncludedForAgent() { + SkillService skillService = mock(SkillService.class); + SkillPackageResolver resolver = mock(SkillPackageResolver.class); + SkillLessonsService lessonsService = mock(SkillLessonsService.class); + McpSkillBridge mcpBridge = mock(McpSkillBridge.class); + AcpSkillBridge acpBridge = mock(AcpSkillBridge.class); + SkillUsageService usageService = mock(SkillUsageService.class); + + SkillEntity recent = entity(7L, "browser-cdp", "builtin"); + when(skillService.listEnabledSkills()).thenReturn(List.of(recent)); + ResolvedSkill recentResolved = resolved(recent); + when(resolver.resolve(recent)).thenReturn(recentResolved); + when(usageService.recentLoadedSkillNames(42L, 8)).thenReturn(Set.of("browser-cdp")); + when(usageService.frequentlyLoadedSkillNames(8)).thenReturn(Set.of()); + when(lessonsService.readLessonsBody(recentResolved)).thenReturn("Prefer inspecting the live page."); + + SkillRuntimeService runtime = new SkillRuntimeService( + skillService, resolver, lessonsService, mcpBridge, acpBridge, usageService); + + String prompt = runtime.buildSkillPromptEnhancement(null, null, 8192, 42L); + + assertTrue(prompt.contains("Prefer inspecting the live page.")); + verify(lessonsService).readLessonsBody(recentResolved); + } + + @Test + @DisplayName("bound prompt 包含被显式勾选的 MCP 虚拟 skill(虚拟 skill 不丢 catalog 行)") + void boundPromptIncludesVirtualMcpSkill() { + // Regression for: an agent that explicitly binds an MCP-derived + // virtual skill (via /skills/enabled picker) used to get its + // tools — via AgentBindingService.getEffectiveToolNames — + // but lost the corresponding `## Skills` catalog row, because + // the bound branch sourced only real mate_skill entries. + SkillService skillService = mock(SkillService.class); + SkillPackageResolver resolver = mock(SkillPackageResolver.class); + SkillLessonsService lessonsService = mock(SkillLessonsService.class); + McpSkillBridge mcpBridge = mock(McpSkillBridge.class); + AcpSkillBridge acpBridge = mock(AcpSkillBridge.class); + SkillUsageService usageService = mock(SkillUsageService.class); + + // No real skill rows — the agent only ever bound the virtual one. + when(skillService.listEnabledSkills()).thenReturn(List.of()); + long virtualMcpId = McpSkillBridge.VIRTUAL_ID_BASE + 7L; + ResolvedSkill virtualMcp = ResolvedSkill.builder() + .id(virtualMcpId) + .name("mcp-virtual-skill") + .description("Bridged from an enabled MCP server") + .enabled(true) + .runtimeAvailable(true) + .dependencyReady(true) + .securityBlocked(false) + .build(); + when(mcpBridge.listMcpDerivedResolvedSkills()).thenReturn(List.of(virtualMcp)); + when(acpBridge.listAcpDerivedResolvedSkills()).thenReturn(List.of()); + when(usageService.recentLoadedSkillNames(null, 8)).thenReturn(Set.of()); + when(usageService.frequentlyLoadedSkillNames(8)).thenReturn(Set.of()); + + SkillRuntimeService runtime = new SkillRuntimeService( + skillService, resolver, lessonsService, mcpBridge, acpBridge, usageService); + + String prompt = runtime.buildSkillPromptEnhancement(Set.of(virtualMcpId), null, 8192); + + assertTrue(prompt.contains("mcp-virtual-skill"), + "bound MCP virtual skill must appear in the rendered catalog; " + + "prompt was: " + prompt); + } + + private static SkillEntity entity(Long id, String name, String type) { + SkillEntity entity = new SkillEntity(); + entity.setId(id); + entity.setName(name); + entity.setDescription("Description for " + name); + entity.setSkillType(type); + entity.setEnabled(true); + entity.setSecurityScanStatus("PASSED"); + return entity; + } + + private static ResolvedSkill resolved(SkillEntity entity) { + return ResolvedSkill.builder() + .id(entity.getId()) + .name(entity.getName()) + .description(entity.getDescription()) + .enabled(Boolean.TRUE.equals(entity.getEnabled())) + .runtimeAvailable(true) + .dependencyReady(true) + .securityBlocked(false) + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServiceRecencyBoostTest.java b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServiceRecencyBoostTest.java new file mode 100644 index 00000000..ec61f9a0 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillRuntimeServiceRecencyBoostTest.java @@ -0,0 +1,87 @@ +package vip.mate.skill.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.runtime.model.ResolvedSkill; + +import java.time.LocalDateTime; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Tests for the freshly-installed-skill boost + * ({@link SkillRuntimeService#isRecentlyInstalled}). + * + *

Issue context: a brand-new skill (e.g. tencent-meeting-mcp uploaded + * minutes ago) has zero usage stats and so falls behind ~40 existing skills + * in the prompt-catalog ranker. With qwen-turbo's 8-entry budget the agent + * never sees it and tells the user "no such skill". The boost lifts skills + * created within the configured window to the top of the secondary sort + * so the user can actually find what they just installed. + */ +class SkillRuntimeServiceRecencyBoostTest { + + private static ResolvedSkill skill(String name, LocalDateTime createTime, boolean builtin) { + return ResolvedSkill.builder() + .id(name.hashCode() & 0x7fffffffL) + .name(name) + .builtin(builtin) + .createTime(createTime) + .build(); + } + + @Test + @DisplayName("skill installed inside the window is recent") + void freshSkillIsRecent() { + LocalDateTime now = LocalDateTime.now(); + LocalDateTime cutoff = now.minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW); + ResolvedSkill fresh = skill("tencent-meeting-mcp", now.minusHours(2), false); + + assertTrue(SkillRuntimeService.isRecentlyInstalled(fresh, cutoff)); + } + + @Test + @DisplayName("skill installed before the window is not recent") + void oldSkillIsNotRecent() { + LocalDateTime cutoff = LocalDateTime.now().minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW); + ResolvedSkill old = skill("legacy", cutoff.minusDays(30), false); + + assertFalse(SkillRuntimeService.isRecentlyInstalled(old, cutoff)); + } + + @Test + @DisplayName("builtin skills are never boosted (the user didn't install them)") + void builtinIsNotRecent() { + LocalDateTime cutoff = LocalDateTime.now().minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW); + // Even if create_time happens to fall inside the window (e.g. fresh DB seed), + // a builtin row was not a user install and shouldn't claim a top slot. + ResolvedSkill recentBuiltin = skill("file_reader", LocalDateTime.now().minusHours(1), true); + + assertFalse(SkillRuntimeService.isRecentlyInstalled(recentBuiltin, cutoff)); + } + + @Test + @DisplayName("missing createTime → not recent (virtual MCP/ACP rows)") + void missingCreateTimeIsNotRecent() { + LocalDateTime cutoff = LocalDateTime.now().minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW); + ResolvedSkill virt = ResolvedSkill.builder().id(1L).name("virt").build(); + + assertFalse(SkillRuntimeService.isRecentlyInstalled(virt, cutoff)); + } + + @Test + @DisplayName("null skill is safe to query") + void nullSkillIsSafe() { + assertFalse(SkillRuntimeService.isRecentlyInstalled(null, + LocalDateTime.now().minus(SkillRuntimeService.NEW_SKILL_BOOST_WINDOW))); + } + + @Test + @DisplayName("default window is 7 days — long enough to span a weekend") + void defaultWindowIsAWeek() { + // Sanity-pin so future tweaks have to deliberately update the test. + // The window matters: too short and a Friday installer is invisible + // by Monday; too long and the boost slot crowds out useful skills. + assertEquals(7, SkillRuntimeService.NEW_SKILL_BOOST_WINDOW.toDays()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/runtime/model/ResolvedSkillEffectiveToolsTest.java b/mateclaw-server/src/test/java/vip/mate/skill/runtime/model/ResolvedSkillEffectiveToolsTest.java new file mode 100644 index 00000000..f8e2dfe5 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/runtime/model/ResolvedSkillEffectiveToolsTest.java @@ -0,0 +1,154 @@ +package vip.mate.skill.runtime.model; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.manifest.SkillManifest; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-090 §14.2 — getEffectiveAllowedTools regression tests. + * + *

Pinned scenarios: + *

    + *
  1. No manifest → empty set (legacy fallback)
  2. + *
  3. Manifest, no features → returns allowed-tools wholesale
  4. + *
  5. Manifest with READY feature carrying its own tool subset → + * only the subset is exposed
  6. + *
  7. Manifest with READY feature using inheritance → + * manifest-level allowed-tools surface
  8. + *
  9. Manifest with SETUP_NEEDED feature → its tools stay hidden + * (the LLM must not see unavailable capabilities, §10.2 Q8)
  10. + *
+ */ +class ResolvedSkillEffectiveToolsTest { + + @Test + @DisplayName("no manifest yields empty set") + void noManifest() { + ResolvedSkill r = ResolvedSkill.builder().name("legacy").build(); + assertTrue(r.getEffectiveAllowedTools().isEmpty()); + } + + @Test + @DisplayName("manifest with no features returns allowed-tools wholesale") + void manifestNoFeaturesReturnsAllAllowedTools() { + SkillManifest manifest = SkillManifest.builder() + .name("simple") + .allowedTools(List.of("web_search", "file_read")) + .build(); + ResolvedSkill r = ResolvedSkill.builder() + .name("simple") + .manifest(manifest) + .build(); + assertEquals(Set.of("web_search", "file_read"), r.getEffectiveAllowedTools()); + } + + @Test + @DisplayName("READY feature with its own tools narrows surface") + void readyFeatureWithOwnToolsSubset() { + SkillManifest manifest = SkillManifest.builder() + .name("clip") + .allowedTools(List.of("web_search", "shell_exec", "file_read")) + .features(List.of( + SkillManifest.FeatureDef.builder() + .id("trim_video").tools(List.of("shell_exec")).build())) + .build(); + ResolvedSkill r = ResolvedSkill.builder() + .manifest(manifest) + .featureStatuses(Map.of("trim_video", "READY")) + .activeFeatures(Set.of("trim_video")) + .build(); + assertEquals(Set.of("shell_exec"), r.getEffectiveAllowedTools()); + } + + @Test + @DisplayName("READY feature with empty tools inherits manifest-level allowed-tools") + void readyFeatureInheritsAllowedTools() { + SkillManifest manifest = SkillManifest.builder() + .name("clip") + .allowedTools(List.of("web_search", "shell_exec")) + .features(List.of( + SkillManifest.FeatureDef.builder().id("default").build())) + .build(); + ResolvedSkill r = ResolvedSkill.builder() + .manifest(manifest) + .featureStatuses(Map.of("default", "READY")) + .activeFeatures(Set.of("default")) + .build(); + assertEquals(Set.of("web_search", "shell_exec"), r.getEffectiveAllowedTools()); + } + + @Test + @DisplayName("SETUP_NEEDED feature stays hidden from advertisement") + void setupNeededFeatureHidden() { + SkillManifest manifest = SkillManifest.builder() + .name("clip") + .allowedTools(List.of("file_read")) + .features(List.of( + SkillManifest.FeatureDef.builder() + .id("trim_video").tools(List.of("shell_exec")).build(), + SkillManifest.FeatureDef.builder() + .id("captions").tools(List.of("ai_caption")).build())) + .build(); + ResolvedSkill r = ResolvedSkill.builder() + .manifest(manifest) + .featureStatuses(Map.of("trim_video", "READY", "captions", "SETUP_NEEDED")) + .activeFeatures(Set.of("trim_video")) + .build(); + Set tools = r.getEffectiveAllowedTools(); + assertTrue(tools.contains("shell_exec")); + assertFalse(tools.contains("ai_caption")); + } + + @Test + @DisplayName("inheritance does not re-expose tools owned by SETUP_NEEDED features") + void inheritanceFencedAgainstSetupNeededTools() { + // Two features: + // - "trim_video" READY but uses inheritance (empty tools list) + // - "captions" SETUP_NEEDED and explicitly claims `ai_caption` + // The manifest-level allowed-tools includes both `shell_exec` + // (general) and `ai_caption` (claimed by captions). The + // READY-via-inheritance branch must surface shell_exec but + // NOT re-expose ai_caption. + SkillManifest manifest = SkillManifest.builder() + .name("clip") + .allowedTools(List.of("shell_exec", "ai_caption")) + .features(List.of( + SkillManifest.FeatureDef.builder() + .id("trim_video") + .build(), // empty tools → inherits + SkillManifest.FeatureDef.builder() + .id("captions") + .tools(List.of("ai_caption")) + .build())) + .build(); + ResolvedSkill r = ResolvedSkill.builder() + .manifest(manifest) + .featureStatuses(Map.of( + "trim_video", "READY", + "captions", "SETUP_NEEDED")) + .activeFeatures(Set.of("trim_video")) + .build(); + Set tools = r.getEffectiveAllowedTools(); + assertTrue(tools.contains("shell_exec")); + assertFalse(tools.contains("ai_caption"), + "inheritance must NOT re-expose tools claimed by a SETUP_NEEDED feature"); + } + + @Test + @DisplayName("hasAnyActiveFeature reflects activeFeatures set") + void hasAnyActiveFeatureFlag() { + ResolvedSkill empty = ResolvedSkill.builder().build(); + assertFalse(empty.hasAnyActiveFeature()); + + ResolvedSkill withActive = ResolvedSkill.builder() + .activeFeatures(Set.of("default")) + .build(); + assertTrue(withActive.hasAnyActiveFeature()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/secret/SkillSecretServiceTest.java b/mateclaw-server/src/test/java/vip/mate/skill/secret/SkillSecretServiceTest.java new file mode 100644 index 00000000..37cd9d25 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/secret/SkillSecretServiceTest.java @@ -0,0 +1,162 @@ +package vip.mate.skill.secret; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.test.util.ReflectionTestUtils; +import vip.mate.exception.MateClawException; +import vip.mate.skill.repository.SkillSecretMapper; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +/** + * Locks in the security-sensitive bits of {@link SkillSecretService}: + * AES round-trip, value masking, env-var-shaped key validation, and + * cascade purge. Mapper queries are mocked — wrappers are opaque for + * unit tests, so we only verify which mapper methods get hit + * and what they receive. + */ +class SkillSecretServiceTest { + + private SkillSecretMapper mapper; + private SkillSecretService service; + + @BeforeEach + void setUp() { + mapper = mock(SkillSecretMapper.class); + service = new SkillSecretService(mapper); + ReflectionTestUtils.setField(service, "encryptKey", "TestKey-1234567"); + } + + @Test + @DisplayName("put encrypts the plaintext before persisting (ciphertext != plaintext)") + void putEncryptsBeforePersist() { + when(mapper.selectOne(any())).thenReturn(null); + + service.put(42L, "AIRTABLE_API_KEY", "pat_secret_value_123"); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SkillSecretEntity.class); + verify(mapper).insert((SkillSecretEntity) captor.capture()); + SkillSecretEntity stored = captor.getValue(); + assertEquals("AIRTABLE_API_KEY", stored.getSecretKey()); + assertNotEquals("pat_secret_value_123", stored.getEncryptedValue(), + "stored value must be encrypted"); + assertTrue(stored.getEncryptedValue().length() >= 32, + "AES hex output should be at least one block"); + } + + @Test + @DisplayName("put → getDecrypted round-trip recovers the original plaintext") + void roundTripRecoversPlaintext() { + // Capture what put() persists, then feed it back to the mapper for getDecrypted. + ArgumentCaptor captor = ArgumentCaptor.forClass(SkillSecretEntity.class); + when(mapper.selectOne(any())).thenReturn(null); + + service.put(7L, "TOKEN", "hello-world-12345"); + verify(mapper).insert((SkillSecretEntity) captor.capture()); + SkillSecretEntity stored = captor.getValue(); + // Wire the mapper to return the captured row on subsequent reads. + when(mapper.selectList(any())).thenReturn(List.of(stored)); + + Map decrypted = service.getDecrypted(7L); + assertEquals(1, decrypted.size()); + assertEquals("hello-world-12345", decrypted.get("TOKEN")); + } + + @Test + @DisplayName("put with existing row updates instead of inserting a duplicate") + void putUpdatesExisting() { + SkillSecretEntity existing = new SkillSecretEntity(); + existing.setId(1L); + existing.setSkillId(42L); + existing.setSecretKey("API_KEY"); + existing.setEncryptedValue("oldcipher"); + when(mapper.selectOne(any())).thenReturn(existing); + + service.put(42L, "API_KEY", "new-value"); + + verify(mapper).updateById(any(SkillSecretEntity.class)); + verify(mapper, times(0)).insert(any(SkillSecretEntity.class)); + assertNotEquals("oldcipher", existing.getEncryptedValue(), + "encryptedValue must be replaced with the new ciphertext"); + } + + @Test + @DisplayName("put with empty value short-circuits to remove (no insert/update)") + void putEmptyDelegatesToRemove() { + service.put(42L, "API_KEY", ""); + + verify(mapper).delete(any()); + verify(mapper, times(0)).insert(any(SkillSecretEntity.class)); + verify(mapper, times(0)).updateById(any(SkillSecretEntity.class)); + } + + @Test + @DisplayName("listSummaries returns masked previews; never plaintext") + void listSummariesMasked() { + SkillSecretEntity row = new SkillSecretEntity(); + row.setSkillId(7L); + row.setSecretKey("TOKEN"); + // Encrypt a known value through the service so the test isn't + // coupled to the AES output format directly. + when(mapper.selectOne(any())).thenReturn(null); + ArgumentCaptor captor = ArgumentCaptor.forClass(SkillSecretEntity.class); + service.put(7L, "TOKEN", "supersecret_credentials"); + verify(mapper).insert((SkillSecretEntity) captor.capture()); + when(mapper.selectList(any())).thenReturn(List.of(captor.getValue())); + + List summaries = service.listSummaries(7L); + assertEquals(1, summaries.size()); + String preview = summaries.get(0).preview(); + assertFalse(preview.contains("supersecret"), "preview must not leak plaintext"); + assertTrue(preview.contains("•"), "preview should contain mask dots: " + preview); + } + + @Test + @DisplayName("getDecrypted returns empty map for null skillId without touching the mapper") + void getDecryptedNullSkillIsNoop() { + assertTrue(service.getDecrypted(null).isEmpty()); + verifyNoInteractions(mapper); + } + + @Test + @DisplayName("rejects keys that aren't env-var-shaped; mapper never called") + void rejectsBadKeys() { + assertThrows(MateClawException.class, () -> service.put(1L, "with-dash", "v")); + assertThrows(MateClawException.class, () -> service.put(1L, "1leading-digit", "v")); + assertThrows(MateClawException.class, () -> service.put(1L, "", "v")); + assertThrows(MateClawException.class, () -> service.put(1L, null, "v")); + assertThrows(MateClawException.class, () -> service.put(null, "FOO", "v")); + verifyNoInteractions(mapper); + } + + @Test + @DisplayName("mask: <=4 chars → all dots; >4 → first 2 + dots + last 2") + void maskShape() { + assertEquals("ab••••yz", SkillSecretService.mask("abcdefxyz")); + assertEquals("••••", SkillSecretService.mask("abc")); + assertEquals("••••", SkillSecretService.mask("")); + assertEquals("", SkillSecretService.mask(null)); + } + + @Test + @DisplayName("purgeForSkill delegates to the cascade hard-delete query") + void purgeDelegates() { + when(mapper.hardDeleteBySkillId(42L)).thenReturn(3); + + int purged = service.purgeForSkill(42L); + + assertEquals(3, purged); + verify(mapper).hardDeleteBySkillId(42L); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java new file mode 100644 index 00000000..d9c0389d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java @@ -0,0 +1,117 @@ +package vip.mate.skill.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.skill.model.SkillFileEntity; +import vip.mate.skill.repository.SkillFileMapper; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.*; + +/** + * Unit tests for the empty-bundle guard on the canonical-store side. + *

+ * Mirrors the FS-side guard in {@code SkillWorkspaceManagerApplyBundleTest}: + * if the new bundle has zero entries for a bucket, existing rows for that + * bucket are preserved unless {@code force=true}. Issue #104 hit this on + * the FS path; the DB path now has the same protection so the canonical + * store cannot be silently wiped either. + */ +class SkillFileServiceTest { + + private SkillFileMapper mapper; + private SkillFileService service; + + @BeforeEach + void setUp() { + mapper = mock(SkillFileMapper.class); + service = new SkillFileService(mapper); + } + + @Test + @DisplayName("empty bundle preserves existing scripts rows") + void emptyBundlePreservesScripts() { + SkillFileEntity row = newRow(1L, "scripts/run.py", "important"); + when(mapper.selectList(any())).thenReturn(List.of(row)); + + var result = service.applyBundleFiles(42L, Map.of(), false); + + assertTrue(result.scriptsPreservedDueToEmptyBundle()); + assertEquals(0, result.rowsWritten()); + assertEquals(0, result.rowsPruned()); + verify(mapper, never()).deleteById(anyLong()); + } + + @Test + @DisplayName("force=true removes even preserved rows") + void forceFlagPrunesScripts() { + SkillFileEntity row = newRow(1L, "scripts/run.py", "doomed"); + when(mapper.selectList(any())).thenReturn(List.of(row)); + + var result = service.applyBundleFiles(42L, Map.of(), true); + + assertFalse(result.scriptsPreservedDueToEmptyBundle()); + assertEquals(1, result.rowsPruned()); + verify(mapper).deleteById(1L); + } + + @Test + @DisplayName("write-then-prune updates changed rows, drops removed ones, inserts new") + void mixedApply() { + SkillFileEntity keep = newRow(1L, "scripts/keep.py", "v1"); + SkillFileEntity removed = newRow(2L, "scripts/old.py", "obsolete"); + when(mapper.selectList(any())).thenReturn(new ArrayList<>(List.of(keep, removed))); + + var result = service.applyBundleFiles(42L, Map.of( + "scripts/keep.py", "v2", // changed → update + "scripts/new.py", "fresh" // new → insert + ), false); + + assertEquals(2, result.rowsWritten(), "1 updated + 1 inserted"); + assertEquals(1, result.rowsPruned(), "old.py removed"); + verify(mapper, times(1)).insert(any(SkillFileEntity.class)); + ArgumentCaptor updateCaptor = ArgumentCaptor.forClass(SkillFileEntity.class); + verify(mapper, times(1)).updateById((SkillFileEntity) updateCaptor.capture()); + assertEquals("v2", updateCaptor.getValue().getContent()); + verify(mapper).deleteById(2L); + } + + @Test + @DisplayName("unchanged rows skip the update (sha256 idempotency)") + void unchangedRowSkipped() { + String content = "stable"; + SkillFileEntity row = newRow(7L, "scripts/run.py", content); + + when(mapper.selectList(any())).thenReturn(List.of(row)); + + var result = service.applyBundleFiles(42L, Map.of("scripts/run.py", content), false); + + assertEquals(0, result.rowsWritten()); + assertEquals(0, result.rowsPruned()); + verify(mapper, never()).updateById(any(SkillFileEntity.class)); + verify(mapper, never()).insert(any(SkillFileEntity.class)); + verify(mapper, never()).deleteById(anyLong()); + } + + private static final AtomicLong IDS = new AtomicLong(1); + + private static SkillFileEntity newRow(Long id, String path, String content) { + SkillFileEntity e = new SkillFileEntity(); + e.setId(id == null ? IDS.incrementAndGet() : id); + e.setSkillId(42L); + e.setFilePath(path); + e.setContent(content); + e.setContentSize(content.length()); + e.setSha256(SkillFileService.sha256Hex(content)); + return e; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/service/SkillServiceUpdatePartialTest.java b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillServiceUpdatePartialTest.java new file mode 100644 index 00000000..c47ca877 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillServiceUpdatePartialTest.java @@ -0,0 +1,174 @@ +package vip.mate.skill.service; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.repository.SkillMapper; +import vip.mate.skill.runtime.SkillRuntimeService; +import vip.mate.skill.secret.SkillSecretService; +import vip.mate.skill.workspace.SkillWorkspaceManager; +import vip.mate.skill.workspace.SkillWorkspaceProperties; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Regression test for issue #93 — saving a SKILL.md from the admin + * dialog blew up with "Internal server error". + * + *

The UI sends a partial PUT body containing only the fields the user + * edited (e.g. {@code skillContent}, optionally {@code sourceCode}). + * Two latent problems hit at once: + *

    + *
  1. Identity fields on the partial entity (notably {@code name}) + * are {@code null}; the service forwarded the partial entity + * straight to {@code syncSkillContentToWorkspace}, which + * eventually called {@code String.replaceAll} on the {@code null} + * name → NPE.
  2. + *
  3. {@code FieldStrategy.ALWAYS} columns ({@code name_zh}, + * {@code name_en}, {@code config_json}, {@code manifest_json}, + * {@code security_scan_result}) were nulled on every save + * because MyBatis Plus writes ALWAYS columns even when the + * entity field is {@code null}. That's a regression of the + * earlier #45 fix, which only patched the resolver write path.
  4. + *
+ * + *

Both have to be fixed by merging the partial update into the + * existing row server-side before persisting and syncing. + */ +class SkillServiceUpdatePartialTest { + + @Test + @DisplayName("partial update for a dynamic skill preserves identity and avoids NPE") + void partialUpdateMergesIntoExisting() throws Exception { + SkillMapper mapper = mock(SkillMapper.class); + SkillWorkspaceManager workspaceManager = mock(SkillWorkspaceManager.class); + SkillWorkspaceProperties workspaceProps = mock(SkillWorkspaceProperties.class); + SkillSecretService secretService = mock(SkillSecretService.class); + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + + SkillService service = new SkillService( + mapper, mock(vip.mate.skill.repository.SkillFileMapper.class), + workspaceManager, workspaceProps, secretService); + service.setRuntimeService(runtimeService); + + SkillEntity existing = new SkillEntity(); + existing.setId(101L); + existing.setName("docx"); + existing.setDescription("placeholder"); + existing.setSkillType("dynamic"); + existing.setVersion("1.0.0"); + existing.setEnabled(true); + existing.setBuiltin(false); + // Fields that #45 protected — they were already valid pre-update, + // and must survive an unrelated body edit. + existing.setNameZh("文档"); + existing.setNameEn("Word docs"); + existing.setConfigJson("{\"foo\":1}"); + existing.setManifestJson("{\"name\":\"docx\"}"); + when(mapper.selectById(101L)).thenReturn(existing); + + // Workspace exists from the create step, so the sync path runs + // — triggering the NPE on the unpatched code. + Path tempRoot = Files.createTempDirectory("skill-svc-test"); + Path skillDir = tempRoot.resolve("docx"); + Files.createDirectories(skillDir); + when(workspaceManager.conventionWorkspaceExists("docx")).thenReturn(true); + when(workspaceManager.resolveConventionPath("docx")).thenReturn(skillDir); + + // What the controller deserializes from the partial PUT body: + // only id + skillContent + sourceCode. + SkillEntity partial = new SkillEntity(); + partial.setId(101L); + partial.setSkillContent("---\nname: docx\nversion: \"1.1.0\"\n---\n# body\n"); + partial.setSourceCode(""); + + assertDoesNotThrow(() -> service.updateSkill(partial), + "saving a partial body update must not blow up — issue #93"); + + // The merged entity that actually hit the DB must keep all the + // identity / projection fields that were on the row already. + ArgumentCaptor written = ArgumentCaptor.forClass(SkillEntity.class); + verify(mapper, times(1)).updateById(written.capture()); + SkillEntity persisted = written.getValue(); + assertEquals("docx", persisted.getName(), + "name must survive a partial body PUT (no FieldStrategy.ALWAYS regression on name)"); + assertEquals("文档", persisted.getNameZh(), + "name_zh is FieldStrategy.ALWAYS — partial save must not null it out (issue #45 regression)"); + assertEquals("Word docs", persisted.getNameEn(), + "name_en is FieldStrategy.ALWAYS — partial save must not null it out"); + assertEquals("{\"foo\":1}", persisted.getConfigJson(), + "config_json is FieldStrategy.ALWAYS — partial save must not null it out"); + assertEquals("{\"name\":\"docx\"}", persisted.getManifestJson(), + "manifest_json is FieldStrategy.ALWAYS — partial save must not null it out"); + // The user-edited fields actually do get the new values. + assertNotNull(persisted.getSkillContent()); + org.junit.jupiter.api.Assertions.assertTrue( + persisted.getSkillContent().contains("version: \"1.1.0\""), + "skill_content from the partial PUT must be applied"); + + // Workspace sync runs — using the merged name, not the partial null. + verify(workspaceManager).conventionWorkspaceExists("docx"); + + // Best-effort cleanup of the temp workspace. + Files.deleteIfExists(skillDir.resolve("SKILL.md")); + Files.deleteIfExists(skillDir); + Files.deleteIfExists(tempRoot); + } + + @Test + @DisplayName("partial identity edit (no body) keeps skill_content intact") + void partialIdentityEditDoesNotClobberBody() { + SkillMapper mapper = mock(SkillMapper.class); + SkillWorkspaceManager workspaceManager = mock(SkillWorkspaceManager.class); + SkillWorkspaceProperties workspaceProps = mock(SkillWorkspaceProperties.class); + SkillSecretService secretService = mock(SkillSecretService.class); + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + + SkillService service = new SkillService( + mapper, mock(vip.mate.skill.repository.SkillFileMapper.class), + workspaceManager, workspaceProps, secretService); + service.setRuntimeService(runtimeService); + + SkillEntity existing = new SkillEntity(); + existing.setId(202L); + existing.setName("notes"); + existing.setSkillType("dynamic"); + existing.setBuiltin(false); + existing.setSkillContent("---\nname: notes\n---\n# previously authored body\n"); + when(mapper.selectById(202L)).thenReturn(existing); + when(workspaceManager.conventionWorkspaceExists(anyString())).thenReturn(false); + + // Identity edit: nameZh / description only — skill_content is + // never touched and must survive. + SkillEntity partial = new SkillEntity(); + partial.setId(202L); + partial.setNameZh("笔记"); + partial.setDescription("New tag line"); + + service.updateSkill(partial); + + ArgumentCaptor written = ArgumentCaptor.forClass(SkillEntity.class); + verify(mapper).updateById(written.capture()); + SkillEntity persisted = written.getValue(); + assertEquals("notes", persisted.getName()); + assertEquals("笔记", persisted.getNameZh()); + assertEquals("New tag line", persisted.getDescription()); + // skill_content was untouched in the PUT body — must keep the old + // body, not be nulled out by FieldStrategy.ALWAYS on the partial. + assertNotNull(persisted.getSkillContent(), + "identity-only PUT must not wipe skill_content"); + org.junit.jupiter.api.Assertions.assertTrue( + persisted.getSkillContent().contains("previously authored body")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/template/SkillTemplateRegistryTest.java b/mateclaw-server/src/test/java/vip/mate/skill/template/SkillTemplateRegistryTest.java new file mode 100644 index 00000000..b53398a9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/template/SkillTemplateRegistryTest.java @@ -0,0 +1,110 @@ +package vip.mate.skill.template; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-091 — verifies the built-in templates parse cleanly and expose + * the form fields the wizard expects. Catches regressions where a + * shipped template.json drifts out of schema. + */ +class SkillTemplateRegistryTest { + + private SkillTemplateRegistry registry; + + @BeforeEach + void setUp() { + registry = new SkillTemplateRegistry(new ObjectMapper()); + registry.load(); + } + + @Test + @DisplayName("ships at least one knowledge template and one prompt template") + void shipsBothTemplateTypes() { + List all = registry.all(); + assertFalse(all.isEmpty(), "expected at least one shipped template"); + assertTrue(all.stream().anyMatch(t -> "knowledge".equals(t.getType())), + "expected at least one type=knowledge template"); + assertTrue(all.stream().anyMatch(t -> "prompt".equals(t.getType())), + "expected at least one type=prompt template"); + } + + @Test + @DisplayName("starter library hits the RFC-091 §2.1 floor of 10 templates") + void starterLibraryFloor() { + // RFC-091 §2.1 期望 10–20 个起步模板。本仓库目前 ship 10 个 v1 + // (tcm-qa / legal-clauses-qa / training-qa / meeting-summarizer / + // crm-assistant / weekly-report / email-summarizer / data-analyst-prompt / + // codex-coding-helper / claude-code-helper)。若降到 10 以下视为回归。 + assertTrue(registry.all().size() >= 10, + "starter library should ship >= 10 templates; got " + registry.all().size()); + } + + @Test + @DisplayName("codex-coding-helper template demonstrates type=acp wiring") + void codexAcpTemplateShape() { + SkillTemplate t = registry.find("codex-coding-helper"); + assertNotNull(t, "codex-coding-helper template missing"); + assertEquals("acp", t.getType()); + assertTrue(t.getSkillMd().contains("type: acp")); + assertTrue(t.getSkillMd().contains("endpoint: codex")); + assertTrue(t.getFields().stream().anyMatch(f -> "system_prefix".equals(f.getKey()))); + } + + @Test + @DisplayName("claude-code-helper mirrors codex template wiring with endpoint=claude-code") + void claudeAcpTemplateShape() { + SkillTemplate t = registry.find("claude-code-helper"); + assertNotNull(t, "claude-code-helper template missing"); + assertEquals("acp", t.getType()); + assertTrue(t.getSkillMd().contains("type: acp")); + assertTrue(t.getSkillMd().contains("endpoint: claude-code")); + } + + @Test + @DisplayName("legal-clauses-qa template exists, knowledge type, kb-picker present") + void legalTemplateShape() { + SkillTemplate t = registry.find("legal-clauses-qa"); + assertNotNull(t); + assertEquals("knowledge", t.getType()); + assertTrue(t.getFields().stream().anyMatch(f -> "kb-picker".equals(f.getType()))); + } + + @Test + @DisplayName("data-analyst-prompt template exposes SQL dialect select") + void dataAnalystTemplateShape() { + SkillTemplate t = registry.find("data-analyst-prompt"); + assertNotNull(t); + assertEquals("prompt", t.getType()); + assertTrue(t.getFields().stream().anyMatch(f -> + "sql_dialect".equals(f.getKey()) && "select".equals(f.getType()))); + } + + @Test + @DisplayName("tcm-qa template exposes kb-picker + skill_name fields") + void tcmTemplateShape() { + SkillTemplate t = registry.find("tcm-qa"); + assertNotNull(t, "tcm-qa template missing"); + assertEquals("knowledge", t.getType()); + assertNotNull(t.getSkillMd()); + assertTrue(t.getSkillMd().contains("{{skill_name}}")); + assertTrue(t.getSkillMd().contains("{{kb_slug}}")); + assertTrue(t.getFields().stream().anyMatch(f -> "kb-picker".equals(f.getType()))); + assertTrue(t.getFields().stream().anyMatch(f -> "skill_name".equals(f.getKey()) && f.isRequired())); + } + + @Test + @DisplayName("meeting-summarizer is a prompt-only template with no kb-picker") + void meetingSummarizerShape() { + SkillTemplate t = registry.find("meeting-summarizer"); + assertNotNull(t); + assertEquals("prompt", t.getType()); + assertFalse(t.getFields().stream().anyMatch(f -> "kb-picker".equals(f.getType()))); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/usage/SkillUsageMigrationTest.java b/mateclaw-server/src/test/java/vip/mate/skill/usage/SkillUsageMigrationTest.java new file mode 100644 index 00000000..36216f2e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/usage/SkillUsageMigrationTest.java @@ -0,0 +1,41 @@ +package vip.mate.skill.usage; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class SkillUsageMigrationTest { + + private static final Path MIGRATIONS = Path.of("src/main/resources/db/migration"); + + @Test + @DisplayName("skill usage table migration uses a version after existing V86 repair migration") + void skillUsageMigrationUsesV87() { + Path h2 = MIGRATIONS.resolve("h2/V87__skill_usage_stat.sql"); + Path mysql = MIGRATIONS.resolve("mysql/V87__skill_usage_stat.sql"); + + assertTrue(Files.exists(h2), "H2 usage migration must be V87 so already-applied V86 databases run it"); + assertTrue(Files.exists(mysql), "MySQL usage migration must be V87 so already-applied V86 databases run it"); + assertFalse(Files.exists(MIGRATIONS.resolve("h2/V86__skill_usage_stat.sql")), + "Do not reuse V86 for usage stats; some installations already applied a different V86"); + assertFalse(Files.exists(MIGRATIONS.resolve("mysql/V86__skill_usage_stat.sql")), + "Do not reuse V86 for usage stats; some installations already applied a different V86"); + } + + @Test + @DisplayName("skill usage migrations create the expected table") + void skillUsageMigrationCreatesExpectedTable() throws Exception { + String h2 = Files.readString(MIGRATIONS.resolve("h2/V87__skill_usage_stat.sql")); + String mysql = Files.readString(MIGRATIONS.resolve("mysql/V87__skill_usage_stat.sql")); + + assertTrue(h2.contains("CREATE TABLE IF NOT EXISTS mate_skill_usage_stat")); + assertTrue(mysql.contains("CREATE TABLE IF NOT EXISTS mate_skill_usage_stat")); + assertTrue(h2.contains("uk_skill_usage_scope")); + assertTrue(mysql.contains("uk_skill_usage_scope")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java new file mode 100644 index 00000000..50e44f8a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java @@ -0,0 +1,149 @@ +package vip.mate.skill.workspace; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.context.ApplicationEventPublisher; +import vip.mate.skill.model.SkillEntity; +import vip.mate.skill.model.SkillFileEntity; +import vip.mate.skill.repository.SkillFileMapper; +import vip.mate.skill.service.SkillFileService; +import vip.mate.skill.service.SkillService; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Tests for {@link SkillFileSyncer} covering the multi-instance scenarios: + * + *

    + *
  • DB has rows, FS is missing them (new node receives shared DB) → + * files materialized to disk.
  • + *
  • FS already current with DB → nothing rewritten.
  • + *
  • DB empty, FS has files (pre-V112 install) → files backfilled into + * canonical store.
  • + *
+ */ +class SkillFileSyncerTest { + + @TempDir + Path tmp; + + private SkillService skillService; + private SkillFileMapper mapper; + private SkillFileService fileService; + private SkillWorkspaceManager workspaceManager; + private SkillFileSyncer syncer; + + @BeforeEach + void setUp() { + skillService = mock(SkillService.class); + mapper = mock(SkillFileMapper.class); + fileService = new SkillFileService(mapper); + SkillWorkspaceProperties props = new SkillWorkspaceProperties(); + props.setRoot(tmp.toString()); + workspaceManager = new SkillWorkspaceManager(props, mock(ApplicationEventPublisher.class)); + syncer = new SkillFileSyncer(skillService, fileService, workspaceManager); + } + + @Test + @DisplayName("DB rows materialize to a missing local cache") + void materializesDbRowsOntoDisk() throws IOException { + SkillEntity skill = newSkill(10L, "demo"); + when(skillService.listSkills()).thenReturn(List.of(skill)); + + // DB has scripts/run.py and references/notes.md but local FS has neither. + when(mapper.selectList(any())).thenReturn(List.of( + newRow(1L, 10L, "scripts/run.py", "print('a')\n"), + newRow(2L, 10L, "references/notes.md", "hello") + )); + + var report = syncer.syncAll(); + + Path workspace = tmp.resolve("demo"); + assertEquals("print('a')\n", Files.readString(workspace.resolve("scripts/run.py"))); + assertEquals("hello", Files.readString(workspace.resolve("references/notes.md"))); + assertEquals(2, report.filesMaterialized()); + assertEquals(0, report.filesAlreadyCurrent()); + assertEquals(0, report.filesBackfilledFromDisk()); + } + + @Test + @DisplayName("FS already in sync with DB → no rewrites") + void skipsAlreadyCurrentFiles() throws IOException { + SkillEntity skill = newSkill(10L, "demo"); + when(skillService.listSkills()).thenReturn(List.of(skill)); + + Path workspace = tmp.resolve("demo"); + Files.createDirectories(workspace.resolve("scripts")); + Files.writeString(workspace.resolve("scripts/run.py"), "stable"); + + when(mapper.selectList(any())).thenReturn(List.of( + newRow(1L, 10L, "scripts/run.py", "stable") + )); + + var report = syncer.syncAll(); + + assertEquals(0, report.filesMaterialized()); + assertEquals(1, report.filesAlreadyCurrent()); + } + + @Test + @DisplayName("FS has files, DB is empty (pre-V112): backfill into DB") + void backfillsFromDiskWhenDbEmpty() throws IOException { + SkillEntity skill = newSkill(10L, "demo"); + when(skillService.listSkills()).thenReturn(List.of(skill)); + + Path workspace = tmp.resolve("demo"); + Files.createDirectories(workspace.resolve("scripts")); + Files.createDirectories(workspace.resolve("references")); + Files.writeString(workspace.resolve("scripts/run.py"), "legacy"); + Files.writeString(workspace.resolve("references/cfg.md"), "old-ref"); + + // selectList call sequence inside syncOne with backfill: + // 1. syncOne reads dbFiles → empty (triggers backfill) + // 2. applyBundleFiles inside backfill reads existing rows → empty (none inserted yet) + // 3. syncOne re-reads dbFiles after backfill → freshly inserted rows + List after = new ArrayList<>(List.of( + newRow(1L, 10L, "scripts/run.py", "legacy"), + newRow(2L, 10L, "references/cfg.md", "old-ref") + )); + when(mapper.selectList(any())).thenReturn(List.of(), List.of(), after); + + var report = syncer.syncAll(); + + assertEquals(2, report.filesBackfilledFromDisk(), + "Both legacy files should be ingested into the canonical store"); + assertEquals(1, report.skillsBackfilled()); + // After backfill, the reread "current" rows match what's already on disk. + assertEquals(2, report.filesAlreadyCurrent()); + verify(mapper, times(2)).insert(any(SkillFileEntity.class)); + } + + private static SkillEntity newSkill(Long id, String name) { + SkillEntity s = new SkillEntity(); + s.setId(id); + s.setName(name); + return s; + } + + private static SkillFileEntity newRow(Long id, Long skillId, String path, String content) { + SkillFileEntity e = new SkillFileEntity(); + e.setId(id); + e.setSkillId(skillId); + e.setFilePath(path); + e.setContent(content); + e.setContentSize(content.getBytes(StandardCharsets.UTF_8).length); + e.setSha256(SkillFileService.sha256Hex(content)); + return e; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillWorkspaceManagerApplyBundleTest.java b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillWorkspaceManagerApplyBundleTest.java new file mode 100644 index 00000000..ccf7fea6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillWorkspaceManagerApplyBundleTest.java @@ -0,0 +1,117 @@ +package vip.mate.skill.workspace; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.context.ApplicationEventPublisher; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * Regression tests for {@link SkillWorkspaceManager#applyBundleFiles}. + * + *

Issue #104: a malformed ZIP that produced an empty {@code scripts} + * map used to wipe pre-existing scripts because the installer ran + * "clean-then-write". Write-then-prune + empty-bundle guard preserves + * existing files when the new bundle has nothing to say about a bucket. + */ +class SkillWorkspaceManagerApplyBundleTest { + + @TempDir + Path tmp; + + private SkillWorkspaceManager manager; + private final String skill = "demo"; + + @BeforeEach + void setUp() { + SkillWorkspaceProperties props = new SkillWorkspaceProperties(); + props.setRoot(tmp.toString()); + ApplicationEventPublisher publisher = mock(ApplicationEventPublisher.class); + manager = new SkillWorkspaceManager(props, publisher); + manager.initWorkspace(skill, "---\nname: demo\n---\nbody\n"); + } + + @Test + @DisplayName("write-then-prune: new files added, removed files pruned") + void writeThenPruneNormalCase() throws IOException { + Path scripts = tmp.resolve(skill).resolve("scripts"); + Files.writeString(scripts.resolve("old.py"), "old"); + Files.writeString(scripts.resolve("keep.py"), "v1"); + + var result = manager.applyBundleFiles(skill, + Map.of(), + Map.of("keep.py", "v2", "new.py", "fresh"), + false); + + assertEquals(2, result.scriptsWritten()); + assertEquals(1, result.scriptsPruned(), "old.py should be pruned"); + assertFalse(result.scriptsPreservedDueToEmptyBundle()); + assertEquals("v2", Files.readString(scripts.resolve("keep.py"))); + assertEquals("fresh", Files.readString(scripts.resolve("new.py"))); + assertFalse(Files.exists(scripts.resolve("old.py"))); + } + + @Test + @DisplayName("empty-bundle guard: existing scripts preserved when new bundle has none") + void emptyBundleGuardPreservesExistingScripts() throws IOException { + Path scripts = tmp.resolve(skill).resolve("scripts"); + Files.writeString(scripts.resolve("run.py"), "important"); + Files.writeString(scripts.resolve("helper.py"), "more important"); + + var result = manager.applyBundleFiles(skill, + Map.of("notes.md", "ref"), + Map.of(), // empty scripts — simulates the issue #104 extractor bug + false); + + assertEquals(0, result.scriptsWritten()); + assertEquals(0, result.scriptsPruned()); + assertTrue(result.scriptsPreservedDueToEmptyBundle(), + "Empty-bundle guard must mark scripts as preserved"); + assertEquals("important", Files.readString(scripts.resolve("run.py")), + "Existing script must NOT be wiped by an empty bundle"); + assertEquals("more important", Files.readString(scripts.resolve("helper.py"))); + } + + @Test + @DisplayName("force=true bypasses empty-bundle guard and prunes everything") + void forceFlagPrunesEvenWhenBundleEmpty() throws IOException { + Path scripts = tmp.resolve(skill).resolve("scripts"); + Files.writeString(scripts.resolve("doomed.py"), "x"); + + var result = manager.applyBundleFiles(skill, + Map.of(), + Map.of(), + true); + + assertFalse(result.scriptsPreservedDueToEmptyBundle()); + assertEquals(1, result.scriptsPruned()); + assertFalse(Files.exists(scripts.resolve("doomed.py"))); + } + + @Test + @DisplayName("references and scripts buckets prune independently") + void bucketsAreIndependent() throws IOException { + Path scripts = tmp.resolve(skill).resolve("scripts"); + Path references = tmp.resolve(skill).resolve("references"); + Files.writeString(scripts.resolve("run.py"), "stay-on-disk"); + Files.writeString(references.resolve("notes.md"), "stale-ref"); + + var result = manager.applyBundleFiles(skill, + Map.of("notes.md", "fresh-ref"), + Map.of(), // empty scripts → preserved + false); + + assertTrue(result.scriptsPreservedDueToEmptyBundle()); + assertFalse(result.referencesPreservedDueToEmptyBundle()); + assertEquals("stay-on-disk", Files.readString(scripts.resolve("run.py"))); + assertEquals("fresh-ref", Files.readString(references.resolve("notes.md"))); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/workspace/bundle/SkillBundleMaterializerTest.java b/mateclaw-server/src/test/java/vip/mate/skill/workspace/bundle/SkillBundleMaterializerTest.java new file mode 100644 index 00000000..3dae256e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/workspace/bundle/SkillBundleMaterializerTest.java @@ -0,0 +1,103 @@ +package vip.mate.skill.workspace.bundle; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.core.io.support.PathMatchingResourcePatternResolver; +import org.springframework.core.io.support.ResourcePatternResolver; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Strategy + materializer round-trip. Uses {@code test-bundles/sample/} + * (in {@code src/test/resources}) as a deterministic fixture so the test + * doesn't depend on whatever real builtin skills happen to ship. + */ +class SkillBundleMaterializerTest { + + private static final String FIXTURE_ROOT = "test-bundles/sample"; + + private final SkillBundleMaterializer materializer = new SkillBundleMaterializer(); + private final ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + + @Test + @DisplayName("verbatim mode copies SKILL.md + scripts + references with subdirs preserved") + void verbatimCopiesEverything(@TempDir Path target) throws IOException { + SkillBundleSource source = new ClasspathBundleSource(resolver, FIXTURE_ROOT); + + SkillBundleMaterializer.Result result = materializer.materialize( + source, target, MaterializeOptions.verbatim()); + + assertEquals(3, result.copied(), "expected SKILL.md + scripts/run.sh + references/notes.md"); + assertEquals(0, result.skipped()); + assertTrue(Files.exists(target.resolve("SKILL.md"))); + assertTrue(Files.exists(target.resolve("scripts/run.sh"))); + assertTrue(Files.exists(target.resolve("references/notes.md"))); + // Spot-check content survived the InputStream round-trip. + assertTrue(Files.readString(target.resolve("scripts/run.sh")) + .contains("hello from sample bundle")); + } + + @Test + @DisplayName("templateOverlay mode skips top-level SKILL.md so the wizard's manifest stays authoritative") + void templateOverlaySkipsSkillMd(@TempDir Path target) throws IOException { + SkillBundleSource source = new ClasspathBundleSource(resolver, FIXTURE_ROOT); + + // Pretend the wizard already wrote its rendered manifest. + Files.writeString(target.resolve("SKILL.md"), "RENDERED_BY_WIZARD"); + + SkillBundleMaterializer.Result result = materializer.materialize( + source, target, MaterializeOptions.templateOverlay()); + + assertEquals(2, result.copied(), "scripts/run.sh + references/notes.md only"); + assertEquals(1, result.skipped(), "top-level SKILL.md should be skipped"); + assertEquals("RENDERED_BY_WIZARD", Files.readString(target.resolve("SKILL.md")), + "wizard-owned SKILL.md must not be overwritten"); + assertTrue(Files.exists(target.resolve("scripts/run.sh"))); + assertTrue(Files.exists(target.resolve("references/notes.md"))); + } + + @Test + @DisplayName("path traversal entries are rejected without writing outside targetDir") + void pathTraversalGuard(@TempDir Path target) throws IOException { + SkillBundleSource malicious = new SkillBundleSource() { + @Override public String origin() { return "test:malicious"; } + @Override public List assets() { + return List.of( + new BundleAsset("../escaped.txt", + () -> new ByteArrayInputStream("nope".getBytes())), + new BundleAsset("ok.txt", + () -> new ByteArrayInputStream("ok".getBytes()))); + } + }; + + SkillBundleMaterializer.Result result = materializer.materialize( + malicious, target, MaterializeOptions.verbatim()); + + assertEquals(1, result.copied(), "only the safe entry should be copied"); + assertEquals(1, result.skipped(), "the .. entry must be skipped"); + assertTrue(Files.exists(target.resolve("ok.txt"))); + assertFalse(Files.exists(target.getParent().resolve("escaped.txt")), + "traversal target must not exist on disk"); + } + + @Test + @DisplayName("creates the target directory when it doesn't yet exist") + void createsTargetDirectory(@TempDir Path tmp) throws IOException { + Path nested = tmp.resolve("a/b/c"); + assertFalse(Files.exists(nested)); + + SkillBundleSource source = new ClasspathBundleSource(resolver, FIXTURE_ROOT); + SkillBundleMaterializer.Result result = materializer.materialize( + source, nested, MaterializeOptions.verbatim()); + + assertTrue(Files.isDirectory(nested)); + assertEquals(3, result.copied()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/AudioMimeTypesTest.java b/mateclaw-server/src/test/java/vip/mate/stt/AudioMimeTypesTest.java new file mode 100644 index 00000000..1f5b39f3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/AudioMimeTypesTest.java @@ -0,0 +1,50 @@ +package vip.mate.stt; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Pinned behaviour for the filename / content-type inference. The pre-fix bug + * was a single hardcoded {@code "audio.ogg"} default that lied about WebM + * content — DashScope inspected the extension and rejected the bytes. These + * tests pin the new contract: filename and content-type stay in sync with the + * real audio format whichever side the caller supplied. + */ +class AudioMimeTypesTest { + + @Test + @DisplayName("resolveFileName: trusts a caller filename with a known extension") + void resolveFileName_trustsKnownExtension() { + assertEquals("clip.mp3", AudioMimeTypes.resolveFileName("clip.mp3", null)); + assertEquals("speech.WAV", AudioMimeTypes.resolveFileName("speech.WAV", null)); + } + + @Test + @DisplayName("resolveFileName: synthesises from content-type when filename is missing") + void resolveFileName_synthesisesFromContentType() { + // The crucial case — frontend sends bare bytes + content-type only. + assertEquals("audio.mp3", AudioMimeTypes.resolveFileName(null, "audio/mpeg")); + assertEquals("audio.wav", AudioMimeTypes.resolveFileName(null, "audio/wav")); + assertEquals("audio.webm", AudioMimeTypes.resolveFileName(null, "audio/webm")); + assertEquals("audio.m4a", AudioMimeTypes.resolveFileName(null, "audio/mp4")); + } + + @Test + @DisplayName("resolveFileName: falls back to wav when both inputs are blank/unknown") + void resolveFileName_fallsBackToWav() { + // WAV is the lowest common denominator every STT provider accepts. + assertEquals("audio.wav", AudioMimeTypes.resolveFileName(null, null)); + assertEquals("audio.wav", AudioMimeTypes.resolveFileName("", "")); + // Unknown extension on filename → re-derive from contentType / fallback. + assertEquals("audio.wav", AudioMimeTypes.resolveFileName("blob.bin", null)); + } + + @Test + @DisplayName("resolveFileName: strips content-type parameters before lookup") + void resolveFileName_handlesContentTypeWithParameters() { + // MediaRecorder emits "audio/webm;codecs=opus" — must not break the lookup. + assertEquals("audio.webm", AudioMimeTypes.resolveFileName(null, "audio/webm;codecs=opus")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/SttServiceTest.java b/mateclaw-server/src/test/java/vip/mate/stt/SttServiceTest.java new file mode 100644 index 00000000..83c74c2f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/SttServiceTest.java @@ -0,0 +1,346 @@ +package vip.mate.stt; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link SttService} — the dispatch + fallback orchestration. + * + *

Pre-fix behavior had two failure paths that were indistinguishable to + * the user (both surfaced as "STT 不可用"): + *

    + *
  1. {@code sttEnabled=false} (the default) — no STT call ever attempted.
  2. + *
  3. No provider had API key configured — silent fallthrough to "no provider".
  4. + *
+ * These tests pin the new behavior: distinct error messages, fallback engages + * when configured, primary's error is preserved when fallback also fails. + */ +class SttServiceTest { + + private SystemSettingService systemSettingService; + + @BeforeEach + void setUp() { + systemSettingService = mock(SystemSettingService.class); + } + + @Test + @DisplayName("transcribe returns clear 'STT 未启用' when sttEnabled is false") + void transcribe_returnsDisabledMessageWhenOff() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttEnabled(false); + when(systemSettingService.getAllSettings()).thenReturn(config); + + SttService svc = new SttService(systemSettingService, registryWith(/* providers */)); + Map result = svc.transcribe(new byte[]{1, 2, 3}, "audio.wav", "audio/wav", null); + + assertFalse((boolean) result.get("success")); + assertTrue(result.get("error").toString().contains("未启用"), + "User must see 'feature is off' rather than a generic provider error"); + } + + @Test + @DisplayName("transcribe returns success from primary provider when it works") + void transcribe_primarySuccess() { + SystemSettingsDTO config = enabledConfig("auto", true); + when(systemSettingService.getAllSettings()).thenReturn(config); + + StubProvider primary = new StubProvider("openai", 100, true, SttResult.success("hello world")); + SttService svc = new SttService(systemSettingService, registryWith(primary)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertTrue((boolean) result.get("success")); + assertEquals("hello world", result.get("text")); + assertEquals(1, primary.callCount.get()); + } + + @Test + @DisplayName("transcribe falls back to next provider when primary fails AND fallback is enabled") + void transcribe_fallsBackWhenEnabled() { + SystemSettingsDTO config = enabledConfig("auto", true); + when(systemSettingService.getAllSettings()).thenReturn(config); + + StubProvider primary = new StubProvider("openai", 100, true, SttResult.failure("HTTP 500")); + StubProvider fallback = new StubProvider("dashscope", 200, true, SttResult.success("叫我 fallback")); + SttService svc = new SttService(systemSettingService, registryWith(primary, fallback)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertTrue((boolean) result.get("success")); + assertEquals("叫我 fallback", result.get("text")); + assertEquals(1, primary.callCount.get(), "primary must still have been tried first"); + assertEquals(1, fallback.callCount.get(), "fallback should kick in only after primary fails"); + } + + @Test + @DisplayName("transcribe does NOT fall back when fallback is disabled") + void transcribe_noFallbackWhenDisabled() { + SystemSettingsDTO config = enabledConfig("auto", false); + when(systemSettingService.getAllSettings()).thenReturn(config); + + StubProvider primary = new StubProvider("openai", 100, true, SttResult.failure("HTTP 500")); + StubProvider candidate = new StubProvider("dashscope", 200, true, SttResult.success("never reached")); + SttService svc = new SttService(systemSettingService, registryWith(primary, candidate)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertFalse((boolean) result.get("success")); + assertEquals(0, candidate.callCount.get(), "fallback must NOT be tried when sttFallbackEnabled=false"); + } + + @Test + @DisplayName("transcribe surfaces all failures when every provider rejects") + void transcribe_allFailedAggregatesErrors() { + SystemSettingsDTO config = enabledConfig("auto", true); + when(systemSettingService.getAllSettings()).thenReturn(config); + + StubProvider p1 = new StubProvider("openai", 100, true, SttResult.failure("HTTP 401")); + StubProvider p2 = new StubProvider("dashscope", 200, true, SttResult.failure("HTTP 400")); + SttService svc = new SttService(systemSettingService, registryWith(p1, p2)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + String error = result.get("error").toString(); + assertFalse((boolean) result.get("success")); + // Both provider IDs must appear so the operator can tell which API + // keys are wrong without grep-ing the server log. + assertTrue(error.contains("openai"), "aggregate error must mention every failed provider"); + assertTrue(error.contains("dashscope")); + assertTrue(error.contains("HTTP 401")); + assertTrue(error.contains("HTTP 400")); + } + + @Test + @DisplayName("transcribe returns actionable hint when no provider has a key configured") + void transcribe_returnsActionableHintWhenNoProviderAvailable() { + SystemSettingsDTO config = enabledConfig("auto", true); + when(systemSettingService.getAllSettings()).thenReturn(config); + + // Neither provider available — the most common real-world failure + // mode. Pre-fix this surfaced as a generic "no provider" with no + // actionable hint pointing the user at the model-management page. + StubProvider p1 = new StubProvider("openai", 100, false, null); + StubProvider p2 = new StubProvider("dashscope", 200, false, null); + SttService svc = new SttService(systemSettingService, registryWith(p1, p2)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + String error = result.get("error").toString(); + assertFalse((boolean) result.get("success")); + assertTrue(error.contains("API Key") || error.contains("模型管理"), + "error message must point the user at the API key configuration UI"); + assertEquals(0, p1.callCount.get()); + assertEquals(0, p2.callCount.get()); + } + + @Test + @DisplayName("Chinese language hint pulls DashScope (Paraformer) above Whisper") + void transcribe_chineseLanguagePrefersDashScope() { + // Stub provider mirrors DashScopeSttProvider's real + // autoDetectOrder(zh) so the routing test pins the actual numbers + // we ship, not arbitrary values. + SystemSettingsDTO config = enabledConfig("auto", true); + config.setLanguage("zh-CN"); + when(systemSettingService.getAllSettings()).thenReturn(config); + + AtomicReference calledFirst = new AtomicReference<>(); + StubProvider openai = recordingStub("openai", 100, calledFirst, + p -> p.startsWith("zh") ? 250 : 100); // mirrors OpenAiSttProvider + StubProvider zhProvider = recordingStub("dashscope", 150, calledFirst, + p -> p.startsWith("zh") ? 60 : 150); + SttService svc = new SttService(systemSettingService, registryWith(openai, zhProvider)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertTrue((boolean) result.get("success")); + assertEquals("dashscope", calledFirst.get(), + "Chinese hint should put the dashscope provider ahead of Whisper"); + } + + @Test + @DisplayName("English language hint keeps Whisper as primary") + void transcribe_englishLanguagePrefersWhisper() { + SystemSettingsDTO config = enabledConfig("auto", true); + config.setLanguage("en-US"); + when(systemSettingService.getAllSettings()).thenReturn(config); + + AtomicReference calledFirst = new AtomicReference<>(); + StubProvider openai = recordingStub("openai", 100, calledFirst, + p -> p != null && p.startsWith("en") ? 80 : 100); + StubProvider zhProvider = recordingStub("dashscope", 150, calledFirst, + p -> p != null && p.startsWith("zh") ? 60 : 150); + SttService svc = new SttService(systemSettingService, registryWith(openai, zhProvider)); + + svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertEquals("openai", calledFirst.get(), + "English hint should keep Whisper primary"); + } + + @Test + @DisplayName("explicit per-call language hint overrides system-settings language") + void transcribe_explicitLanguageOverridesSetting() { + // System UI is English but the caller passes zh — the request-level + // hint must win so a Chinese-speaking user inside an English UI still + // gets the dashscope provider. + SystemSettingsDTO config = enabledConfig("auto", true); + config.setLanguage("en-US"); + when(systemSettingService.getAllSettings()).thenReturn(config); + + AtomicReference calledFirst = new AtomicReference<>(); + StubProvider openai = recordingStub("openai", 100, calledFirst, + p -> p != null && p.startsWith("zh") ? 250 : 80); + StubProvider zhProvider = recordingStub("dashscope", 150, calledFirst, + p -> p != null && p.startsWith("zh") ? 60 : 150); + SttService svc = new SttService(systemSettingService, registryWith(openai, zhProvider)); + + svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", "zh"); + + assertEquals("dashscope", calledFirst.get(), + "Per-call language must override system UI language for routing"); + } + + @Test + @DisplayName("fallback list also respects language ordering") + void transcribe_fallbackOrderRespectsLanguage() { + // Three providers; primary fails. Verify the fallback we hit next is + // the language-preferred one, not whatever default order picked. With + // language=zh: dashscope=60, openai=250, fake=200 → fallback after + // openai (forced primary) should pick dashscope before fake. + SystemSettingsDTO config = enabledConfig("openai", true); // pin openai as primary + config.setLanguage("zh-CN"); + when(systemSettingService.getAllSettings()).thenReturn(config); + + StubProvider openai = new StubProvider("openai", 100, true, SttResult.failure("primary fail")); + StubProvider zhProvider = new StubProvider("dashscope", 150, true, SttResult.success("from dashscope")) { + @Override public int autoDetectOrder(String language) { + return language != null && language.startsWith("zh") ? 60 : 150; + } + }; + StubProvider fake = new StubProvider("fake-cloud", 200, true, SttResult.success("from fake")); + SttService svc = new SttService(systemSettingService, registryWith(openai, zhProvider, fake)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertTrue((boolean) result.get("success")); + assertEquals("from dashscope", result.get("text"), + "Chinese fallback must hit the dashscope provider before language-agnostic fallbacks"); + } + + @Test + @DisplayName("explicit sttProvider selection overrides auto-detect order") + void transcribe_explicitProviderOverridesOrder() { + // User explicitly chose a non-default provider. Registry must honour + // the explicit pick even when another provider has a lower + // autoDetectOrder. + SystemSettingsDTO config = enabledConfig("explicit-pick", true); + when(systemSettingService.getAllSettings()).thenReturn(config); + + AtomicReference calledFirst = new AtomicReference<>(); + StubProvider openai = new StubProvider("openai", 100, true, SttResult.success("from openai")) { + @Override public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + calledFirst.compareAndSet(null, id()); + return super.transcribe(request, config); + } + }; + StubProvider explicitPick = new StubProvider("explicit-pick", 200, true, SttResult.success("from explicit")) { + @Override public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + calledFirst.compareAndSet(null, id()); + return super.transcribe(request, config); + } + }; + SttService svc = new SttService(systemSettingService, registryWith(openai, explicitPick)); + + Map result = svc.transcribe(new byte[]{1, 2, 3}, "a.wav", "audio/wav", null); + + assertTrue((boolean) result.get("success")); + assertEquals("from explicit", result.get("text")); + assertEquals("explicit-pick", calledFirst.get(), "explicit provider must run first"); + } + + /* --------------------------------- helpers --------------------------------- */ + + private static SystemSettingsDTO enabledConfig(String provider, boolean fallback) { + SystemSettingsDTO c = new SystemSettingsDTO(); + c.setSttEnabled(true); + c.setSttProvider(provider); + c.setSttFallbackEnabled(fallback); + return c; + } + + private static SttProviderRegistry registryWith(SttProvider... providers) { + return new SttProviderRegistry(List.of(providers)); + } + + /** + * Variant of {@link StubProvider} that records which stub got hit first + * (so tests can assert ordering) and exposes a custom + * {@link SttProvider#autoDetectOrder(String)} hook for the language- + * routing tests. Returns a successful canned result so the call chain + * doesn't try fallbacks unrelated to the test's intent. + */ + private static StubProvider recordingStub(String id, int defaultOrder, + AtomicReference firstCalled, + java.util.function.Function langOrder) { + return new StubProvider(id, defaultOrder, true, SttResult.success(id + ":ok")) { + @Override + public int autoDetectOrder(String language) { + return langOrder.apply(language); + } + @Override + public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + firstCalled.compareAndSet(null, id()); + return super.transcribe(request, config); + } + }; + } + + /** + * Test double: returns a canned result and counts invocations. Avoids + * pulling in Mockito for the {@link SttProvider} interface — call + * counting is the only behaviour these tests need. + */ + private static class StubProvider implements SttProvider { + private final String id; + private final int order; + private final boolean available; + private final SttResult canned; + final AtomicInteger callCount = new AtomicInteger(); + + StubProvider(String id, int order, boolean available, SttResult canned) { + this.id = id; + this.order = order; + this.available = available; + this.canned = canned; + } + + @Override public String id() { return id; } + @Override public String label() { return id; } + @Override public boolean requiresCredential() { return true; } + @Override public int autoDetectOrder() { return order; } + @Override public boolean isAvailable(SystemSettingsDTO config) { return available; } + + @Override + public SttResult transcribe(SttRequest request, SystemSettingsDTO config) { + callCount.incrementAndGet(); + assertNotNull(canned, "stub for " + id + " was called but no canned result was set"); + return canned; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/WavPcmExtractorTest.java b/mateclaw-server/src/test/java/vip/mate/stt/WavPcmExtractorTest.java new file mode 100644 index 00000000..657788e6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/WavPcmExtractorTest.java @@ -0,0 +1,93 @@ +package vip.mate.stt; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Pinned behaviour for the WAV → raw-PCM helper. + * + *

Why this matters: DashScope's realtime ASR rejects bare WAV with + * "format mismatch" because the first 44 bytes look like garbage when + * interpreted as PCM. {@link WavPcmExtractor} is the chokepoint that + * converts the frontend's WAV blob to the bytes DashScope actually wants. + * Wrong header offset → silent garbage transcripts; wrong sample-rate read + * → audibly distorted. + */ +class WavPcmExtractorTest { + + @Test + @DisplayName("extract: drops the 44-byte canonical header and returns the PCM tail") + void extract_dropsCanonicalHeader() { + // Build a minimal valid WAV: 44-byte header + 8 bytes of fake PCM. + byte[] wav = buildWav(16_000, 16, new byte[]{1, 2, 3, 4, 5, 6, 7, 8}); + byte[] pcm = WavPcmExtractor.extract(wav); + assertArrayEquals(new byte[]{1, 2, 3, 4, 5, 6, 7, 8}, pcm); + } + + @Test + @DisplayName("extract: rejects non-WAV input loudly (no silent garbage)") + void extract_rejectsNonWav() { + // Anything without the RIFF/WAVE magic must fail fast — sending non-WAV + // bytes to DashScope wastes API quota and produces confusing errors. + byte[] junk = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45}; + assertThrows(IllegalArgumentException.class, () -> WavPcmExtractor.extract(junk)); + } + + @Test + @DisplayName("extract: rejects too-short input (no out-of-bounds)") + void extract_rejectsTooShort() { + assertThrows(IllegalArgumentException.class, () -> WavPcmExtractor.extract(new byte[10])); + assertThrows(IllegalArgumentException.class, () -> WavPcmExtractor.extract(null)); + } + + @Test + @DisplayName("sampleRate: reads 16 kHz from the canonical header offset") + void sampleRate_reads16kHz() { + byte[] wav = buildWav(16_000, 16, new byte[8]); + assertEquals(16_000, WavPcmExtractor.sampleRate(wav)); + } + + @Test + @DisplayName("sampleRate: reads 44.1 kHz when Safari-style mic captures at the device default") + void sampleRate_reads44100() { + // Defends against the Safari-on-iOS path where the frontend can't + // force 16 kHz at capture time. We resample on the way out, but the + // server-side helper still needs to read the actual rate. + byte[] wav = buildWav(44_100, 16, new byte[8]); + assertEquals(44_100, WavPcmExtractor.sampleRate(wav)); + } + + /* ------------------------------------------------------------------ */ + /* Helper: build a minimal valid WAV with the canonical 44-byte header.*/ + /* Mirrors the layout produced by mateclaw-ui/src/utils/wavEncoder.ts. */ + /* ------------------------------------------------------------------ */ + private static byte[] buildWav(int sampleRate, int bitsPerSample, byte[] pcmData) { + int dataSize = pcmData.length; + int numChannels = 1; + ByteBuffer buf = ByteBuffer.allocate(44 + dataSize).order(ByteOrder.LITTLE_ENDIAN); + buf.put("RIFF".getBytes()); + buf.putInt(36 + dataSize); + buf.put("WAVE".getBytes()); + buf.put("fmt ".getBytes()); + buf.putInt(16); // fmt chunk size + buf.putShort((short) 1); // PCM + buf.putShort((short) numChannels); + buf.putInt(sampleRate); + buf.putInt(sampleRate * numChannels * (bitsPerSample / 8)); // byte rate + buf.putShort((short) (numChannels * (bitsPerSample / 8))); // block align + buf.putShort((short) bitsPerSample); + buf.put("data".getBytes()); + buf.putInt(dataSize); + buf.put(pcmData); + return buf.array(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/provider/DashScopeSttProviderTest.java b/mateclaw-server/src/test/java/vip/mate/stt/provider/DashScopeSttProviderTest.java new file mode 100644 index 00000000..81dfb888 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/provider/DashScopeSttProviderTest.java @@ -0,0 +1,250 @@ +package vip.mate.stt.provider; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.stt.provider.DashScopeSttProvider.DashScopeSession; + +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit tests for the message-handling state machine of + * {@link DashScopeSttProvider}. The end-to-end WebSocket flow can't be + * exercised without a mock WS server, but the JSON parsing + transcript + * aggregation + latch transitions are fully testable in isolation by + * driving {@link DashScopeSession#handleMessage(String)} directly. + * + *

What these tests guard against: + *

    + *
  • "Two events for the same begin_time" — the second event must + * overwrite the first (interim → final), not append. + * Otherwise you get duplicated text in the final transcript.
  • + *
  • Sentence ordering — multi-sentence speech must come out in + * arrival order regardless of begin_time int values.
  • + *
  • task-failed must surface the error message on both latches so + * the caller doesn't time out for the full 60s budget.
  • + *
+ */ +class DashScopeSttProviderTest { + + private DashScopeSession session; + private DashScopeSttProvider provider; + private ObjectMapper mapper; + + @BeforeEach + void setUp() { + mapper = new ObjectMapper(); + session = new DashScopeSession("test-task-id", mapper); + provider = new DashScopeSttProvider(null, mapper); + } + + @Test + @DisplayName("task-started event releases the start latch") + void taskStarted_releasesLatch() throws Exception { + session.handleMessage(""" + {"header":{"task_id":"test-task-id","event":"task-started"},"payload":{}} + """); + assertTrue(session.awaitTaskStarted(100, TimeUnit.MILLISECONDS)); + assertFalse(session.failed()); + } + + @Test + @DisplayName("result-generated builds transcript text") + void resultGenerated_appendsToTranscript() { + session.handleMessage(""" + {"header":{"task_id":"test-task-id","event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":0,"end_time":1500,"text":"你好"}}}} + """); + assertEquals("你好", session.aggregatedText()); + } + + @Test + @DisplayName("interim updates for the same begin_time overwrite (not append)") + void resultGenerated_overwritesSameBeginTime() { + // Real DashScope behaviour: each sentence starts as a partial + // transcript and gets refined on subsequent events. Both events + // share the same begin_time. If we appended instead of overwriting + // we'd produce "你你好" instead of "你好". + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":0,"end_time":500,"text":"你"}}}} + """); + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":0,"end_time":1500,"text":"你好"}}}} + """); + assertEquals("你好", session.aggregatedText()); + } + + @Test + @DisplayName("multiple sentences concatenate in arrival order") + void resultGenerated_concatenatesSentencesInOrder() { + // Different begin_time → different sentences. Final transcript is + // the concat of all sentences in arrival order (LinkedHashMap). + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":0,"end_time":1500,"text":"你好"}}}} + """); + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":1500,"end_time":3000,"text":"世界"}}}} + """); + assertEquals("你好世界", session.aggregatedText()); + } + + @Test + @DisplayName("task-finished releases the finish latch") + void taskFinished_releasesLatch() throws Exception { + session.handleMessage(""" + {"header":{"event":"task-finished"},"payload":{}} + """); + assertTrue(session.awaitTaskFinished(100, TimeUnit.MILLISECONDS)); + assertFalse(session.failed()); + } + + @Test + @DisplayName("task-failed surfaces error message and unblocks both latches") + void taskFailed_surfacesErrorAndUnblocks() throws Exception { + // Critical for fail-fast behaviour: without this the caller would + // time out after the full 60s OVERALL_TIMEOUT_MS instead of seeing + // the typed error within milliseconds. + session.handleMessage(""" + {"header":{"event":"task-failed", + "error_code":"InvalidParameter.SampleRate", + "error_message":"sample rate not supported"}, + "payload":{}} + """); + assertTrue(session.awaitTaskStarted(100, TimeUnit.MILLISECONDS)); + assertTrue(session.awaitTaskFinished(100, TimeUnit.MILLISECONDS)); + assertTrue(session.failed()); + assertTrue(session.errorMessage().contains("InvalidParameter.SampleRate")); + assertTrue(session.errorMessage().contains("sample rate not supported")); + } + + @Test + @DisplayName("resultEventCount tracks every result-generated event (regardless of text)") + void resultEventCount_isIncrementedPerEvent() { + // Distinguishing "server got our audio but didn't recognise anything" + // (>0 events with empty text) from "server saw 0 audio frames" + // (0 events) is the diagnostic that fingered the chunk-pacing bug. + // Pin the counter behaviour so it doesn't regress. + assertEquals(0, session.resultEventCount()); + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":0,"text":"hi"}}}} + """); + session.handleMessage(""" + {"header":{"event":"result-generated"}, + "payload":{"output":{"sentence":{"begin_time":1000,"text":""}}}} + """); + assertEquals(2, session.resultEventCount()); + } + + @Test + @DisplayName("taskFinishedRaised flips once task-finished arrives — sender uses it to bail out early") + void taskFinishedRaised_signalsSender() { + // The sender loop polls this between paced chunks so a server that + // closes the stream early doesn't make us sleep through the rest of + // the audio for nothing. + assertFalse(session.taskFinishedRaised()); + session.handleMessage(""" + {"header":{"event":"task-finished"},"payload":{}} + """); + assertTrue(session.taskFinishedRaised()); + } + + @Test + @DisplayName("malformed JSON doesn't crash the session") + void malformedJson_isLoggedNotThrown() { + // The session is fed straight from WS frames — corrupt input must + // not bubble up into the WebSocket.Listener and tear down the + // connection. + session.handleMessage("not valid json"); + session.handleMessage("{\"missing_header\":true}"); + // No event released either latch; session is still waiting. + assertFalse(session.failed()); + } + + @Test + @DisplayName("buildRunTask serialises the documented run-task envelope") + void buildRunTask_envelopeShape() throws Exception { + // The wire format is documented by Aliyun — pin it so future + // refactors don't accidentally drop a required field. + String json = provider.buildRunTask( + "abcd1234efgh5678", "paraformer-realtime-v2", 16_000, "zh-CN"); + JsonNode node = mapper.readTree(json); + assertEquals("run-task", node.path("header").path("action").asText()); + assertEquals("abcd1234efgh5678", node.path("header").path("task_id").asText()); + assertEquals("duplex", node.path("header").path("streaming").asText()); + assertEquals("audio", node.path("payload").path("task_group").asText()); + assertEquals("asr", node.path("payload").path("task").asText()); + assertEquals("recognition", node.path("payload").path("function").asText()); + assertEquals("paraformer-realtime-v2", node.path("payload").path("model").asText()); + assertEquals("pcm", node.path("payload").path("parameters").path("format").asText()); + assertEquals(16_000, node.path("payload").path("parameters").path("sample_rate").asInt()); + // language_hints strips the locale: zh-CN → zh + assertEquals("zh", node.path("payload").path("parameters").path("language_hints").get(0).asText()); + } + + @Test + @DisplayName("buildRunTask omits language_hints when language is null") + void buildRunTask_skipsLanguageHintsWhenNull() throws Exception { + // Null language means "let DashScope auto-detect" — sending an + // empty array would flag as a parameter error on some accounts. + String json = provider.buildRunTask("task1", "paraformer-realtime-v2", 16_000, null); + JsonNode node = mapper.readTree(json); + assertTrue(node.path("payload").path("parameters").path("language_hints").isMissingNode(), + "language_hints should be omitted when language is null"); + } + + @Test + @DisplayName("buildFinishTask serialises the documented finish-task envelope") + void buildFinishTask_envelopeShape() throws Exception { + String json = provider.buildFinishTask("abcd1234"); + JsonNode node = mapper.readTree(json); + assertEquals("finish-task", node.path("header").path("action").asText()); + assertEquals("abcd1234", node.path("header").path("task_id").asText()); + assertEquals("duplex", node.path("header").path("streaming").asText()); + // payload.input is required to be an empty object — DashScope + // rejects requests where it's missing or null. + assertTrue(node.path("payload").path("input").isObject()); + } + + @Test + @DisplayName("computePcmPeakRms returns 0,0 on silence; non-zero on synthetic tone") + void computePcmPeakRms_distinguishesSilenceFromSignal() { + // The diagnostic distinguishing "mic captured silence" (peak=0) from + // "DashScope rejected non-empty audio" (peak>0 but 0 events) is a + // critical user-visible signal — pin its math. + byte[] silent = new byte[1000]; // all zeros + int[] silentStats = DashScopeSttProvider.computePcmPeakRms(silent); + assertEquals(0, silentStats[0]); + assertEquals(0, silentStats[1]); + + // Two samples: 0x4000 (16384, positive) and 0xC000 (-16384, negative). + // peak should be 16384, rms = sqrt((16384^2 + 16384^2) / 2) = 16384. + byte[] tone = new byte[]{ + 0x00, 0x40, // 16384 little-endian + 0x00, (byte) 0xC0 // -16384 little-endian + }; + int[] toneStats = DashScopeSttProvider.computePcmPeakRms(tone); + assertEquals(16384, toneStats[0]); + assertEquals(16384, toneStats[1]); + } + + @Test + @DisplayName("autoDetectOrder boosts DashScope on Chinese, defaults otherwise") + void autoDetectOrder_languageRouting() { + assertEquals(60, provider.autoDetectOrder("zh")); + assertEquals(60, provider.autoDetectOrder("zh-CN")); + assertEquals(60, provider.autoDetectOrder("ZH-Hant")); // case-insensitive + assertEquals(150, provider.autoDetectOrder("en-US")); // default order + assertEquals(150, provider.autoDetectOrder(null)); // language unknown + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/provider/OpenAiSttProviderTest.java b/mateclaw-server/src/test/java/vip/mate/stt/provider/OpenAiSttProviderTest.java new file mode 100644 index 00000000..ffc9eac6 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/provider/OpenAiSttProviderTest.java @@ -0,0 +1,170 @@ +package vip.mate.stt.provider; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import vip.mate.exception.MateClawException; +import vip.mate.llm.model.ModelProviderEntity; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.stt.SttRequest; +import vip.mate.stt.SttResult; +import vip.mate.stt.SttTransportConfig; +import vip.mate.stt.transport.OpenAiCompatibleSttTransport; +import vip.mate.system.model.SystemSettingsDTO; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Issue #76: covers the credential-routing thin-wrapper logic — the actual + * wire transport is exercised separately by {@code SttServiceTest} and the + * transport's own pure-logic test. + */ +class OpenAiSttProviderTest { + + private ModelProviderService modelProviderService; + private OpenAiCompatibleSttTransport transport; + private OpenAiSttProvider provider; + + @BeforeEach + void setUp() { + modelProviderService = mock(ModelProviderService.class); + transport = mock(OpenAiCompatibleSttTransport.class); + provider = new OpenAiSttProvider(modelProviderService, transport); + } + + @Test + @DisplayName("Default config routes to id=openai with whisper-1 (legacy compatibility)") + void defaultsToLegacyOpenai() { + SystemSettingsDTO config = new SystemSettingsDTO(); + // Both fields null — the provider should fall back to the legacy defaults. + ModelProviderEntity entity = providerRow("openai", "https://api.openai.com", "sk-test", true); + when(modelProviderService.getProviderConfig("openai")).thenReturn(entity); + when(transport.transcribe(any(), any())).thenReturn(SttResult.success("ok")); + + SttResult result = provider.transcribe(req(), config); + + assertTrue(result.isSuccess()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SttTransportConfig.class); + verify(transport).transcribe(any(), captor.capture()); + SttTransportConfig sent = captor.getValue(); + assertEquals("https://api.openai.com", sent.baseUrl()); + assertEquals("sk-test", sent.apiKey()); + assertEquals("whisper-1", sent.model()); + } + + @Test + @DisplayName("Issue #76: configured providerId routes to that row's baseUrl + key") + void honoursConfiguredProviderId() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttOpenAiCompatProviderId("funasr-internal"); + config.setSttOpenAiCompatModel("paraformer-large"); + ModelProviderEntity entity = providerRow("funasr-internal", + "http://10.0.0.5:9999/v1", "internal-token", false); + when(modelProviderService.getProviderConfig("funasr-internal")).thenReturn(entity); + when(transport.transcribe(any(), any())).thenReturn(SttResult.success("hello")); + + SttResult result = provider.transcribe(req(), config); + + assertTrue(result.isSuccess()); + ArgumentCaptor captor = ArgumentCaptor.forClass(SttTransportConfig.class); + verify(transport).transcribe(any(), captor.capture()); + SttTransportConfig sent = captor.getValue(); + assertEquals("http://10.0.0.5:9999/v1", sent.baseUrl()); + assertEquals("internal-token", sent.apiKey()); + assertEquals("paraformer-large", sent.model()); + } + + @Test + @DisplayName("requireApiKey=false provider with blank key still goes through (self-hosted FunASR)") + void allowsBlankKeyWhenProviderDoesNotRequireOne() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttOpenAiCompatProviderId("funasr-noauth"); + ModelProviderEntity entity = providerRow("funasr-noauth", + "http://10.0.0.5:9999/v1", "", false); + when(modelProviderService.getProviderConfig("funasr-noauth")).thenReturn(entity); + when(transport.transcribe(any(), any())).thenReturn(SttResult.success("ok")); + + SttResult result = provider.transcribe(req(), config); + + assertTrue(result.isSuccess()); + verify(transport).transcribe(any(), any()); + } + + @Test + @DisplayName("requireApiKey=true provider with blank key fails fast with actionable message") + void rejectsBlankKeyWhenRequired() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttOpenAiCompatProviderId("openai"); + ModelProviderEntity entity = providerRow("openai", "https://api.openai.com", "", true); + when(modelProviderService.getProviderConfig("openai")).thenReturn(entity); + + SttResult result = provider.transcribe(req(), config); + + assertFalse(result.isSuccess()); + assertTrue(result.getErrorMessage().contains("openai")); + verifyNoInteractions(transport); + } + + @Test + @DisplayName("Unknown providerId surfaces a typed failure instead of leaking the underlying exception") + void missingProviderIsSurfacedAsTypedFailure() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttOpenAiCompatProviderId("does-not-exist"); + when(modelProviderService.getProviderConfig("does-not-exist")) + .thenThrow(new MateClawException("err.llm.provider_not_found", "missing")); + + SttResult result = provider.transcribe(req(), config); + + assertFalse(result.isSuccess()); + assertTrue(result.getErrorMessage().contains("does-not-exist")); + verifyNoInteractions(transport); + } + + @Test + @DisplayName("Empty baseUrl on provider row falls back to https://api.openai.com") + void emptyBaseUrlFallsBackToOpenAiDefault() { + SystemSettingsDTO config = new SystemSettingsDTO(); + ModelProviderEntity entity = providerRow("openai", "", "sk-test", true); + when(modelProviderService.getProviderConfig("openai")).thenReturn(entity); + when(transport.transcribe(any(), any())).thenReturn(SttResult.success("ok")); + + provider.transcribe(req(), config); + + ArgumentCaptor captor = ArgumentCaptor.forClass(SttTransportConfig.class); + verify(transport).transcribe(any(), captor.capture()); + assertEquals("https://api.openai.com", captor.getValue().baseUrl()); + } + + @Test + @DisplayName("isAvailable defers to the configured provider row, not hard-coded \"openai\"") + void isAvailableHonoursConfiguredProviderId() { + SystemSettingsDTO config = new SystemSettingsDTO(); + config.setSttOpenAiCompatProviderId("siliconflow"); + when(modelProviderService.isProviderConfigured("siliconflow")).thenReturn(true); + + assertTrue(provider.isAvailable(config)); + verify(modelProviderService).isProviderConfigured("siliconflow"); + verify(modelProviderService, never()).isProviderConfigured("openai"); + } + + private static ModelProviderEntity providerRow(String id, String baseUrl, String apiKey, boolean requireApiKey) { + ModelProviderEntity p = new ModelProviderEntity(); + p.setProviderId(id); + p.setName(id); + p.setBaseUrl(baseUrl); + p.setApiKey(apiKey); + p.setRequireApiKey(requireApiKey); + return p; + } + + private static SttRequest req() { + return SttRequest.builder() + .audioData(new byte[]{1, 2, 3}) + .fileName("a.wav") + .contentType("audio/wav") + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/stt/transport/OpenAiCompatibleSttTransportTest.java b/mateclaw-server/src/test/java/vip/mate/stt/transport/OpenAiCompatibleSttTransportTest.java new file mode 100644 index 00000000..688c2cfb --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/stt/transport/OpenAiCompatibleSttTransportTest.java @@ -0,0 +1,62 @@ +package vip.mate.stt.transport; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Issue #76: pure-logic coverage for the path resolver + base URL normalization. + * Network-side behaviour is exercised by the existing {@code SttServiceTest} + * via Mockito stubs on the provider, so this class deliberately stays small + * and unit-only — no Spring, no HTTP. + */ +class OpenAiCompatibleSttTransportTest { + + @Test + @DisplayName("Base URL with no /vN suffix appends /v1/audio/transcriptions") + void resolveAudioPathDefault() { + assertEquals("/v1/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("https://api.openai.com")); + assertEquals("/v1/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("http://10.0.0.5:9999")); + } + + @Test + @DisplayName("Base URL ending in /v1 (lmstudio-style) appends only /audio/transcriptions") + void resolveAudioPathSkipsDoubledVersion() { + assertEquals("/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("http://localhost:1234/v1")); + assertEquals("/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("https://api.siliconflow.cn/v1")); + assertEquals("/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("http://127.0.0.1:9999/v3")); + } + + @Test + @DisplayName("Mid-path /v1 segment is NOT treated as suffix (only end-of-string match)") + void resolveAudioPathRejectsMidPath() { + assertEquals("/v1/audio/transcriptions", + OpenAiCompatibleSttTransport.resolveAudioPath("https://example.com/v1/foo")); + } + + @Test + @DisplayName("Base URL trims trailing slash; null/blank → null sentinel") + void normalizeBaseUrl() { + assertEquals("https://api.openai.com", + OpenAiCompatibleSttTransport.normalizeBaseUrl("https://api.openai.com/")); + assertEquals("https://api.openai.com", + OpenAiCompatibleSttTransport.normalizeBaseUrl(" https://api.openai.com ")); + assertNull(OpenAiCompatibleSttTransport.normalizeBaseUrl("")); + assertNull(OpenAiCompatibleSttTransport.normalizeBaseUrl(" ")); + assertNull(OpenAiCompatibleSttTransport.normalizeBaseUrl(null)); + } + + @Test + @DisplayName("apiMode is the stable family id every profile selects on") + void apiModeIsStable() { + OpenAiCompatibleSttTransport t = new OpenAiCompatibleSttTransport(null); + assertEquals("openai_compatible_audio", t.apiMode()); + assertEquals(OpenAiCompatibleSttTransport.API_MODE, t.apiMode()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/system/featureflag/FeatureFlagServiceTest.java b/mateclaw-server/src/test/java/vip/mate/system/featureflag/FeatureFlagServiceTest.java new file mode 100644 index 00000000..70f7136a --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/system/featureflag/FeatureFlagServiceTest.java @@ -0,0 +1,173 @@ +package vip.mate.system.featureflag; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentMatchers; +import vip.mate.system.featureflag.repository.FeatureFlagMapper; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link FeatureFlagService}. + * + *

The service is exercised against a mocked mapper so the test does not + * depend on a database. All evaluation modes are covered: + * disabled-master-switch, KB-whitelist hit/miss, percentage rollout + * stability, unknown flags, and post-write invalidation. + */ +class FeatureFlagServiceTest { + + private FeatureFlagMapper mapper; + private FeatureFlagService service; + + @BeforeEach + void setUp() { + mapper = mock(FeatureFlagMapper.class); + when(mapper.selectList(ArgumentMatchers.>any())) + .thenReturn(List.of()); + service = new FeatureFlagService(mapper); + service.init(); // primes empty cache + } + + @Test + @DisplayName("Master switch off → isEnabled returns false even with whitelist hit") + void disabled_returnsFalseEverywhere() { + primeFlag(flag("wiki.test.disabled", false, "1,2", null, 100)); + + assertThat(service.isEnabled("wiki.test.disabled")).isFalse(); + assertThat(service.isEnabledForKb("wiki.test.disabled", 1L)).isFalse(); + assertThat(service.isEnabledForKb("wiki.test.disabled", 99L)).isFalse(); + } + + @Test + @DisplayName("Enabled with no whitelist and 0% rollout still returns true (no gate to fail)") + void enabled_noWhitelist_zeroPercent_returnsTrue() { + primeFlag(flag("wiki.test.simple", true, null, null, 0)); + + assertThat(service.isEnabled("wiki.test.simple")).isTrue(); + assertThat(service.isEnabledForKb("wiki.test.simple", 42L)).isTrue(); + } + + @Test + @DisplayName("KB whitelist gates by membership when context has kbId") + void kbWhitelist_membersOnly() { + primeFlag(flag("wiki.test.kbgated", true, "1,2,3", null, 0)); + + assertThat(service.isEnabledForKb("wiki.test.kbgated", 1L)).isTrue(); + assertThat(service.isEnabledForKb("wiki.test.kbgated", 2L)).isTrue(); + assertThat(service.isEnabledForKb("wiki.test.kbgated", 99L)).isFalse(); + } + + @Test + @DisplayName("KB whitelist with no kbId in context allows through (whitelist not applicable)") + void kbWhitelist_noContext_passesThrough() { + primeFlag(flag("wiki.test.kbgated2", true, "1,2,3", null, 0)); + // No kbId in context → kb whitelist not consulted; falls through to default true. + assertThat(service.isEnabled("wiki.test.kbgated2")).isTrue(); + } + + @Test + @DisplayName("User whitelist independently gates by user id") + void userWhitelist_membersOnly() { + primeFlag(flag("wiki.test.usergated", true, null, "10,20", 0)); + + assertThat(service.isEnabledForUser("wiki.test.usergated", 10L)).isTrue(); + assertThat(service.isEnabledForUser("wiki.test.usergated", 99L)).isFalse(); + } + + @Test + @DisplayName("Percentage rollout is deterministic for the same kbId across calls") + void percentageRollout_stableForSameKey() { + primeFlag(flag("wiki.test.rollout", true, null, null, 50)); + + boolean first = service.isEnabledForKb("wiki.test.rollout", 7L); + boolean second = service.isEnabledForKb("wiki.test.rollout", 7L); + boolean third = service.isEnabledForKb("wiki.test.rollout", 7L); + + assertThat(first).isEqualTo(second); + assertThat(second).isEqualTo(third); + } + + @Test + @DisplayName("Percentage rollout: 100% always passes, 0% rollout treated as no gate") + void percentageRollout_boundaryValues() { + primeFlag(flag("wiki.test.always", true, null, null, 100)); + primeFlag(flag("wiki.test.never_gate", true, null, null, 0)); + + // 100% means rollout doesn't actually gate (logic only applies for 0>any())) + .thenThrow(new RuntimeException("DB temporarily unavailable")); + + boolean result = service.isEnabled("wiki.flaky.flag"); + + assertThat(result).isFalse(); + verify(mapper, atLeastOnce()) + .selectOne(ArgumentMatchers.>any()); + } + + // ==================== helpers ==================== + + private FeatureFlagEntity flag(String key, boolean enabled, String kbWhitelist, + String userWhitelist, Integer rollout) { + FeatureFlagEntity f = new FeatureFlagEntity(); + f.setFlagKey(key); + f.setEnabled(enabled); + f.setWhitelistKbIds(kbWhitelist); + f.setWhitelistUserIds(userWhitelist); + f.setRolloutPercent(rollout); + f.setDeleted(0); + return f; + } + + /** Sets up the mapper so that the given flag is returned for both selectOne and selectList. */ + private void primeFlag(FeatureFlagEntity flag) { + when(mapper.selectOne(ArgumentMatchers.>any())) + .thenReturn(flag); + when(mapper.selectList(ArgumentMatchers.>any())) + .thenReturn(List.of(flag)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserLauncherManualProbe.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserLauncherManualProbe.java new file mode 100644 index 00000000..fb81aa7d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/BrowserLauncherManualProbe.java @@ -0,0 +1,69 @@ +package vip.mate.tool.browser; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * Manual end-to-end probe for BrowserLauncher. Not a JUnit test — run via + * {@code mvn -q compile exec:java -Dexec.mainClass=vip.mate.tool.browser.BrowserLauncherManualProbe + * -Dexec.classpathScope=test} + * + *

Exercises the real launcher on the host machine: creates a Playwright instance, + * asks the launcher to pick a strategy, navigates to about:blank, screenshots, and + * reports which strategy succeeded. Exits non-zero if nothing worked. + */ +public final class BrowserLauncherManualProbe { + + public static void main(String[] args) { + System.out.println("=== BrowserLauncher probe ==="); + System.out.println("os.name = " + System.getProperty("os.name")); + System.out.println("user = " + System.getProperty("user.name")); + + BrowserProperties props = new BrowserProperties(); + BrowserLauncher launcher = new BrowserLauncher(props); + + System.out.println("\nCandidate paths on this OS:"); + for (Path p : BrowserLauncher.systemBrowserCandidates()) { + System.out.printf(" %s [%s]%n", p, Files.exists(p) ? "FOUND" : "missing"); + } + + System.out.println("\nDiagnostics report:"); + BrowserDiagnosticsService diag = new BrowserDiagnosticsService(props); + BrowserDiagnosticsService.Report report = diag.run(); + System.out.println(BrowserDiagnosticsService.summarise(report)); + + System.out.println("\nAttempting real launch via Playwright..."); + int exit = 0; + try (Playwright pw = Playwright.create()) { + BrowserLauncher.Result r = launcher.launch(pw, /* headed */ false); + System.out.println("Launch trace:\n" + BrowserLauncher.formatTrace(r.getAttempts())); + if (!r.isSuccess()) { + System.err.println("FAIL: " + r.getFailureSummary()); + exit = 1; + } else { + try (Browser browser = r.getBrowser()) { + Page page = r.getPage(); + page.navigate("about:blank"); + byte[] png = page.screenshot(); + Path shot = Paths.get(System.getProperty("java.io.tmpdir"), + "mateclaw-browser-probe-" + System.currentTimeMillis() + ".png"); + Files.write(shot, png); + System.out.printf("OK via %s: page title='%s', screenshot=%d bytes -> %s%n", + r.getStrategy(), page.title(), png.length, shot); + } + } + } catch (Exception e) { + System.err.println("EXCEPTION: " + e.getClass().getSimpleName() + ": " + e.getMessage()); + e.printStackTrace(); + exit = 2; + } + System.exit(exit); + } + + private BrowserLauncherManualProbe() {} +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/browser/ExternalCdpCleanupProbe.java b/mateclaw-server/src/test/java/vip/mate/tool/browser/ExternalCdpCleanupProbe.java new file mode 100644 index 00000000..4d3e9414 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/browser/ExternalCdpCleanupProbe.java @@ -0,0 +1,162 @@ +package vip.mate.tool.browser; + +import com.microsoft.playwright.Browser; +import com.microsoft.playwright.BrowserContext; +import com.microsoft.playwright.Page; +import com.microsoft.playwright.Playwright; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** + * Manual probe for the EXTERNAL_CDP cleanup path. Mirrors the production launch + + * close logic without going through the private launcher path, so we can verify on + * a real Windows machine that: + * + *

    + *
  1. Chrome spawned with {@code --user-data-dir=} prints "DevTools listening on..." + * to stderr (so {@code readDevToolsUrl} can parse it) — fix #1.
  2. + *
  3. After the session closes (browser disconnect → process destroyForcibly → + * wait → deleteQuietly), the temp profile dir is fully removed — follow-up cleanup fix.
  4. + *
+ * + *

Run via: + * {@code mvn -f mateclaw-server/pom.xml exec:java + * -Dexec.mainClass=vip.mate.tool.browser.ExternalCdpCleanupProbe -Dexec.classpathScope=test} + */ +public final class ExternalCdpCleanupProbe { + + public static void main(String[] args) throws Exception { + System.out.println("=== ExternalCdpCleanupProbe ==="); + System.out.println("os.name = " + System.getProperty("os.name")); + + Path browserBin = pickBrowserBin(); + if (browserBin == null) { + System.err.println("FAIL: no Chrome/Edge/Brave found via systemBrowserCandidates"); + System.exit(1); + return; + } + System.out.println("Browser binary: " + browserBin); + + Path userDataDir = Files.createTempDirectory("mateclaw-cdp-probe-"); + System.out.println("Temp profile: " + userDataDir); + + // Same flag set as BrowserLauncher.tryExternalCdpLaunch. + boolean isWindows = System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win"); + List command = new ArrayList<>(); + command.add(browserBin.toString()); + command.add("--remote-debugging-port=0"); + command.add("--user-data-dir=" + userDataDir.toAbsolutePath()); + command.add("--no-first-run"); + command.add("--no-default-browser-check"); + command.add("--disable-extensions"); + command.add("--disable-background-networking"); + command.add("--headless=new"); + if (isWindows) command.add("--no-sandbox"); + command.add("about:blank"); + + ProcessBuilder pb = new ProcessBuilder(command).redirectErrorStream(false); + Process proc = pb.start(); + System.out.println("Chrome PID: " + proc.pid()); + + String wsUrl = readDevToolsUrl(proc, 20); + System.out.println("Got DevTools: " + wsUrl); + String cdpBase = wsUrl.replaceFirst("^ws://", "http://").replaceFirst("/devtools/.*", ""); + + try (Playwright pw = Playwright.create()) { + Browser browser = pw.chromium().connectOverCDP(cdpBase); + BrowserContext context = browser.contexts().isEmpty() ? browser.newContext() : browser.contexts().get(0); + Page page = context.pages().isEmpty() ? context.newPage() : context.pages().get(0); + page.navigate("about:blank"); + System.out.println("Page loaded: title='" + page.title() + "'"); + + // === Mirror BrowserSession.close() for the EXTERNAL_CDP path === + long t0 = System.currentTimeMillis(); + try { browser.close(); } catch (Exception ignored) {} + try { + List children = proc.descendants().toList(); + System.out.println("Chrome children: " + children.size()); + proc.destroyForcibly(); + for (ProcessHandle h : children) { + try { h.destroyForcibly(); } catch (Exception ignored) {} + } + proc.waitFor(5, TimeUnit.SECONDS); + for (ProcessHandle h : children) { + try { h.onExit().get(2, TimeUnit.SECONDS); } catch (Exception ignored) {} + } + } catch (Exception ignored) {} + BrowserLauncher.deleteQuietly(userDataDir); + long elapsedMs = System.currentTimeMillis() - t0; + System.out.printf("Cleanup ran in %dms%n", elapsedMs); + } + + // Verify the dir is gone. + if (Files.exists(userDataDir)) { + long leftBytes = sizeOf(userDataDir); + long leftFiles; + try (var s = Files.walk(userDataDir)) { leftFiles = s.count() - 1; } + System.err.printf("LEAK: profile dir still exists with %d files / %d bytes -> %s%n", + leftFiles, leftBytes, userDataDir); + System.err.println("Remaining files:"); + try (var s = Files.walk(userDataDir)) { + s.filter(Files::isRegularFile).forEach(p -> + System.err.println(" " + userDataDir.relativize(p))); + } + System.exit(2); + } else { + System.out.println("CLEAN: profile dir fully deleted ✓"); + } + } + + private static Path pickBrowserBin() { + for (Path candidate : BrowserLauncher.systemBrowserCandidates()) { + if (Files.exists(candidate)) return candidate; + } + return null; + } + + private static String readDevToolsUrl(Process proc, int timeoutSeconds) throws Exception { + long deadline = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(timeoutSeconds); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(proc.getErrorStream(), StandardCharsets.UTF_8))) { + StringBuilder accumulated = new StringBuilder(); + String line; + while (System.currentTimeMillis() < deadline) { + if (!reader.ready()) { + if (!proc.isAlive()) { + throw new IllegalStateException("Chrome exited early. stderr=" + accumulated); + } + Thread.sleep(50); + continue; + } + line = reader.readLine(); + if (line == null) break; + accumulated.append(line).append('\n'); + int idx = line.indexOf("DevTools listening on "); + if (idx >= 0) { + return line.substring(idx + "DevTools listening on ".length()).trim(); + } + } + } + throw new IllegalStateException("Timed out waiting for 'DevTools listening on'"); + } + + private static long sizeOf(Path dir) { + try (var s = Files.walk(dir)) { + return s.filter(Files::isRegularFile).mapToLong(p -> { + try { return Files.size(p); } catch (Exception e) { return 0; } + }).sum(); + } catch (Exception e) { + return -1; + } + } + + private ExternalCdpCleanupProbe() {} +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolContextInheritanceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolContextInheritanceTest.java new file mode 100644 index 00000000..3be2997e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolContextInheritanceTest.java @@ -0,0 +1,143 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.model.MessageEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * RFC-03 Lane C2 — covers {@link DelegateAgentTool#formatInheritedContext(List, int)}, + * the helper that builds the parent-context prefix injected into a child + * agent's task when {@code inheritParentContext=true}. + * + *

Behavioral contracts under test: + *

    + *
  • null / empty input → empty string (caller skips prefix injection cleanly).
  • + *
  • system messages are dropped — the child has its own system prompt + * and parent's identity-shaping instructions don't transfer.
  • + *
  • blank content is filtered.
  • + *
  • per-message char limit truncates; truncation marker exposes how + * many chars were dropped so debugging long-tool-result cases is + * straightforward.
  • + *
  • role labels are uppercased for distinct visual blocks in the + * child's system context.
  • + *
+ */ +class DelegateAgentToolContextInheritanceTest { + + private static MessageEntity msg(String role, String content) { + MessageEntity m = new MessageEntity(); + m.setRole(role); + m.setContent(content); + return m; + } + + @Test + @DisplayName("null input → empty prefix (caller skips injection)") + void nullInputReturnsEmpty() { + assertEquals("", DelegateAgentTool.formatInheritedContext(null, 1000)); + } + + @Test + @DisplayName("empty input → empty prefix") + void emptyInputReturnsEmpty() { + assertEquals("", DelegateAgentTool.formatInheritedContext(List.of(), 1000)); + } + + @Test + @DisplayName("only system messages → empty prefix (system role is filtered)") + void onlySystemMessagesReturnEmpty() { + List messages = List.of( + msg("system", "You are a helpful assistant."), + msg("system", "Always respond in JSON.") + ); + assertEquals("", DelegateAgentTool.formatInheritedContext(messages, 1000)); + } + + @Test + @DisplayName("blank-content messages are filtered") + void blankContentFiltered() { + List messages = List.of( + msg("user", ""), + msg("user", " "), + msg("user", "real question?") + ); + String prefix = DelegateAgentTool.formatInheritedContext(messages, 1000); + // Only one usable message after filtering. + assertTrue(prefix.contains("(1 message)")); + assertTrue(prefix.contains("USER: real question?")); + } + + @Test + @DisplayName("happy path — alternating dialogue is formatted in order with role labels") + void typicalDialogueFormatted() { + List messages = List.of( + msg("user", "What is context inheritance?"), + msg("assistant", "Context inheritance is the follow-up fix."), + msg("user", "Tell me about how it works specifically."), + msg("assistant", "It inherits parent context into child agents.") + ); + + String prefix = DelegateAgentTool.formatInheritedContext(messages, 1000); + + assertTrue(prefix.startsWith("--- Parent conversation recent context (4 messages) ---")); + assertTrue(prefix.endsWith("--- End of context ---")); + // Role label uppercase + colon-space separator, in original order. + int userIdx = prefix.indexOf("USER: What is context inheritance?"); + int asstIdx = prefix.indexOf("ASSISTANT: Context inheritance is the follow-up"); + int user2Idx = prefix.indexOf("USER: Tell me about how it works"); + assertTrue(userIdx > 0); + assertTrue(asstIdx > userIdx, "messages must preserve chronological order"); + assertTrue(user2Idx > asstIdx, "messages must preserve chronological order"); + } + + @Test + @DisplayName("singular vs plural — '1 message' not '1 messages'") + void grammaticalNumber() { + String oneMsg = DelegateAgentTool.formatInheritedContext( + List.of(msg("user", "hi")), 1000); + assertTrue(oneMsg.contains("(1 message)"), "header must say '1 message': " + oneMsg); + assertFalse(oneMsg.contains("(1 messages)")); + } + + @Test + @DisplayName("oversized message body is truncated with explicit dropped-chars marker") + void oversizedTruncated() { + String longBody = "x".repeat(2000); + List messages = List.of(msg("user", longBody)); + + String prefix = DelegateAgentTool.formatInheritedContext(messages, 100); + + // Body kept = 100 chars. Marker mentions dropped chars (1900) so + // anyone debugging "why is context cut off" sees the exact size. + assertTrue(prefix.contains("[truncated, 1900 chars omitted]"), + "truncation marker missing or wrong char count: " + prefix); + // Marker must be appended, not prefixed; first usable char is still the body. + assertTrue(prefix.contains("USER: " + "x".repeat(100) + "...")); + } + + @Test + @DisplayName("system messages mixed with dialogue → only dialogue survives") + void systemMessagesFilteredFromMixedConversation() { + List messages = List.of( + msg("system", "Hidden system prompt"), + msg("user", "Hi"), + msg("assistant", "Hello!"), + msg("system", "Another hidden instruction") + ); + + String prefix = DelegateAgentTool.formatInheritedContext(messages, 1000); + + assertFalse(prefix.contains("Hidden system prompt"), + "system role must be filtered to avoid leaking parent identity instructions"); + assertFalse(prefix.contains("Another hidden instruction")); + assertTrue(prefix.contains("USER: Hi")); + assertTrue(prefix.contains("ASSISTANT: Hello!")); + assertTrue(prefix.contains("(2 messages)")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolDenyListTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolDenyListTest.java new file mode 100644 index 00000000..67e24718 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolDenyListTest.java @@ -0,0 +1,160 @@ +package vip.mate.tool.builtin; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.agent.AgentService; +import vip.mate.agent.delegation.SubagentRegistry; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.audit.service.AuditEventService; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.workspace.conversation.ConversationService; + +import java.lang.reflect.Field; +import java.util.List; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Coverage for the deny-list expansion + spawn-pause integration on + * {@link DelegateAgentTool}. Builds the tool by hand so we can poke + * private final fields without spinning up Mockito's full {@code @InjectMocks} + * machinery. + */ +class DelegateAgentToolDenyListTest { + + private DelegateAgentTool tool; + private SubagentRegistry registry; + private AgentMapper agentMapper; + + @BeforeAll + static void initMyBatisPlusCache() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new org.apache.ibatis.session.Configuration(), ""), + AgentEntity.class); + } + + @BeforeEach + void setUp() { + AgentService agentService = mock(AgentService.class); + agentMapper = mock(AgentMapper.class); + ChatStreamTracker streamTracker = mock(ChatStreamTracker.class); + ConversationService conversationService = mock(ConversationService.class); + ObjectMapper objectMapper = new ObjectMapper(); + registry = new SubagentRegistry(); + AuditEventService auditEventService = mock(AuditEventService.class); + + tool = new DelegateAgentTool(agentService, agentMapper, streamTracker, conversationService, + objectMapper, registry, auditEventService); + } + + @AfterEach + void cleanup() { + while (DelegationContext.currentDepth() > 0) { + DelegationContext.exit(); + } + ToolExecutionContext.clear(); + } + + @Test + @DisplayName("Default deny set covers recursion guards and memory writers; no shell/IM names") + void defaultDenyListShape() { + Set defaults = DelegateAgentTool.DEFAULT_CHILD_DENIED_TOOLS; + // Recursion guards. + assertThat(defaults).contains("delegateToAgent", "delegateParallel", "listAvailableAgents"); + // Memory writers (canonical Spring AI tool method names — do not include + // any speculative names that would silently no-op). + assertThat(defaults).contains("remember", "remember_structured", "forget_structured"); + // Shell stays out by design — see comment on DEFAULT_CHILD_DENIED_TOOLS. + assertThat(defaults).doesNotContain("execute_shell_command"); + } + + @Test + @DisplayName("Operator-supplied additions merge into the effective deny list") + void additionalDeniedToolsMergeWithDefaults() throws Exception { + injectAdditional(List.of("custom_tool", "another_tool")); + + Set effective = tool.deniedToolsForChild(); + + assertThat(effective).containsAll(DelegateAgentTool.DEFAULT_CHILD_DENIED_TOOLS); + assertThat(effective).contains("custom_tool", "another_tool"); + // Defaults stay untouched — we returned a fresh merged set. + assertThat(DelegateAgentTool.DEFAULT_CHILD_DENIED_TOOLS).doesNotContain("custom_tool"); + } + + @Test + @DisplayName("Empty additional list returns the default set unchanged") + void emptyAdditionalReturnsDefault() throws Exception { + injectAdditional(List.of()); + assertThat(tool.deniedToolsForChild()).isEqualTo(DelegateAgentTool.DEFAULT_CHILD_DENIED_TOOLS); + + injectAdditional(null); + assertThat(tool.deniedToolsForChild()).isEqualTo(DelegateAgentTool.DEFAULT_CHILD_DENIED_TOOLS); + } + + @Test + @DisplayName("Blank entries in additional list are ignored") + void blankEntriesIgnored() throws Exception { + injectAdditional(List.of("", " ", "real_tool")); + Set effective = tool.deniedToolsForChild(); + assertThat(effective).contains("real_tool"); + assertThat(effective).doesNotContain(""); + assertThat(effective).doesNotContain(" "); + } + + @Test + @DisplayName("delegateToAgent short-circuits when the parent conversation is spawn-paused") + void delegateToAgentRespectsSpawnPause() { + // Set up a real agent the lookup will return so we'd otherwise fall + // through to child execution. The short-circuit must beat that. + AgentEntity agent = new AgentEntity(); + agent.setId(1L); + agent.setName("Worker"); + agent.setEnabled(true); + agent.setWorkspaceId(1L); + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(agent); + + ToolExecutionContext.set("parent-conv", "alice"); + registry.setSpawnPaused("parent-conv", true); + + String result = tool.delegateToAgent("Worker", "do thing", null, null); + + assertThat(result).contains("Spawning paused"); + // No child registered when the spawn is rejected. + assertThat(registry.snapshot("parent-conv")).isEmpty(); + } + + @Test + @DisplayName("delegateParallel short-circuits when the parent conversation is spawn-paused") + void delegateParallelRespectsSpawnPause() { + ToolExecutionContext.set("parent-conv", "alice"); + registry.setSpawnPaused("parent-conv", true); + + String result = tool.delegateParallel( + "[{\"agentName\":\"Worker\",\"task\":\"task1\"}]", null); + + assertThat(result).contains("Spawning paused"); + assertThat(registry.snapshot("parent-conv")).isEmpty(); + } + + /** + * Inject the {@code additionalDeniedTools} field bypassing Spring's + * {@code @Value} binding so the test can drive merge logic deterministically. + */ + private void injectAdditional(List values) throws Exception { + Field f = DelegateAgentTool.class.getDeclaredField("additionalDeniedTools"); + f.setAccessible(true); + f.set(tool, values); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolTest.java new file mode 100644 index 00000000..a8e6d1ec --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateAgentToolTest.java @@ -0,0 +1,266 @@ +package vip.mate.tool.builtin; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.agent.AgentService; +import vip.mate.agent.delegation.SubagentRegistry; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.audit.service.AuditEventService; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.workspace.conversation.ConversationService; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * Unit tests for {@link DelegateAgentTool}. + * Covers: parallel timeout returns explicit error, partial completion, + * and agent-not-found returns readable error. + */ +@ExtendWith(MockitoExtension.class) +class DelegateAgentToolTest { + + @Mock AgentService agentService; + @Mock AgentMapper agentMapper; + @Mock ChatStreamTracker streamTracker; + @Mock ConversationService conversationService; + @Mock AuditEventService auditEventService; + @Spy SubagentRegistry subagentRegistry = new SubagentRegistry(); + + @InjectMocks DelegateAgentTool delegateAgentTool; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeAll + static void initMyBatisPlusCache() { + // Initialize MyBatis Plus lambda cache for AgentEntity so LambdaQueryWrapper works in unit tests + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new org.apache.ibatis.session.Configuration(), ""), + AgentEntity.class); + } + + @BeforeEach + void setUp() throws Exception { + // Inject the real ObjectMapper into the tool via reflection + // (Lombok @RequiredArgsConstructor includes final fields, but ObjectMapper is final) + var field = DelegateAgentTool.class.getDeclaredField("objectMapper"); + field.setAccessible(true); + field.set(delegateAgentTool, objectMapper); + + // Production default is 300 s (configured via @Value) — too long for + // unit tests that simulate a stuck child via Thread.sleep. Force a + // short budget so the timeout assertions fire quickly. Picked 3 s as + // a balance: long enough to mask single-digit-ms scheduling jitter on + // CI, short enough that a hanging test fails fast. + var timeoutField = DelegateAgentTool.class.getDeclaredField("parallelTimeoutSeconds"); + timeoutField.setAccessible(true); + timeoutField.setInt(delegateAgentTool, 3); + } + + @AfterEach + void cleanup() { + while (DelegationContext.currentDepth() > 0) { + DelegationContext.exit(); + } + ToolExecutionContext.clear(); + } + + // ===== delegateToAgent: agent not found ===== + + @Test + @DisplayName("delegateToAgent returns readable error when agent not found") + void delegateToAgentNotFound() { + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + when(agentMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(java.util.List.of()); + + String result = delegateAgentTool.delegateToAgent("NonExistentAgent", "do something", null, null); + + assertTrue(result.contains("NonExistentAgent"), "Should mention the missing agent name"); + assertTrue(result.contains("[错误]") || result.contains("未找到"), "Should indicate an error"); + } + + @Test + @DisplayName("delegateToAgent returns error when agentName is blank") + void delegateToAgentBlankName() { + when(agentMapper.selectList(any(LambdaQueryWrapper.class))).thenReturn(java.util.List.of()); + + String result = delegateAgentTool.delegateToAgent("", "do something", null, null); + + assertTrue(result.contains("[错误]"), "Should indicate an error for blank name"); + } + + @Test + @DisplayName("delegateToAgent returns error when task is blank") + void delegateToAgentBlankTask() { + String result = delegateAgentTool.delegateToAgent("SomeAgent", "", null, null); + + assertTrue(result.contains("[错误]"), "Should indicate an error for blank task"); + } + + // ===== delegateToAgent: depth limit ===== + + @Test + @DisplayName("delegateToAgent rejects when delegation depth reaches limit") + void delegateToAgentDepthLimit() { + // Push depth to MAX_DELEGATION_DEPTH (3) + DelegationContext.enter("a", null); + DelegationContext.enter("b", null); + DelegationContext.enter("c", null); + + String result = delegateAgentTool.delegateToAgent("SomeAgent", "task", null, null); + + assertTrue(result.contains("上限"), "Should mention the depth limit"); + } + + // ===== delegateParallel: invalid JSON ===== + + @Test + @DisplayName("delegateParallel returns error for malformed JSON input") + void delegateParallelBadJson() { + String result = delegateAgentTool.delegateParallel("not valid json", null); + + assertTrue(result.contains("[错误]"), "Should indicate parse error"); + assertTrue(result.contains("JSON"), "Should mention JSON"); + } + + // ===== delegateParallel: empty task list ===== + + @Test + @DisplayName("delegateParallel returns error for empty task list") + void delegateParallelEmptyList() { + String result = delegateAgentTool.delegateParallel("[]", null); + + assertTrue(result.contains("[错误]"), "Should indicate empty list error"); + } + + // ===== delegateParallel: all agents not found ===== + + @Test + @DisplayName("delegateParallel returns error when all agents are not found") + void delegateParallelAllAgentsNotFound() { + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(null); + + String json = "[{\"agentName\":\"Missing1\",\"task\":\"task1\"},{\"agentName\":\"Missing2\",\"task\":\"task2\"}]"; + String result = delegateAgentTool.delegateParallel(json, null); + + assertTrue(result.contains("[错误]"), "Should indicate error"); + assertTrue(result.contains("校验失败"), "Should mention validation failure"); + } + + // ===== delegateParallel: timeout returns explicit error ===== + + @Test + @DisplayName("delegateParallel returns timeout error for slow child agents") + void delegateParallelTimeout() { + AgentEntity agent = new AgentEntity(); + agent.setId(1L); + agent.setName("SlowAgent"); + agent.setEnabled(true); + agent.setWorkspaceId(1L); + + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(agent); + when(streamTracker.isRunning(any())).thenReturn(false); + + // Simulate a child agent that takes longer than the test budget (3 s). + // 10 s is plenty: parent times out at 3 s and abandons the child, then + // the test thread returns immediately. The orphan keeps sleeping on a + // virtual thread until JVM teardown — that's the same behavior as + // production (cancel is best-effort). + when(agentService.chat(anyLong(), anyString(), anyString(), any())).thenAnswer(invocation -> { + Thread.sleep(10_000); + return "should not reach here"; + }); + + // Set a conversationId so resolveParentConversationId works + ToolExecutionContext.set("parent-conv", "admin"); + + String json = "[{\"agentName\":\"SlowAgent\",\"task\":\"slow task\"}]"; + String result = delegateAgentTool.delegateParallel(json, null); + + // The result should contain a timeout error, not hang for 300s + assertTrue(result.contains("超时") || result.contains("timeout") || result.contains("✗"), + "Should contain timeout indicator in result: " + result); + } + + // ===== delegateParallel: exceeds max children ===== + + @Test + @DisplayName("delegateParallel rejects when exceeding max parallel children") + void delegateParallelExceedsMax() { + // MAX_PARALLEL_CHILDREN is 8 — send 9 to trip the guard. + StringBuilder sb = new StringBuilder("["); + for (int i = 1; i <= 9; i++) { + if (i > 1) sb.append(','); + sb.append("{\"agentName\":\"A").append(i).append("\",\"task\":\"t").append(i).append("\"}"); + } + sb.append("]"); + + String result = delegateAgentTool.delegateParallel(sb.toString(), null); + + assertTrue(result.contains("[错误]"), "Should indicate error for too many tasks"); + assertTrue(result.contains("最多"), "Should mention the limit"); + } + + // ===== delegateParallel: partial completion + partial timeout (mixed case) ===== + + @Test + @DisplayName("delegateParallel returns partial results: one fast success + one timeout") + void delegateParallelPartialCompletionPartialTimeout() { + AgentEntity fastAgent = new AgentEntity(); + fastAgent.setId(10L); + fastAgent.setName("FastAgent"); + fastAgent.setEnabled(true); + fastAgent.setWorkspaceId(1L); + + AgentEntity slowAgent = new AgentEntity(); + slowAgent.setId(11L); + slowAgent.setName("SlowAgent"); + slowAgent.setEnabled(true); + slowAgent.setWorkspaceId(1L); + + // Return correct agent per sequential selectOne calls + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(fastAgent) + .thenReturn(slowAgent); + when(streamTracker.isRunning(any())).thenReturn(false); + + // FastAgent completes immediately + when(agentService.chat(eq(10L), anyString(), anyString(), any())) + .thenReturn("Fast result completed successfully"); + + // SlowAgent blocks longer than the (test-overridden) 3 s budget. + when(agentService.chat(eq(11L), anyString(), anyString(), any())).thenAnswer(invocation -> { + Thread.sleep(10_000); + return "should not reach here"; + }); + + ToolExecutionContext.set("parent-mixed", "admin"); + + String json = "[{\"agentName\":\"FastAgent\",\"task\":\"quick task\"},{\"agentName\":\"SlowAgent\",\"task\":\"slow task\"}]"; + String result = delegateAgentTool.delegateParallel(json, null); + + // FastAgent's result should be preserved + assertTrue(result.contains("FastAgent"), "Should mention FastAgent"); + assertTrue(result.contains("Fast result completed successfully") || result.contains("✓"), + "Should contain successful result from FastAgent: " + result); + + // SlowAgent should have a timeout error + assertTrue(result.contains("SlowAgent"), "Should mention SlowAgent"); + assertTrue(result.contains("超时") || result.contains("✗"), + "Should contain timeout indicator for SlowAgent: " + result); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateEventSequenceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateEventSequenceTest.java new file mode 100644 index 00000000..bdf04eb2 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegateEventSequenceTest.java @@ -0,0 +1,261 @@ +package vip.mate.tool.builtin; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import com.baomidou.mybatisplus.core.metadata.TableInfoHelper; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.ibatis.builder.MapperBuilderAssistant; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.Spy; +import vip.mate.agent.AgentService; +import vip.mate.agent.delegation.SubagentRegistry; +import vip.mate.agent.model.AgentEntity; +import vip.mate.agent.repository.AgentMapper; +import vip.mate.audit.service.AuditEventService; +import vip.mate.channel.web.ChatStreamTracker; +import vip.mate.workspace.conversation.ConversationService; + +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.BiConsumer; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +/** + * Minimal E2E-style test verifying the delegation event sequence: + * delegation_start → delegation_progress → delegation_end. + *

+ * To cover delegation_progress, the test captures the relay listener registered via + * {@code addEventRelay} and simulates child events during {@code agentService.chat()}, + * triggering the relay path that broadcasts progress to the parent conversation. + */ +@ExtendWith(MockitoExtension.class) +class DelegateEventSequenceTest { + + @Mock AgentService agentService; + @Mock AgentMapper agentMapper; + @Mock ChatStreamTracker streamTracker; + @Mock ConversationService conversationService; + @Mock AuditEventService auditEventService; + @Spy SubagentRegistry subagentRegistry = new SubagentRegistry(); + + @InjectMocks DelegateAgentTool delegateAgentTool; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @BeforeAll + static void initMyBatisPlusCache() { + TableInfoHelper.initTableInfo( + new MapperBuilderAssistant(new org.apache.ibatis.session.Configuration(), ""), + AgentEntity.class); + } + + @BeforeEach + void setUp() throws Exception { + var field = DelegateAgentTool.class.getDeclaredField("objectMapper"); + field.setAccessible(true); + field.set(delegateAgentTool, objectMapper); + } + + @AfterEach + void cleanup() { + while (DelegationContext.currentDepth() > 0) { + DelegationContext.exit(); + } + ToolExecutionContext.clear(); + } + + private AgentEntity makeAgent(Long id, String name) { + AgentEntity agent = new AgentEntity(); + agent.setId(id); + agent.setName(name); + agent.setEnabled(true); + agent.setWorkspaceId(1L); + agent.setAgentType("react"); + return agent; + } + + // ===== Full sequence: delegation_start → delegation_progress → delegation_end ===== + + @Test + @DisplayName("Single delegation produces start → progress → end event sequence") + void singleDelegationFullEventSequence() { + AgentEntity target = makeAgent(100L, "HelperAgent"); + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(target); + + String parentConvId = "parent-conv-123"; + ToolExecutionContext.set(parentConvId, "admin"); + when(streamTracker.isRunning(parentConvId)).thenReturn(true); + + // Capture the relay listener so we can simulate child events + AtomicReference> relayRef = new AtomicReference<>(); + // Single + parallel delegation now route the relay through the batched API. + when(streamTracker.addBatchedEventRelay(anyString(), anyString(), anyInt(), anyLong(), any())) + .thenAnswer(invocation -> { + relayRef.set(invocation.getArgument(4)); + return (Runnable) () -> {}; + }); + + // During chat(), simulate the child broadcasting a tool_call_started event + when(agentService.chat(eq(100L), eq("summarize the report"), anyString(), any())) + .thenAnswer(invocation -> { + // The relay listener should have been registered by now — fire it + BiConsumer relay = relayRef.get(); + assertNotNull(relay, "Relay should be registered before child chat starts"); + relay.accept("tool_call_started", "{\"name\":\"searchWeb\"}"); + relay.accept("tool_call_completed", "{\"name\":\"searchWeb\",\"success\":true}"); + return "The report shows growth of 15% YoY."; + }); + + // Act + String result = delegateAgentTool.delegateToAgent("HelperAgent", "summarize the report", null, null); + + // Assert: result is successful + assertTrue(result.contains("15%"), "Should contain the child's response"); + + // Capture all broadcastObject calls + ArgumentCaptor convIdCaptor = ArgumentCaptor.forClass(String.class); + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(String.class); + verify(streamTracker, atLeast(3)).broadcastObject( + convIdCaptor.capture(), eventCaptor.capture(), any()); + + List eventNames = eventCaptor.getAllValues(); + + // Verify full sequence: start → progress(es) → end + assertTrue(eventNames.size() >= 3, + "Should have at least 3 events (start + progress + end), got: " + eventNames); + assertEquals("delegation_start", eventNames.get(0), + "First event should be delegation_start"); + + // There should be at least one delegation_progress between start and end + List middle = eventNames.subList(1, eventNames.size() - 1); + assertTrue(middle.contains("delegation_progress"), + "Should have delegation_progress between start and end, got: " + eventNames); + + assertEquals("delegation_end", eventNames.get(eventNames.size() - 1), + "Last event should be delegation_end"); + + // All events target the parent conversation + for (String convId : convIdCaptor.getAllValues()) { + assertEquals(parentConvId, convId, "Events should target parent conversation"); + } + } + + // ===== Parallel delegation event sequence ===== + + @Test + @DisplayName("Parallel delegation broadcasts delegation_start and delegation_end with parallel=true") + void parallelDelegationEventSequence() { + AgentEntity agentA = makeAgent(101L, "AgentA"); + AgentEntity agentB = makeAgent(102L, "AgentB"); + + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))) + .thenReturn(agentA) + .thenReturn(agentB); + + String parentConvId = "parent-parallel-456"; + ToolExecutionContext.set(parentConvId, "admin"); + when(streamTracker.isRunning(parentConvId)).thenReturn(true); + when(streamTracker.addBatchedEventRelay(anyString(), anyString(), anyInt(), anyLong(), any())) + .thenReturn(() -> {}); + + when(agentService.chat(eq(101L), anyString(), anyString(), any())).thenReturn("Result A"); + when(agentService.chat(eq(102L), anyString(), anyString(), any())).thenReturn("Result B"); + + String json = "[{\"agentName\":\"AgentA\",\"task\":\"task A\"},{\"agentName\":\"AgentB\",\"task\":\"task B\"}]"; + + // Act + String result = delegateAgentTool.delegateParallel(json, null); + + assertTrue(result.contains("AgentA"), "Should mention AgentA"); + assertTrue(result.contains("AgentB"), "Should mention AgentB"); + + // Capture events + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(String.class); + verify(streamTracker, atLeast(2)).broadcastObject( + eq(parentConvId), eventCaptor.capture(), any()); + + List eventNames = eventCaptor.getAllValues(); + assertEquals("delegation_start", eventNames.get(0), "First event should be delegation_start"); + assertEquals("delegation_end", eventNames.get(eventNames.size() - 1), + "Last event should be delegation_end"); + } + + // ===== No events when parent inactive ===== + + @Test + @DisplayName("No events are broadcast when parent conversation is not active") + void noEventsWhenParentInactive() { + AgentEntity target = makeAgent(200L, "QuietAgent"); + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(target); + + ToolExecutionContext.set("inactive-parent", "admin"); + when(streamTracker.isRunning("inactive-parent")).thenReturn(false); + + when(agentService.chat(eq(200L), anyString(), anyString(), any())).thenReturn("done"); + + // Act + delegateAgentTool.delegateToAgent("QuietAgent", "quiet task", null, null); + + // Assert: no events broadcast, no relay registered + verify(streamTracker, never()).broadcastObject(anyString(), anyString(), any()); + verify(streamTracker, never()).addEventRelay(anyString(), any()); + verify(streamTracker, never()) + .addBatchedEventRelay(anyString(), anyString(), anyInt(), anyLong(), any()); + } + + // ===== Relay only forwards recognized event types ===== + + @Test + @DisplayName("Relay ignores unrecognized event types, only forwards tool_call_started/completed/phase") + void relayFiltersEventTypes() { + AgentEntity target = makeAgent(300L, "FilterAgent"); + when(agentMapper.selectOne(any(LambdaQueryWrapper.class))).thenReturn(target); + + String parentConvId = "parent-filter-789"; + ToolExecutionContext.set(parentConvId, "admin"); + when(streamTracker.isRunning(parentConvId)).thenReturn(true); + + AtomicReference> relayRef = new AtomicReference<>(); + // Single + parallel delegation now route the relay through the batched API. + when(streamTracker.addBatchedEventRelay(anyString(), anyString(), anyInt(), anyLong(), any())) + .thenAnswer(invocation -> { + relayRef.set(invocation.getArgument(4)); + return (Runnable) () -> {}; + }); + + when(agentService.chat(eq(300L), anyString(), anyString(), any())) + .thenAnswer(invocation -> { + BiConsumer relay = relayRef.get(); + // These should produce delegation_progress: + relay.accept("tool_call_started", "{\"name\":\"search\"}"); + relay.accept("phase", "{\"phase\":\"reasoning\"}"); + // These should be ignored by the relay filter: + relay.accept("heartbeat", "{}"); + relay.accept("token", "{\"text\":\"hello\"}"); + return "filtered result"; + }); + + delegateAgentTool.delegateToAgent("FilterAgent", "filter task", null, null); + + ArgumentCaptor eventCaptor = ArgumentCaptor.forClass(String.class); + verify(streamTracker, atLeast(1)).broadcastObject( + eq(parentConvId), eventCaptor.capture(), any()); + + List events = eventCaptor.getAllValues(); + long progressCount = events.stream().filter("delegation_progress"::equals).count(); + // 2 recognized events → 2 progress broadcasts (heartbeat and token are filtered out) + assertEquals(2, progressCount, + "Should have exactly 2 delegation_progress events (tool_call_started + phase), got: " + events); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegationContextTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegationContextTest.java new file mode 100644 index 00000000..0331a726 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DelegationContextTest.java @@ -0,0 +1,152 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@link DelegationContext} stack-based context management. + * Covers: single-layer enter/exit, nested two-layer restore, depth consistency, + * and ThreadLocal cleanup. + */ +class DelegationContextTest { + + @AfterEach + void cleanup() { + // Ensure ThreadLocal is cleared after each test + while (DelegationContext.currentDepth() > 0) { + DelegationContext.exit(); + } + } + + // ===== Single-layer enter/exit ===== + + @Test + @DisplayName("Top-level enter/exit cleans up all state") + void topLevelEnterExitCleansUp() { + DelegationContext.enter("conv-parent", Set.of("toolA")); + + assertEquals(1, DelegationContext.currentDepth()); + assertEquals("conv-parent", DelegationContext.parentConversationId()); + assertEquals(Set.of("toolA"), DelegationContext.childDeniedTools()); + + DelegationContext.exit(); + + assertEquals(0, DelegationContext.currentDepth()); + assertNull(DelegationContext.parentConversationId()); + assertEquals(Set.of(), DelegationContext.childDeniedTools()); + } + + @Test + @DisplayName("No-arg enter sets null parentConversationId and empty deniedTools") + void noArgEnterDefaults() { + DelegationContext.enter(); + + assertEquals(1, DelegationContext.currentDepth()); + assertNull(DelegationContext.parentConversationId()); + assertEquals(Set.of(), DelegationContext.childDeniedTools()); + + DelegationContext.exit(); + assertEquals(0, DelegationContext.currentDepth()); + } + + // ===== Nested two-layer enter/exit ===== + + @Test + @DisplayName("Nested exit restores previous parentConversationId") + void nestedExitRestoresParentConversationId() { + // Layer 1 + DelegationContext.enter("conv-L1", Set.of("toolA")); + assertEquals("conv-L1", DelegationContext.parentConversationId()); + + // Layer 2 + DelegationContext.enter("conv-L2", Set.of("toolB")); + assertEquals(2, DelegationContext.currentDepth()); + assertEquals("conv-L2", DelegationContext.parentConversationId()); + + // Exit layer 2 → should restore layer 1 + DelegationContext.exit(); + assertEquals(1, DelegationContext.currentDepth()); + assertEquals("conv-L1", DelegationContext.parentConversationId()); + + // Exit layer 1 → should be clean + DelegationContext.exit(); + assertEquals(0, DelegationContext.currentDepth()); + assertNull(DelegationContext.parentConversationId()); + } + + @Test + @DisplayName("Nested exit restores previous deniedTools") + void nestedExitRestoresDeniedTools() { + Set layer1Tools = Set.of("delegateToAgent", "delegateParallel"); + Set layer2Tools = Set.of("searchWeb"); + + DelegationContext.enter("conv-1", layer1Tools); + DelegationContext.enter("conv-2", layer2Tools); + + assertEquals(layer2Tools, DelegationContext.childDeniedTools()); + + DelegationContext.exit(); + assertEquals(layer1Tools, DelegationContext.childDeniedTools()); + + DelegationContext.exit(); + assertEquals(Set.of(), DelegationContext.childDeniedTools()); + } + + // ===== Depth consistency ===== + + @Test + @DisplayName("Depth tracks push/pop correctly across 3 layers") + void depthTracksCorrectly() { + assertEquals(0, DelegationContext.currentDepth()); + + DelegationContext.enter("a", null); + assertEquals(1, DelegationContext.currentDepth()); + + DelegationContext.enter("b", null); + assertEquals(2, DelegationContext.currentDepth()); + + DelegationContext.enter("c", null); + assertEquals(3, DelegationContext.currentDepth()); + + DelegationContext.exit(); + assertEquals(2, DelegationContext.currentDepth()); + + DelegationContext.exit(); + assertEquals(1, DelegationContext.currentDepth()); + + DelegationContext.exit(); + assertEquals(0, DelegationContext.currentDepth()); + } + + @Test + @DisplayName("Exit on empty stack is a safe no-op") + void exitOnEmptyStackIsNoOp() { + assertEquals(0, DelegationContext.currentDepth()); + DelegationContext.exit(); // should not throw + assertEquals(0, DelegationContext.currentDepth()); + } + + // ===== ThreadLocal isolation ===== + + @Test + @DisplayName("Separate threads have independent delegation contexts") + void threadLocalIsolation() throws Exception { + DelegationContext.enter("main-thread-conv", Set.of("toolX")); + + Thread otherThread = new Thread(() -> { + assertEquals(0, DelegationContext.currentDepth()); + assertNull(DelegationContext.parentConversationId()); + }); + otherThread.start(); + otherThread.join(); + + // Main thread state should be unaffected + assertEquals(1, DelegationContext.currentDepth()); + assertEquals("main-thread-conv", DelegationContext.parentConversationId()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/DocumentExtractToolReadableRatioTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DocumentExtractToolReadableRatioTest.java new file mode 100644 index 00000000..34070063 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/DocumentExtractToolReadableRatioTest.java @@ -0,0 +1,189 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for the extraction-quality classifier in {@link DocumentExtractTool}. + * + *

The decisive cases: + *

    + *
  • CJK font encoding leak — many "characters", almost all junk → OCR.
  • + *
  • Scanned PDF with empty text layer → OCR.
  • + *
  • Mixed CN/EN body with realistic OCR noise → stays out of OCR.
  • + *
  • Pure ASCII body → stays out of OCR.
  • + *
+ */ +class DocumentExtractToolReadableRatioTest { + + /** + * Sample of the byte pattern observed when a PDF uses CID fonts without a + * {@code ToUnicode} CMap and the extractor dumps glyph indices as bytes. + * Mixes C0 control bytes, the C1 / Latin-1 Supplement block, and the + * tail "(¢" pair that dominated the real incident's extraction — + * a typical 8-page CID-encoded PDF lands here under 0.40 readable. + * Written with explicit escapes so the source file stays pure ASCII. + */ + private static final String CID_GLYPH_NOISE = + " " + + "Ç£¨±Ð¼½" + + "Ò®º¶¡¥æ" + + "òÙçÄÚÊÅ" + + "(¢(¢(¢(¢(¢"; + + @Test + @DisplayName("readableRatio: pure CJK text scores near 1.0") + void readableRatio_pureCjk_high() { + String text = "向量检索在自然语言处" + + "理中扮演重要角色。"; + assertThat(DocumentExtractTool.readableRatio(text)).isGreaterThan(0.95); + } + + @Test + @DisplayName("readableRatio: pure English text scores near 1.0") + void readableRatio_pureAscii_high() { + String text = "Vector retrieval improves recall on paraphrased queries by 18% over BM25."; + assertThat(DocumentExtractTool.readableRatio(text)).isGreaterThan(0.95); + } + + @Test + @DisplayName("readableRatio: mixed Chinese / English / punctuation scores near 1.0") + void readableRatio_mixed_high() { + String text = "评估章节:accuracy 提升 12%" + + ",latency 增加 ~15ms。详见 §3.2。"; + // § (section sign) is not in our readable ranges, so the mixed + // string lands just under "near-1.0" but still well above the threshold. + assertThat(DocumentExtractTool.readableRatio(text)).isGreaterThan(0.85); + } + + @Test + @DisplayName("readableRatio: CID glyph dump (PDFBox leak) scores well below 0.5") + void readableRatio_cidGlyphDump_low() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) { + sb.append(CID_GLYPH_NOISE); + } + assertThat(DocumentExtractTool.readableRatio(sb.toString())).isLessThan(0.40); + } + + @Test + @DisplayName("readableRatio: empty / null inputs return 0") + void readableRatio_emptyOrNull_zero() { + assertThat(DocumentExtractTool.readableRatio(null)).isZero(); + assertThat(DocumentExtractTool.readableRatio("")).isZero(); + } + + @Test + @DisplayName("classifyExtraction: CID glyph dump triggers low_readable_ratio") + void classify_cidGlyphDump_triggersReadableRatio() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 5000; i++) { + sb.append(CID_GLYPH_NOISE); + } + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction(sb.toString(), 8); + assertThat(q.needsOcr()).isTrue(); + assertThat(q.trigger()).isEqualTo("low_readable_ratio"); + assertThat(q.readableRatio()).isLessThan(0.40); + } + + @Test + @DisplayName("classifyExtraction: empty text triggers empty") + void classify_empty_triggersEmpty() { + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction("", 5); + assertThat(q.needsOcr()).isTrue(); + assertThat(q.trigger()).isEqualTo("empty"); + } + + @Test + @DisplayName("classifyExtraction: text under 20 chars triggers too_short") + void classify_tooShort_triggersTooShort() { + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction("hi", 5); + assertThat(q.needsOcr()).isTrue(); + assertThat(q.trigger()).isEqualTo("too_short"); + } + + @Test + @DisplayName("classifyExtraction: thin scanned-PDF text layer triggers low_char_density") + void classify_thinScannedLayer_triggersDensity() { + // 8 pages with only ~13 chars per page: well past the 20-char min so it + // doesn't short-circuit on too_short, but well under the 30-chars-per-page floor. + String pageMarker = "Title page X\n"; // 13 chars + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 8; i++) { + sb.append(pageMarker); + } + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction(sb.toString(), 8); + assertThat(q.needsOcr()).isTrue(); + assertThat(q.trigger()).isEqualTo("low_char_density"); + } + + @Test + @DisplayName("classifyExtraction: real CJK body passes") + void classify_realCjkBody_passes() { + String line = "北京赛区竞赛安排" + + ":报名截止时间 2026.\n"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 50; i++) { + sb.append(line); + } + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction(sb.toString(), 8); + assertThat(q.needsOcr()).isFalse(); + assertThat(q.trigger()).isNull(); + } + + @Test + @DisplayName("classifyExtraction: real English body passes") + void classify_realAsciiBody_passes() { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 50; i++) { + sb.append("Vector retrieval improves recall on paraphrased queries.\n"); + } + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction(sb.toString(), 8); + assertThat(q.needsOcr()).isFalse(); + } + + @Test + @DisplayName("classifyExtraction: noisy OCR output (low-quality but readable) passes") + void classify_noisyOcrOutput_passes() { + // Simulates OCR result with the occasional non-Latin garbage char sprinkled + // in real text. Θ (Greek capital theta) is outside our readable ranges. + String segment = "第 X 题:a/Θ求最大子" + + "序列和?"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 50; i++) { + sb.append(segment); + } + DocumentExtractTool.ExtractionQuality q = + DocumentExtractTool.classifyExtraction(sb.toString(), 4); + assertThat(q.needsOcr()).isFalse(); + } + + @Test + @DisplayName("classifyExtraction: unknown page count falls back to absolute-length check") + void classify_unknownPageCount_usesLengthFallback() { + String short_ = "二十一个字符的中" + + "文示例文本输入"; + DocumentExtractTool.ExtractionQuality shortQ = + DocumentExtractTool.classifyExtraction(short_, 0); + assertThat(shortQ.needsOcr()).isTrue(); + assertThat(shortQ.trigger()).isEqualTo("too_short"); + + String line = "足够长的中文示例文本" + + "一二三四五六七八九十。"; + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 20; i++) { + sb.append(line); + } + DocumentExtractTool.ExtractionQuality longQ = + DocumentExtractTool.classifyExtraction(sb.toString(), 0); + assertThat(longQ.needsOcr()).isFalse(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/ShellExecuteToolShellSelectionTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ShellExecuteToolShellSelectionTest.java new file mode 100644 index 00000000..f935244c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/ShellExecuteToolShellSelectionTest.java @@ -0,0 +1,71 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.util.function.Predicate; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Covers {@link ShellExecuteTool#selectPosixShell(String, Predicate)}, + * the helper that lets the shell tool honor the caller's {@code $SHELL} + * instead of the hardcoded {@code /bin/sh} fallback. + * + *

Tests use the executable-check seam so they're platform-independent — + * Windows CI doesn't have {@code /bin/sh}, POSIX dev hosts have varying + * shells installed. The pure logic is tested here; the real invocation + * goes through {@code Files::isExecutable} via the production overload. + */ +class ShellExecuteToolShellSelectionTest { + + private static final Predicate ALWAYS_EXECUTABLE = p -> true; + private static final Predicate NEVER_EXECUTABLE = p -> false; + + @Test + @DisplayName("null env → fallback to /bin/sh (no executable probe)") + void nullEnvFallsBack() { + assertEquals("/bin/sh", + ShellExecuteTool.selectPosixShell(null, ALWAYS_EXECUTABLE)); + } + + @Test + @DisplayName("empty / blank env → fallback to /bin/sh") + void blankEnvFallsBack() { + assertEquals("/bin/sh", + ShellExecuteTool.selectPosixShell("", ALWAYS_EXECUTABLE)); + assertEquals("/bin/sh", + ShellExecuteTool.selectPosixShell(" ", ALWAYS_EXECUTABLE)); + } + + @Test + @DisplayName("$SHELL points at executable shell → honored verbatim") + void executableShellHonored() { + // The whole point of this lane: prefer the user's interactive shell + // (zsh on macOS, bash on RHEL, fish on personal setups) over the + // dash that /bin/sh symlinks to on Debian/Ubuntu. + assertEquals("/usr/bin/zsh", + ShellExecuteTool.selectPosixShell("/usr/bin/zsh", ALWAYS_EXECUTABLE)); + assertEquals("/usr/local/bin/fish", + ShellExecuteTool.selectPosixShell("/usr/local/bin/fish", ALWAYS_EXECUTABLE)); + } + + @Test + @DisplayName("$SHELL set but not executable → fallback to /bin/sh") + void notExecutableFallsBack() { + assertEquals("/bin/sh", + ShellExecuteTool.selectPosixShell("/usr/bin/zsh", NEVER_EXECUTABLE)); + } + + @Test + @DisplayName("invalid path string → fallback to /bin/sh, no exception") + void invalidPathFallsBack() { + // NUL byte makes Path.of throw InvalidPathException on POSIX. + Predicate shouldNotBeReached = p -> { + throw new AssertionError("executable check must not run on invalid path"); + }; + assertEquals("/bin/sh", + ShellExecuteTool.selectPosixShell("/tmp/has\0null", shouldNotBeReached)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/SkillFileToolTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/SkillFileToolTest.java new file mode 100644 index 00000000..f3422854 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/SkillFileToolTest.java @@ -0,0 +1,152 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.skill.runtime.SkillFileAccessPolicy; +import vip.mate.skill.runtime.SkillRuntimeService; +import vip.mate.skill.runtime.model.ResolvedSkill; +import vip.mate.skill.usage.SkillUsageService; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class SkillFileToolTest { + + @Test + @DisplayName("listAvailableSkills applies keyword, source, status, and limit") + void listAvailableSkillsFiltersAndLimitsRuntimeCatalog() { + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + SkillFileAccessPolicy accessPolicy = mock(SkillFileAccessPolicy.class); + SkillUsageService usageService = mock(SkillUsageService.class); + SkillFileTool tool = new SkillFileTool(runtimeService, accessPolicy, usageService); + when(runtimeService.getActiveSkills()).thenReturn(List.of( + skill("apple-notes", "database", true), + skill("ckjia-shopping", "mcp", false), + skill("claude-code", "acp", false))); + + String result = tool.listAvailableSkills("code", "acp", "ready", 1); + + assertTrue(result.contains("claude-code")); + assertFalse(result.contains("ckjia-shopping")); + assertTrue(result.contains("Showing: 1 of 1")); + } + + @Test + @DisplayName("readSkillFile records SKILL.md usage") + void readSkillFileRecordsUsage() { + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + SkillFileAccessPolicy accessPolicy = mock(SkillFileAccessPolicy.class); + SkillUsageService usageService = mock(SkillUsageService.class); + SkillFileTool tool = new SkillFileTool(runtimeService, accessPolicy, usageService); + ResolvedSkill skill = skill("browser-cdp", "database", true); + skill.setContent("# Browser CDP\nUse devtools."); + when(runtimeService.findActiveSkill("browser-cdp")).thenReturn(skill); + + String content = tool.readSkillFile("browser-cdp", "SKILL.md", null, null, null); + + assertTrue(content.contains("Browser CDP")); + verify(usageService).recordLoaded( + org.mockito.ArgumentMatchers.eq(skill), + org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.isNull(), + org.mockito.ArgumentMatchers.eq("SKILL.md"), + org.mockito.ArgumentMatchers.anyInt()); + } + + @Test + @DisplayName("readSkillFile paginates large SKILL.md only when caller explicitly asks") + void readSkillFilePaginatesLargeContent() { + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + SkillFileAccessPolicy accessPolicy = mock(SkillFileAccessPolicy.class); + SkillUsageService usageService = mock(SkillUsageService.class); + SkillFileTool tool = new SkillFileTool(runtimeService, accessPolicy, usageService); + ResolvedSkill skill = skill("large-skill", "database", true); + skill.setContent("line\n".repeat(500)); + when(runtimeService.findActiveSkill("large-skill")).thenReturn(skill); + + String content = tool.readSkillFile("large-skill", "SKILL.md", 10, 20, null); + + assertTrue(content.startsWith("line\n")); + assertTrue(content.contains("shownLines=10-29")); + assertTrue(content.contains("startLine=30")); + } + + @Test + @DisplayName("oversized single line is head-truncated and lineIndex advances (no infinite loop)") + void readSkillFileAdvancesPastOversizedSingleLine() { + // P2 regression: if the first requested line is itself longer than + // MAX_OUTPUT_CHARS (8KB), the old loop hit `if (out.length() + + // rendered > cap) break;` with emitted=0 and the banner reported + // `shownLines=1-0, startLine=1` — the model would re-call with the + // same start line and never advance. Big JSON / minified scripts / + // base64 fixtures all triggered this. + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + SkillFileAccessPolicy accessPolicy = mock(SkillFileAccessPolicy.class); + SkillUsageService usageService = mock(SkillUsageService.class); + SkillFileTool tool = new SkillFileTool(runtimeService, accessPolicy, usageService); + ResolvedSkill skill = skill("huge-line-skill", "database", true); + // 12 KB single line — well past MAX_OUTPUT_CHARS (8KB). + String hugeLine = "x".repeat(12_000); + skill.setContent(hugeLine + "\nsecond line\nthird line\n"); + when(runtimeService.findActiveSkill("huge-line-skill")).thenReturn(skill); + + String content = tool.readSkillFile("huge-line-skill", "SKILL.md", 1, 5, null); + + // The head of the long line must appear in the output (head-truncated) + assertTrue(content.startsWith("xxxx"), + "Head of the oversized line must be visible to the model"); + // The truncation banner must point to the NEXT line, not the same one + assertTrue(content.contains("startLine=2"), + "Continuation pointer must advance past the over-long line, not stay at startLine=1"); + // Note marker must explain the partial-line situation + assertTrue(content.contains("exceeds per-call budget"), + "Banner should disclose that line content was head-truncated"); + } + + @Test + @DisplayName("readSkillFile returns full SKILL.md when caller did not request pagination") + void readSkillFileReturnsFullSkillMdByDefault() { + // Regression: pagination by default would let the model see only the + // first ~200 lines / 8KB of SKILL.md and silently miss later mandatory + // sections. SKILL.md is the skill contract and must arrive whole when + // the caller did not opt into pagination (startLine == null && maxLines + // == null). Reference / script files keep being paginated because they + // can be arbitrarily large supplementary material. + SkillRuntimeService runtimeService = mock(SkillRuntimeService.class); + SkillFileAccessPolicy accessPolicy = mock(SkillFileAccessPolicy.class); + SkillUsageService usageService = mock(SkillUsageService.class); + SkillFileTool tool = new SkillFileTool(runtimeService, accessPolicy, usageService); + ResolvedSkill skill = skill("large-skill", "database", true); + // 500 lines * 5 chars = 2500 chars; 250 lines is also above DEFAULT_MAX_LINES (200). + String body = "line\n".repeat(500); + skill.setContent(body); + when(runtimeService.findActiveSkill("large-skill")).thenReturn(skill); + + String content = tool.readSkillFile("large-skill", "SKILL.md", null, null, null); + + assertEquals(body, content, + "Default-path SKILL.md must be returned verbatim, not paginated"); + assertFalse(content.contains("[Skill file truncated"), + "No truncation banner should appear when caller did not opt into pagination"); + } + + private static ResolvedSkill skill(String name, String source, boolean builtin) { + return ResolvedSkill.builder() + .id((long) name.hashCode()) + .name(name) + .description("Description for " + name) + .source(source) + .builtin(builtin) + .enabled(true) + .runtimeAvailable(true) + .dependencyReady(true) + .securityBlocked(false) + .build(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/builtin/TikaExtractorTest.java b/mateclaw-server/src/test/java/vip/mate/tool/builtin/TikaExtractorTest.java new file mode 100644 index 00000000..e8582808 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/builtin/TikaExtractorTest.java @@ -0,0 +1,67 @@ +package vip.mate.tool.builtin; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-051 §5.2: pin TikaExtractor's safety guarantees. + *

+ * The actual format-specific extraction quality (PDF, DOCX, etc.) is verified + * by manual testing against real documents — these unit tests only lock down + * the wrapper's contract: null-handling, missing files, and the BodyContentHandler + * output cap. + */ +class TikaExtractorTest { + + @Test + @DisplayName("null path returns null without throwing") + void nullPath() { + assertNull(TikaExtractor.extract(null)); + } + + @Test + @DisplayName("non-existent path returns null without throwing") + void missingFile(@TempDir Path tmp) { + Path missing = tmp.resolve("does-not-exist.txt"); + assertNull(TikaExtractor.extract(missing)); + } + + @Test + @DisplayName("directory (non-regular file) returns null") + void directoryRejected(@TempDir Path tmp) { + assertNull(TikaExtractor.extract(tmp)); + } + + @Test + @DisplayName("plain text file is extracted verbatim under the cap") + void plainTextRoundTrip(@TempDir Path tmp) throws IOException { + Path file = tmp.resolve("note.txt"); + Files.writeString(file, "hello world"); + String out = TikaExtractor.extract(file); + assertNotNull(out); + assertTrue(out.contains("hello world"), "Extracted text should contain the original content. Got: " + out); + } + + @Test + @DisplayName("output is capped at maxChars; truncated parse still returns useful prefix") + void outputCapped(@TempDir Path tmp) throws IOException { + Path file = tmp.resolve("long.txt"); + // Build a file well above the cap so Tika hits the limit mid-parse. + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 1000; i++) sb.append("Lorem ipsum dolor sit amet. "); + Files.writeString(file, sb.toString()); + + // Cap at 100 chars; we expect a non-null, capped output. + String out = TikaExtractor.extract(file, 100); + assertNotNull(out, "should return partial text when cap reached, not null"); + assertTrue(out.length() <= 200, "should respect cap (some whitespace slack OK). Got len=" + out.length()); + assertTrue(out.contains("Lorem"), "partial output should still contain the leading text"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java new file mode 100644 index 00000000..0aac0356 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/document/GeneratedFileCacheScrubTest.java @@ -0,0 +1,107 @@ +package vip.mate.tool.document; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Pin the cache-side scrubber that powers the server-wide fake-URL guard. + * + *

Without this guard, an LLM-hallucinated {@code /api/v1/files/generated/{id}} + * URL surfaces verbatim to every channel (Web, Slack, DingTalk, Telegram, …), + * users tap it, and the IM client saves the resulting 404 HTML body as a + * {@code .docx} which they then report as a "corrupted file". These tests + * pin the cache-vs-text contract so future callers (FinalAnswerNode, + * channel adapters) get a single, consistent behaviour. + */ +class GeneratedFileCacheScrubTest { + + private GeneratedFileCache cache; + + @BeforeEach + void setUp() { + cache = new GeneratedFileCache(); + } + + @Test + @DisplayName("text without any generated-URL is returned unchanged (cheap fast path)") + void noUrlReturnsUnchanged() { + String text = "这是一段普通的回答,没有任何文件链接。"; + assertSame(text, cache.scrubMissingReferences(text), + "scrub must short-circuit when no URL pattern is found"); + } + + @Test + @DisplayName("null and empty input pass through") + void nullEmptyPassThrough() { + assertNull(cache.scrubMissingReferences(null)); + assertEquals("", cache.scrubMissingReferences("")); + } + + @Test + @DisplayName("hallucinated URL whose id is not in the cache → replaced with warning") + void unknownIdReplacedWithWarning() { + // The LLM emitted a UUID-shaped string but never called a render + // tool, so nothing was ever inserted into the cache. + String text = "您的文档已生成: /api/v1/files/generated/a1b2c3d4-e5f6-7890-abcd-ef1234567890"; + String scrubbed = cache.scrubMissingReferences(text); + assertTrue(scrubbed.contains(GeneratedFileCache.MISSING_REFERENCE_NOTICE), + "missing id should be replaced with the user-visible notice; got: " + scrubbed); + assertFalse(scrubbed.contains("/api/v1/files/generated/"), + "the broken URL must not survive in the scrubbed text; got: " + scrubbed); + } + + @Test + @DisplayName("real cached URL → left intact for downstream channel adapters to rewrite") + void liveIdLeftIntact() { + // Genuine render-tool output: bytes are in the cache, id is real. + String id = cache.put("hello".getBytes(), "report.pdf", "application/pdf"); + String text = "下载: /api/v1/files/generated/" + id; + String scrubbed = cache.scrubMissingReferences(text); + assertEquals(text, scrubbed, + "live URLs must pass through verbatim so channel adapters can still rewrite them"); + } + + @Test + @DisplayName("mix of one real + one fake URL — only the fake one is scrubbed") + void mixedRealAndFake() { + String realId = cache.put("real-bytes".getBytes(), "real.pdf", "application/pdf"); + String fakeId = "00000000-0000-0000-0000-000000000000"; + String text = "真实: /api/v1/files/generated/" + realId + + " 伪造: /api/v1/files/generated/" + fakeId; + String scrubbed = cache.scrubMissingReferences(text); + assertTrue(scrubbed.contains("/api/v1/files/generated/" + realId), + "real URL must survive; got: " + scrubbed); + assertFalse(scrubbed.contains(fakeId), + "fake URL must not survive; got: " + scrubbed); + assertTrue(scrubbed.contains(GeneratedFileCache.MISSING_REFERENCE_NOTICE)); + } + + @Test + @DisplayName("two fake URLs in same answer both get individual warnings") + void twoFakesBothScrubbed() { + String text = "/api/v1/files/generated/fake-1 then /api/v1/files/generated/fake-2"; + String scrubbed = cache.scrubMissingReferences(text); + assertFalse(scrubbed.contains("fake-1")); + assertFalse(scrubbed.contains("fake-2")); + // Two fakes → notice should appear twice (each occurrence replaced individually). + int firstHit = scrubbed.indexOf(GeneratedFileCache.MISSING_REFERENCE_NOTICE); + int secondHit = scrubbed.indexOf(GeneratedFileCache.MISSING_REFERENCE_NOTICE, firstHit + 1); + assertTrue(firstHit >= 0 && secondHit > firstHit, + "both fakes should be replaced; got: " + scrubbed); + } + + @Test + @DisplayName("URL pattern is package-shared so channel adapters and graph nodes match identically") + void patternIsExposed() { + // A regression here would mean the graph-side guard and the + // channel-side sniffer scan with different regexes — easy way to + // ship divergent behaviour. Pin the pattern so both call sites + // import the same constant. + assertNotNull(GeneratedFileCache.GENERATED_URL_PATTERN); + assertTrue(GeneratedFileCache.GENERATED_URL_PATTERN + .matcher("/api/v1/files/generated/abc-123").find()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/document/MarkdownDocxRendererTest.java b/mateclaw-server/src/test/java/vip/mate/tool/document/MarkdownDocxRendererTest.java new file mode 100644 index 00000000..4404fc8d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/document/MarkdownDocxRendererTest.java @@ -0,0 +1,140 @@ +package vip.mate.tool.document; + +import org.apache.poi.xwpf.usermodel.XWPFDocument; +import org.apache.poi.xwpf.usermodel.XWPFParagraph; +import org.apache.poi.xwpf.usermodel.XWPFTable; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.math.BigInteger; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Smoke tests for {@link MarkdownDocxRenderer}. Verifies that the renderer + * produces a syntactically valid .docx that POI can re-open and that the + * required Markdown elements actually map to the right OOXML structures. + */ +class MarkdownDocxRendererTest { + + private final MarkdownDocxRenderer renderer = new MarkdownDocxRenderer(); + + @Test + @DisplayName("Empty markdown still produces a valid, openable .docx") + void emptyMarkdownIsValid() throws Exception { + byte[] bytes = renderer.render("", "A4"); + assertNotNull(bytes); + assertTrue(bytes.length > 0, "should produce some bytes"); + try (XWPFDocument reopened = new XWPFDocument(new ByteArrayInputStream(bytes))) { + assertNotNull(reopened); + } + } + + @Test + @DisplayName("Headings, bold, lists, and tables all round-trip") + void mixedMarkdownRoundTrips() throws Exception { + String md = """ + # Title + + ## Subtitle + + ### Section + + A normal paragraph with **bold inside** it. + + - bullet one + - bullet two + + 1. step one + 2. step two + + | Name | Score | + | ---- | ----- | + | Alice | 90 | + | Bob | 85 | + """; + + byte[] bytes = renderer.render(md, "A4"); + + try (XWPFDocument doc = new XWPFDocument(new ByteArrayInputStream(bytes))) { + List paragraphs = doc.getParagraphs(); + assertFalse(paragraphs.isEmpty(), "should have paragraphs"); + + assertTrue(containsParagraphText(paragraphs, "Title")); + assertTrue(containsParagraphText(paragraphs, "Subtitle")); + assertTrue(containsParagraphText(paragraphs, "Section")); + assertTrue(containsParagraphText(paragraphs, "bold inside")); + assertTrue(containsParagraphText(paragraphs, "bullet one")); + assertTrue(containsParagraphText(paragraphs, "step one")); + + assertEquals("Heading1", styleOf(paragraphs, "Title")); + assertEquals("Heading2", styleOf(paragraphs, "Subtitle")); + assertEquals("Heading3", styleOf(paragraphs, "Section")); + + assertTrue(boldRunPresent(paragraphs, "bold inside"), + "**bold inside** should produce a bold run"); + + List tables = doc.getTables(); + assertEquals(1, tables.size(), "exactly one table expected"); + XWPFTable table = tables.get(0); + assertEquals(3, table.getRows().size(), "header + 2 data rows"); + assertEquals("Name", table.getRow(0).getCell(0).getText().trim()); + assertEquals("Alice", table.getRow(1).getCell(0).getText().trim()); + } + } + + @Test + @DisplayName("LETTER page size sets the right page width") + void letterPageSizeSetsWidth() throws Exception { + byte[] bytes = renderer.render("# Hello", "LETTER"); + try (XWPFDocument doc = new XWPFDocument(new ByteArrayInputStream(bytes))) { + var sectPr = doc.getDocument().getBody().getSectPr(); + assertNotNull(sectPr); + assertEquals(BigInteger.valueOf(12240), sectPr.getPgSz().getW()); + assertEquals(BigInteger.valueOf(15840), sectPr.getPgSz().getH()); + } + } + + @Test + @DisplayName("Default A4 sets the right page width") + void defaultPageSizeIsA4() throws Exception { + byte[] bytes = renderer.render("# Hello", null); + try (XWPFDocument doc = new XWPFDocument(new ByteArrayInputStream(bytes))) { + var sectPr = doc.getDocument().getBody().getSectPr(); + assertNotNull(sectPr); + assertEquals(BigInteger.valueOf(11906), sectPr.getPgSz().getW()); + assertEquals(BigInteger.valueOf(16838), sectPr.getPgSz().getH()); + } + } + + // ==================== helpers ==================== + + private boolean containsParagraphText(List paragraphs, String needle) { + for (XWPFParagraph p : paragraphs) { + if (p.getText() != null && p.getText().contains(needle)) return true; + } + return false; + } + + private String styleOf(List paragraphs, String needle) { + for (XWPFParagraph p : paragraphs) { + if (p.getText() != null && p.getText().contains(needle)) return p.getStyle(); + } + return null; + } + + private boolean boldRunPresent(List paragraphs, String needle) { + for (XWPFParagraph p : paragraphs) { + if (p.getText() == null || !p.getText().contains(needle)) continue; + for (var run : p.getRuns()) { + if (run.isBold() && needle.equals(run.getText(0))) return true; + } + } + return false; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/document/pdf/FlyingSaucerPdfCjkTest.java b/mateclaw-server/src/test/java/vip/mate/tool/document/pdf/FlyingSaucerPdfCjkTest.java new file mode 100644 index 00000000..69084631 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/document/pdf/FlyingSaucerPdfCjkTest.java @@ -0,0 +1,169 @@ +package vip.mate.tool.document.pdf; + +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.cos.COSDictionary; +import org.apache.pdfbox.cos.COSName; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDResources; +import org.apache.pdfbox.pdmodel.font.PDFont; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * End-to-end smoke test for the in-process PDF backend's CJK rendering. The + * historical bug we are guarding against: registering the font under the + * alias {@code "CJK"} (or any other override name) succeeded silently but + * the CSS lookup missed it and the body fell back to Times-Roman, leaving + * Chinese characters rendered as {@code .notdef} blank boxes. + * + *

This test renders a markdown body containing Chinese, then uses PDFBox + * to inspect the resulting PDF's embedded fonts. The assertion is that at + * least one font in the document has a name matching a known CJK family — + * Times-Roman alone is a regression. + */ +class FlyingSaucerPdfCjkTest { + + /** + * Substrings that, when present in a font's PostScript / BaseFont name, + * indicate a CJK-capable font has been embedded. The list covers the + * default CjkFontResolver candidates on macOS, Windows, and common + * Linux distros. + */ + private static final List CJK_FONT_MARKERS = List.of( + "STHeiti", "Heiti", "PingFang", "Songti", + "Microsoft YaHei", "MicrosoftYaHei", "MSYH", + "SimHei", "SimSun", "SongTi", "Song", + "NotoSans", "NotoSansCJK", + "HarmonyOS", "Harmony", + "SourceHan", "SourceHanSans", + "WQY", "WenQuanYi", "AR PL", "ArialUnicode" + ); + + @Test + @EnabledOnOs(OS.MAC) + @DisplayName("Chinese markdown renders with an embedded CJK font (not just Times-Roman)") + void chineseRendersWithEmbeddedCjkFont() throws Exception { + PdfProperties properties = new PdfProperties(null, PdfProperties.Engine.HTML, null); + FlyingSaucerPdfBackend backend = new FlyingSaucerPdfBackend(properties); + + // Plain string concatenation, NOT a Java text block: text block's + // relative-indent normalisation makes the empty-line vs body-line + // common-prefix rule unpredictable, and a 4+ space prefix is treated + // as an indented code block by CommonMark — that strips out every + // body line and leaves only the H1, which then renders into a + // 1.3 KB blank-looking PDF. + String markdown = + "# 季度业务回顾\n\n" + + "这是一份**中文**测试文档。\n\n" + + "- 第一条要点:业务增长 30%\n" + + "- 第二条要点:用户达到 100 万\n" + + "- 第三条要点:新增三个企业客户\n\n" + + "## 详细内容\n\n" + + "这里有更多的中文段落,用来验证字体嵌入是否生效。\n"; + + PdfRenderRequest request = new PdfRenderRequest( + markdown, PdfFrontmatter.parseOrSynthesise(markdown), + "A4", PdfProperties.Engine.HTML); + + // Reflectively peek at the intermediate HTML the renderer feeds to + // OpenPDF — when the produced PDF is suspiciously small (just the + // catalog header), the failure is upstream of OpenPDF, in either + // commonmark parsing or wrapHtml's template substitution. + java.lang.reflect.Method wrapHtmlMethod = FlyingSaucerPdfBackend.class + .getDeclaredMethod("wrapHtml", String.class, PdfRenderRequest.class, String.class); + wrapHtmlMethod.setAccessible(true); + java.lang.reflect.Method renderMdMethod = FlyingSaucerPdfBackend.class + .getDeclaredMethod("renderMarkdownToHtml", String.class); + renderMdMethod.setAccessible(true); + + String bodyHtml = (String) renderMdMethod.invoke(backend, markdown); + String fullHtml = (String) wrapHtmlMethod.invoke(backend, bodyHtml, request, "Heiti TC"); + + java.nio.file.Files.writeString(java.nio.file.Path.of("/tmp/mateclaw-pdf-cjk-test.html"), fullHtml); + System.out.println("[probe] body html length=" + bodyHtml.length() + + " sample=" + bodyHtml.substring(0, Math.min(200, bodyHtml.length()))); + System.out.println("[probe] full html length=" + fullHtml.length()); + + byte[] pdfBytes = backend.render(request); + assertNotNull(pdfBytes); + assertTrue(pdfBytes.length > 0, "renderer produced no output"); + + // Dump for manual inspection — useful when the assertion fails so the + // tester can `strings` / `pdftotext` the output without re-running. + java.nio.file.Path dump = java.nio.file.Path.of("/tmp/mateclaw-pdf-cjk-test.pdf"); + java.nio.file.Files.write(dump, pdfBytes); + System.out.println("[probe] wrote " + pdfBytes.length + " bytes to " + dump); + + // Cross-check the raw bytes too. PDFBox's font enumeration sometimes + // misses Type0 + CIDFontType2 wired by OpenPDF; the raw `/BaseFont` + // markers in the byte stream are easier to verify. + String rawText = new String(pdfBytes, java.nio.charset.StandardCharsets.ISO_8859_1); + java.util.regex.Matcher matcher = java.util.regex.Pattern + .compile("/BaseFont\\s*/([A-Za-z0-9+\\-]+)") + .matcher(rawText); + Set rawFontNames = new HashSet<>(); + while (matcher.find()) rawFontNames.add(matcher.group(1)); + System.out.println("[probe] raw /BaseFont names: " + rawFontNames); + + Set fontNames = collectFontNames(pdfBytes); + System.out.println("[probe] PDFBox-enumerated fonts: " + fontNames); + + // Combine both sources before asserting — this lets the test pass + // even if PDFBox's enumeration is incomplete, while still failing + // when the document only carries Times-Roman / Helvetica. + Set allFontNames = new HashSet<>(); + allFontNames.addAll(fontNames); + allFontNames.addAll(rawFontNames); + fontNames = allFontNames; + assertFalse(fontNames.isEmpty(), "PDF has no embedded fonts at all (raw or via PDFBox)"); + + boolean hasCjk = fontNames.stream() + .anyMatch(name -> CJK_FONT_MARKERS.stream() + .anyMatch(marker -> name.toLowerCase().contains(marker.toLowerCase()))); + + assertTrue(hasCjk, + "No CJK font embedded in the PDF — Chinese will render as blanks. " + + "Fonts found: " + fontNames); + } + + /** + * Walk every page's resources and collect the BaseFont names of every + * referenced font. Includes Type0 (composite) fonts for CJK plus their + * descendant CIDFontType2 fonts, where the actual TrueType glyph data + * lives. + */ + private static Set collectFontNames(byte[] pdfBytes) throws Exception { + Set names = new HashSet<>(); + try (PDDocument doc = Loader.loadPDF(pdfBytes)) { + for (PDPage page : doc.getPages()) { + PDResources resources = page.getResources(); + if (resources == null) continue; + List fontKeys = new ArrayList<>(); + resources.getFontNames().forEach(fontKeys::add); + for (COSName key : fontKeys) { + PDFont font = resources.getFont(key); + if (font == null) continue; + String baseFont = font.getName(); + if (baseFont != null) names.add(baseFont); + // Walk descendant fonts of Type0 composite fonts (where CJK lives). + COSDictionary dict = font.getCOSObject(); + Object descendants = dict.getDictionaryObject(COSName.getPDFName("DescendantFonts")); + if (descendants != null) names.add(descendants.toString()); + } + } + } + return names; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/guard/service/ToolGuardRuleServiceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/guard/service/ToolGuardRuleServiceTest.java new file mode 100644 index 00000000..b468012f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/guard/service/ToolGuardRuleServiceTest.java @@ -0,0 +1,118 @@ +package vip.mate.tool.guard.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.tool.guard.engine.ToolGuardRuleRegistry; +import vip.mate.tool.guard.model.ToolGuardRuleEntity; +import vip.mate.tool.guard.repository.ToolGuardRuleMapper; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class ToolGuardRuleServiceTest { + + private ToolGuardRuleMapper ruleMapper; + private ToolGuardRuleRegistry ruleRegistry; + private ToolGuardRuleService service; + + @BeforeEach + void setUp() { + ruleMapper = mock(ToolGuardRuleMapper.class); + ruleRegistry = mock(ToolGuardRuleRegistry.class); + service = new ToolGuardRuleService(ruleMapper, ruleRegistry); + } + + @Test + @DisplayName("createRule rejects blank ruleId before persistence") + void createRuleRejectsBlankRuleId() { + ToolGuardRuleEntity rule = wellFormedRule(); + rule.setRuleId(" "); + + assertThrows(IllegalArgumentException.class, () -> service.createRule(rule)); + + verify(ruleMapper, never()).insert(any(ToolGuardRuleEntity.class)); + verify(ruleRegistry, never()).reload(); + } + + @Test + @DisplayName("createRule rejects blank name before persistence") + void createRuleRejectsBlankName() { + ToolGuardRuleEntity rule = wellFormedRule(); + rule.setName(""); + + assertThrows(IllegalArgumentException.class, () -> service.createRule(rule)); + + verify(ruleMapper, never()).insert(any(ToolGuardRuleEntity.class)); + } + + @Test + @DisplayName("createRule rejects blank pattern before persistence") + void createRuleRejectsBlankPattern() { + ToolGuardRuleEntity rule = wellFormedRule(); + rule.setPattern(null); + + assertThrows(IllegalArgumentException.class, () -> service.createRule(rule)); + + verify(ruleMapper, never()).insert(any(ToolGuardRuleEntity.class)); + } + + @Test + @DisplayName("updateRule rejects explicit blank name") + void updateRuleRejectsExplicitBlankName() { + ToolGuardRuleEntity existing = wellFormedRule(); + existing.setId(7L); + when(ruleMapper.selectOne(any())).thenReturn(existing); + + ToolGuardRuleEntity update = new ToolGuardRuleEntity(); + update.setName(" "); + + assertThrows(IllegalArgumentException.class, + () -> service.updateRule("CUSTOM_RULE", update)); + + verify(ruleMapper, never()).updateById(any(ToolGuardRuleEntity.class)); + } + + @Test + @DisplayName("deleteRuleByPk hard-deletes a custom rule by primary key") + void deleteRuleByPkRemovesCustomRule() { + ToolGuardRuleEntity existing = wellFormedRule(); + existing.setId(42L); + existing.setBuiltin(false); + when(ruleMapper.selectById(42L)).thenReturn(existing); + + service.deleteRuleByPk(42L); + + verify(ruleMapper).deleteById(eq(42L)); + verify(ruleRegistry).reload(); + } + + @Test + @DisplayName("deleteRuleByPk refuses to remove builtin rules") + void deleteRuleByPkRejectsBuiltin() { + ToolGuardRuleEntity existing = wellFormedRule(); + existing.setId(99L); + existing.setBuiltin(true); + when(ruleMapper.selectById(99L)).thenReturn(existing); + + IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, + () -> service.deleteRuleByPk(99L)); + assertEquals(true, ex.getMessage().contains("builtin")); + + verify(ruleMapper, never()).deleteById(any(Long.class)); + } + + private static ToolGuardRuleEntity wellFormedRule() { + ToolGuardRuleEntity rule = new ToolGuardRuleEntity(); + rule.setRuleId("CUSTOM_RULE"); + rule.setName("Custom rule"); + rule.setPattern(".*"); + return rule; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/ImageFileDownloaderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageFileDownloaderTest.java new file mode 100644 index 00000000..6c92d813 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageFileDownloaderTest.java @@ -0,0 +1,135 @@ +package vip.mate.tool.image; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Base64; +import java.util.Comparator; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for the data-URL handling added to {@link ImageFileDownloader}. + * + *

Network-bound HTTP downloads are intentionally not exercised here — + * the regression we care about is the silent failure that happened when a + * provider returned a {@code data:image/png;base64,...} URL: callers fed + * that into {@code HttpUtil.downloadFile}, which mangled it into something + * like {@code file:/cwd/http:/data:image/...} and threw, so the image + * never landed on disk and the assistant message rendered empty. + * + *

The downloader writes under {@code data/chat-uploads//...} + * relative to the JVM's working directory; we sweep that directory after + * each test so the run leaves no artefacts behind. + */ +@Tag("media-gen") +class ImageFileDownloaderTest { + + private ImageFileDownloader downloader; + private final String conv = "test-conv-" + System.nanoTime(); + + @BeforeEach + void setUp() { + downloader = new ImageFileDownloader(); + } + + @AfterEach + void cleanup() throws IOException { + Path dir = Paths.get("data", "chat-uploads", conv); + if (!Files.exists(dir)) return; + try (Stream walk = Files.walk(dir)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { Files.deleteIfExists(p); } catch (IOException ignored) {} + }); + } + } + + @Test + @DisplayName("download writes the decoded bytes when given a base64 data URL") + void download_baseDataUrl_writesDecodedBytes() throws Exception { + // 1x1 transparent PNG — the smallest legal payload we can verify byte-for-byte + byte[] pngBytes = new byte[]{ + (byte) 0x89, 'P', 'N', 'G', '\r', '\n', 0x1A, '\n', + 0, 0, 0, 13, 'I', 'H', 'D', 'R', + 0, 0, 0, 1, 0, 0, 0, 1, 8, 6, 0, 0, 0, + 0x1F, 0x15, (byte) 0xC4, (byte) 0x89 + }; + String dataUrl = "data:image/png;base64," + Base64.getEncoder().encodeToString(pngBytes); + + Path saved = downloader.download(dataUrl, conv, "task1", 0); + + assertTrue(Files.exists(saved), "saved file must exist"); + assertTrue(saved.getFileName().toString().endsWith(".png")); + byte[] readBack = Files.readAllBytes(saved); + assertArrayEquals(pngBytes, readBack, "stored bytes must match decoded payload"); + } + + @Test + @DisplayName("download picks extension from the data-URL media type") + void download_extensionMatchesMediaType() throws Exception { + Path png = downloader.download( + "data:image/png;base64," + Base64.getEncoder().encodeToString(new byte[]{1, 2, 3}), + conv, "ext-png", 0); + assertTrue(png.getFileName().toString().endsWith(".png")); + + Path jpg = downloader.download( + "data:image/jpeg;base64," + Base64.getEncoder().encodeToString(new byte[]{4, 5, 6}), + conv, "ext-jpg", 0); + assertTrue(jpg.getFileName().toString().endsWith(".jpg")); + + Path webp = downloader.download( + "data:image/webp;base64," + Base64.getEncoder().encodeToString(new byte[]{7, 8, 9}), + conv, "ext-webp", 0); + assertTrue(webp.getFileName().toString().endsWith(".webp")); + + // Unknown / missing media type → default to png + Path fallback = downloader.download( + "data:;base64," + Base64.getEncoder().encodeToString(new byte[]{0}), + conv, "ext-fallback", 0); + assertTrue(fallback.getFileName().toString().endsWith(".png")); + } + + @Test + @DisplayName("download accepts the percent-encoded body form (no ;base64)") + void download_percentEncodedDataUrl() throws Exception { + // The ";base64" form is the common one but RFC 2397 also allows a raw + // (URL-encoded) body. Make sure both round-trip safely. + String dataUrl = "data:image/png,hello%20world"; + Path saved = downloader.download(dataUrl, conv, "raw", 0); + assertEquals("hello world", Files.readString(saved)); + } + + @Test + @DisplayName("download rejects malformed data URLs cleanly") + void download_malformedDataUrlIsRejected() { + IOException ex = assertThrows(IOException.class, + () -> downloader.download("data:image/png;base64", conv, "bad", 0)); + assertTrue(ex.getMessage().contains("Malformed data URL"), + "expected explanatory error, got: " + ex.getMessage()); + } + + @Test + @DisplayName("download rejects invalid base64 payloads with a wrapped IOException") + void download_invalidBase64IsWrapped() { + // !!! is not a legal base64 token + IOException ex = assertThrows(IOException.class, + () -> downloader.download("data:image/png;base64,!!!", conv, "badb64", 0)); + assertTrue(ex.getMessage().toLowerCase().contains("base64")); + } + + @Test + @DisplayName("download rejects null URLs without leaking NPE") + void download_nullIsRejected() { + IOException ex = assertThrows(IOException.class, + () -> downloader.download(null, conv, "null", 0)); + assertTrue(ex.getMessage().contains("null")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/ImageProviderCapabilitiesTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageProviderCapabilitiesTest.java new file mode 100644 index 00000000..ec1206fe --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageProviderCapabilitiesTest.java @@ -0,0 +1,103 @@ +package vip.mate.tool.image; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Locks in the orientation-aware {@link ImageProviderCapabilities#normalizeSize} + * contract. The earlier implementation matched purely by area, which collapsed + * portrait/landscape requests onto the wrong supported size when supported + * sizes had identical area (720x1280 vs 1280x720). Each provider previously + * worked around this by re-deriving the size from {@code aspectRatio} inside + * {@code submit()}; centralizing that logic here lets providers trust + * {@code request.getSize()}. + */ +@Tag("media-gen") +class ImageProviderCapabilitiesTest { + + private static ImageProviderCapabilities dashScopeStyle() { + return ImageProviderCapabilities.builder() + .supportedSizes(List.of("1024x1024", "720x1280", "1280x720")) + .aspectRatios(List.of("1:1", "16:9", "9:16")) + .build(); + } + + private static ImageProviderCapabilities falStyle() { + return ImageProviderCapabilities.builder() + .supportedSizes(List.of("1024x1024", "1024x1536", "1536x1024")) + .aspectRatios(List.of("1:1", "16:9", "9:16", "4:3", "3:4")) + .build(); + } + + @Test + void exactMatchPassesThrough() { + assertEquals("1280x720", dashScopeStyle().normalizeSize("1280x720", "16:9")); + assertEquals("720x1280", dashScopeStyle().normalizeSize("720x1280", "9:16")); + } + + @Test + void aspectRatioPicksLandscapeWhenSizeMissing() { + // Without aspect: area-based fallback could pick either 720x1280 or 1280x720 + // (identical area). With aspect 16:9, must select landscape. + assertEquals("1280x720", dashScopeStyle().normalizeSize(null, "16:9")); + } + + @Test + void aspectRatioPicksPortraitWhenSizeMissing() { + assertEquals("720x1280", dashScopeStyle().normalizeSize(null, "9:16")); + } + + @Test + void aspectRatioPreservesOrientationWhenSizeIsUnsupported() { + // 1920x1080 is unsupported; without aspect awareness the area match would + // collapse to whichever 720*1280 entry came first. Aspect 16:9 forces landscape. + assertEquals("1280x720", dashScopeStyle().normalizeSize("1920x1080", "16:9")); + assertEquals("720x1280", dashScopeStyle().normalizeSize("1080x1920", "9:16")); + } + + @Test + void squareAspectFallsBackToSquareSize() { + assertEquals("1024x1024", dashScopeStyle().normalizeSize(null, "1:1")); + assertEquals("1024x1024", falStyle().normalizeSize(null, "1:1")); + } + + @Test + void undeclaredButLandscapeAspectStillRoutesToLandscapeSize() { + // 4:3 is not in aspectRatios but is numerically landscape (4 > 3). + // Orientation filter narrows to landscape candidates (1280x720 only). + assertEquals("1280x720", dashScopeStyle().normalizeSize(null, "4:3")); + assertEquals("720x1280", dashScopeStyle().normalizeSize(null, "3:4")); + } + + @Test + void blankInputReturnsAreaClosest() { + // Blank size + blank aspect: pick by default area (1M). + assertEquals("1024x1024", dashScopeStyle().normalizeSize("", null)); + assertEquals("1024x1024", dashScopeStyle().normalizeSize(null, null)); + } + + @Test + void backwardsCompatibleOverloadStillWorks() { + // Old single-arg overload delegates to the new one with null aspect. + assertEquals("1024x1024", dashScopeStyle().normalizeSize("1024x1024")); + } + + @Test + void normalizeAspectRatioFallsBackToFirstSupported() { + assertEquals("1:1", dashScopeStyle().normalizeAspectRatio("21:9")); + assertEquals("16:9", dashScopeStyle().normalizeAspectRatio("16:9")); + } + + @Test + void normalizeCountClampsWithinBounds() { + ImageProviderCapabilities caps = ImageProviderCapabilities.builder() + .maxCount(4).build(); + assertEquals(1, caps.normalizeCount(0)); + assertEquals(4, caps.normalizeCount(10)); + assertEquals(2, caps.normalizeCount(2)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/ImageReferenceLoaderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageReferenceLoaderTest.java new file mode 100644 index 00000000..5690c069 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/ImageReferenceLoaderTest.java @@ -0,0 +1,185 @@ +package vip.mate.tool.image; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.workspace.conversation.ConversationService; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +/** + * Verifies the five accepted reference forms in {@link ImageReferenceLoader}: + * local path, {@code file://}, {@code data:} URL, {@code http(s)://} (with the + * SSRF guard), and {@code msg::} for an attachment from an earlier + * conversation message. The conversation form is exercised in a separate test + * with a real ConversationService stub; the others need no collaborators. + */ +@Tag("media-gen") +class ImageReferenceLoaderTest { + + private ImageReferenceLoader loader; + private Path tmpDir; + + @BeforeEach + void setUp() throws IOException { + loader = new ImageReferenceLoader(mock(ConversationService.class)); + tmpDir = Files.createTempDirectory("img-ref-loader-test-"); + } + + @AfterEach + void tearDown() throws IOException { + if (tmpDir != null && Files.exists(tmpDir)) { + try (var stream = Files.walk(tmpDir)) { + stream.sorted(Comparator.reverseOrder()).forEach(p -> { + try { Files.deleteIfExists(p); } catch (IOException ignore) {} + }); + } + } + } + + // ==================== form: local path ==================== + + @Test + @DisplayName("local absolute path: reads bytes and infers mime from extension") + void localPath_absolute_loadsBytes() throws Exception { + byte[] bytes = {1, 2, 3, 4}; + Path file = tmpDir.resolve("kitten.jpg"); + Files.write(file, bytes); + + ImageReference ref = loader.load(file.toAbsolutePath().toString(), "conv-x"); + + assertArrayEquals(bytes, ref.data()); + assertEquals("image/jpeg", ref.mimeType()); + assertEquals("kitten.jpg", ref.fileName()); + assertTrue(ref.origin().startsWith("path:")); + } + + @Test + @DisplayName("file:// URL: prefix is stripped before resolving the path") + void fileUrl_resolvesAsLocal() throws Exception { + Path file = tmpDir.resolve("note.png"); + Files.write(file, new byte[]{9}); + + ImageReference ref = loader.load("file://" + file.toAbsolutePath(), "conv-x"); + + assertEquals("image/png", ref.mimeType()); + assertEquals(1, ref.data().length); + } + + @Test + @DisplayName("missing local file fails clearly without leaking the entire path elsewhere") + void localPath_missing_throws() { + IOException err = assertThrows(IOException.class, + () -> loader.load("/tmp/definitely-not-here-" + System.nanoTime() + ".png", "conv-x")); + assertTrue(err.getMessage().contains("not found"), err.getMessage()); + } + + // ==================== form: data: URL ==================== + + @Test + @DisplayName("data: URL with base64 body: decodes bytes and keeps declared mime") + void dataUrl_base64_decodes() throws Exception { + // "hi" in base64 + String dataUrl = "data:image/png;base64,aGk="; + ImageReference ref = loader.load(dataUrl, "conv-x"); + assertArrayEquals(new byte[]{'h', 'i'}, ref.data()); + assertEquals("image/png", ref.mimeType()); + assertEquals("data-url", ref.origin()); + } + + @Test + @DisplayName("data: URL with URL-encoded body: also decodes") + void dataUrl_urlEncoded_decodes() throws Exception { + String dataUrl = "data:image/svg+xml,%3Csvg%2F%3E"; + ImageReference ref = loader.load(dataUrl, "conv-x"); + assertEquals("image/svg+xml", ref.mimeType()); + assertTrue(new String(ref.data()).contains("")); + } + + @Test + @DisplayName("malformed data: URL (missing comma) fails") + void dataUrl_malformed_throws() { + assertThrows(IOException.class, () -> loader.load("data:image/png;base64", "conv-x")); + } + + // ==================== form: http(s):// SSRF guard ==================== + + @Test + @DisplayName("SSRF guard rejects localhost / 127.0.0.1 / private subnets without making any HTTP call") + void httpUrl_ssrfGuard_rejectsInternalHosts() { + for (String url : new String[]{ + "http://localhost/foo.png", + "http://127.0.0.1/foo.png", + "http://10.1.2.3/foo.png", + "http://192.168.1.1/foo.png", + "http://169.254.169.254/foo.png" // AWS instance metadata + }) { + IOException err = assertThrows(IOException.class, () -> loader.load(url, "conv-x"), + "expected SSRF guard to reject " + url); + assertTrue(err.getMessage().toLowerCase().contains("internal"), url); + } + } + + // ==================== form: msg:: parse errors ==================== + + @Test + @DisplayName("msg: ref with non-numeric message id fails fast") + void msgRef_invalidMessageId_throws() { + IOException err = assertThrows(IOException.class, () -> loader.load("msg:abc:0", "conv-x")); + assertTrue(err.getMessage().toLowerCase().contains("invalid"), err.getMessage()); + } + + @Test + @DisplayName("msg: ref without an active conversation id fails fast") + void msgRef_noConversation_throws() { + IOException err = assertThrows(IOException.class, () -> loader.load("msg:123:0", null)); + assertTrue(err.getMessage().toLowerCase().contains("conversation"), err.getMessage()); + } + + @Test + @DisplayName("msg: ref with bad part index format fails fast") + void msgRef_invalidPartIndex_throws() { + IOException err = assertThrows(IOException.class, () -> loader.load("msg:123:nope", "conv-x")); + assertTrue(err.getMessage().toLowerCase().contains("invalid"), err.getMessage()); + } + + // ==================== loadAll ==================== + + @Test + @DisplayName("loadAll: skips null/blank entries, preserves order otherwise") + void loadAll_skipsBlanksAndPreservesOrder() throws Exception { + Path a = tmpDir.resolve("a.png"); + Path b = tmpDir.resolve("b.png"); + Files.write(a, new byte[]{1}); + Files.write(b, new byte[]{2}); + + var refs = loader.loadAll(java.util.Arrays.asList( + a.toAbsolutePath().toString(), + null, + "", + b.toAbsolutePath().toString() + ), "conv-x"); + + assertEquals(2, refs.size()); + assertArrayEquals(new byte[]{1}, refs.get(0).data()); + assertArrayEquals(new byte[]{2}, refs.get(1).data()); + } + + @Test + @DisplayName("loadAll: null / empty input returns an empty list (no NPE)") + void loadAll_nullOrEmpty_returnsEmpty() throws Exception { + assertTrue(loader.loadAll(null, "conv-x").isEmpty()); + assertTrue(loader.loadAll(java.util.List.of(), "conv-x").isEmpty()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/PayloadBuilderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/PayloadBuilderTest.java new file mode 100644 index 00000000..31ce9c58 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/PayloadBuilderTest.java @@ -0,0 +1,178 @@ +package vip.mate.tool.image; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Locks in the configuration-driven payload behaviour: + *

    + *
  1. The model spec's {@code supports} set is the final whitelist — keys not + * on it must be dropped from the produced JSON regardless of how they got + * there (defaults, explicit setters, sizing).
  2. + *
  3. Each {@link SizeStyle} produces the right key and translates from the + * unified {@code size} / {@code aspectRatio} inputs to the model-native + * form (literal dim / aspect ratio / preset).
  4. + *
  5. Empty / null whitelist passes everything through.
  6. + *
+ */ +@Tag("media-gen") +class PayloadBuilderTest { + + private final ObjectMapper mapper = new ObjectMapper(); + + private ImageModelSpec literalSpec(Set supports) { + return ImageModelSpec.builder() + .id("literal-test") + .endpoint("https://example/api") + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMapping("1:1", "1024x1024") + .sizeMapping("16:9", "1280x720") + .sizeMapping("9:16", "720x1280") + .sizeMapping("landscape", "1280x720") + .sizeMapping("square", "1024x1024") + .sizeMapping("portrait", "720x1280") + .supports(supports) + .maxCount(4) + .build(); + } + + @Test + @DisplayName("supports whitelist: keys outside the set are dropped from JSON") + void supportsWhitelistFiltersOutKeys() { + ImageModelSpec spec = literalSpec(Set.of("size", "n")); + ObjectNode body = PayloadBuilder.from(spec) + .withPrompt("hello") + .withCount(2) + .withSize("1024x1024", "1:1") + .withSeed(42) + .put("custom", "yes") + .toJsonNode(mapper); + + assertTrue(body.has("size")); + assertTrue(body.has("n")); + assertFalse(body.has("prompt"), "prompt is not in supports => filtered"); + assertFalse(body.has("seed"), "seed is not in supports => filtered"); + assertFalse(body.has("custom"), "ad-hoc keys not in supports => filtered"); + } + + @Test + @DisplayName("empty supports set means passthrough — no filtering") + void emptySupports_passesEverything() { + ImageModelSpec spec = literalSpec(Set.of()); + ObjectNode body = PayloadBuilder.from(spec) + .withPrompt("p") + .withCount(1) + .withSize("1024x1024", "1:1") + .toJsonNode(mapper); + assertTrue(body.has("prompt")); + assertTrue(body.has("size")); + assertTrue(body.has("n")); + } + + @Test + @DisplayName("LITERAL_DIMENSION: requested size in sizeMap is translated to native form") + void literalDimension_translatesViaSizeMap() { + // sizeMap entry "1024x1024" -> native form would normally be the same; + // legacy DashScope translates to "1024*1024". Provide a custom mapping. + ImageModelSpec spec = ImageModelSpec.builder() + .id("legacy-async") + .endpoint("https://x/api") + .transport(ImageModelSpec.Transport.ASYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMapping("1024x1024", "1024*1024") + .sizeMapping("landscape", "1280*720") + .supports(Set.of("size", "n")) + .maxCount(4) + .build(); + ObjectNode body = PayloadBuilder.from(spec) + .withSize("1024x1024", "1:1") + .toJsonNode(mapper); + assertEquals("1024*1024", body.get("size").asText()); + } + + @Test + @DisplayName("LITERAL_DIMENSION: missing size falls back to orientation lookup in sizeMap") + void literalDimension_orientationFallback() { + ImageModelSpec spec = literalSpec(Set.of("size")); + // No requested size, aspect 16:9 → must pick landscape entry. + ObjectNode body = PayloadBuilder.from(spec).withSize(null, "16:9").toJsonNode(mapper); + assertEquals("1280x720", body.get("size").asText()); + + // 9:16 → portrait + ObjectNode portrait = PayloadBuilder.from(spec).withSize(null, "9:16").toJsonNode(mapper); + assertEquals("720x1280", portrait.get("size").asText()); + } + + @Test + @DisplayName("ASPECT_RATIO style sets aspect_ratio (not size); requested ratio is forwarded") + void aspectRatioStyle_setsAspectRatioKey() { + ImageModelSpec spec = ImageModelSpec.builder() + .id("aspect") + .endpoint("https://x") + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.ASPECT_RATIO) + .supports(Set.of("aspect_ratio")) + .build(); + ObjectNode body = PayloadBuilder.from(spec).withSize(null, "16:9").toJsonNode(mapper); + assertEquals("16:9", body.get("aspect_ratio").asText()); + assertFalse(body.has("size")); + } + + @Test + @DisplayName("PRESET_NAME style sets image_size to the orientation-keyed preset") + void presetStyle_setsImageSizeKey() { + ImageModelSpec spec = ImageModelSpec.builder() + .id("preset") + .endpoint("https://x") + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.PRESET_NAME) + .sizeMapping("landscape", "landscape_16_9") + .sizeMapping("square", "square_hd") + .sizeMapping("portrait", "portrait_16_9") + .supports(Set.of("image_size")) + .build(); + // 16:9 is landscape + assertEquals("landscape_16_9", + PayloadBuilder.from(spec).withSize(null, "16:9").toJsonNode(mapper).get("image_size").asText()); + // 1:1 is square + assertEquals("square_hd", + PayloadBuilder.from(spec).withSize(null, "1:1").toJsonNode(mapper).get("image_size").asText()); + } + + @Test + @DisplayName("defaults from spec are seeded before explicit setters; overrides take precedence") + void defaultsAreSeededFirst() { + ImageModelSpec spec = ImageModelSpec.builder() + .id("with-defaults") + .endpoint("https://x") + .transport(ImageModelSpec.Transport.SYNC) + .sizeStyle(SizeStyle.LITERAL_DIMENSION) + .sizeMapping("1:1", "1024x1024") + .defaultParam("watermark", true) + .defaultParam("n", 1) + .supports(Set.of("watermark", "n", "size")) + .build(); + ObjectNode body = PayloadBuilder.from(spec).withSize(null, "1:1").withCount(3).toJsonNode(mapper); + assertEquals(true, body.get("watermark").asBoolean()); + // explicit count overrides default + assertEquals(3, body.get("n").asInt()); + } + + @Test + @DisplayName("withCount clamps to spec.maxCount when above it") + void withCount_clampsToMaxCount() { + ImageModelSpec spec = literalSpec(Set.of("n")); + ObjectNode body = PayloadBuilder.from(spec).withCount(99).toJsonNode(mapper); + assertEquals(4, body.get("n").asInt()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/provider/ChatGPTOAuthImageProviderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/ChatGPTOAuthImageProviderTest.java new file mode 100644 index 00000000..59eff925 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/ChatGPTOAuthImageProviderTest.java @@ -0,0 +1,219 @@ +package vip.mate.tool.image.provider; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.llm.oauth.OpenAIOAuthService; +import vip.mate.tool.image.ImageGenerationRequest; +import vip.mate.tool.image.ImageProviderCapabilities; + +import java.lang.reflect.Field; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.mock; + +/** + * Unit tests for the OAuth image provider. Focus on the deterministic bits — + * Responses-API body construction, SSE stream parsing, quality/size mapping. + * Network-dependent {@code submit()} is exercised end-to-end via a separate + * integration test once a sandbox token is available. + */ +@Tag("media-gen") +class ChatGPTOAuthImageProviderTest { + + private ChatGPTOAuthImageProvider provider; + private ObjectMapper objectMapper; + + @BeforeEach + void setUp() throws Exception { + objectMapper = new ObjectMapper(); + provider = new ChatGPTOAuthImageProvider(mock(OpenAIOAuthService.class), objectMapper); + // @Value defaults aren't applied in plain new() construction — inject + // them via reflection so the build paths see realistic values. + setField(provider, "chatHostModel", "gpt-5.4"); + setField(provider, "defaultQuality", "medium"); + setField(provider, "timeoutMs", 240000); + } + + private static void setField(Object target, String name, Object value) throws Exception { + Field f = ChatGPTOAuthImageProvider.class.getDeclaredField(name); + f.setAccessible(true); + f.set(target, value); + } + + // ==================== body construction ================================= + + @Test + @DisplayName("body uses chat-host model + image_generation tool pinned to gpt-image-2") + void buildResponsesBody_pinsImageModelAndTool() throws Exception { + String body = provider.buildResponsesBody("a red panda", "1024x1024", "medium"); + JsonNode root = objectMapper.readTree(body); + + assertEquals("gpt-5.4", root.path("model").asText()); + assertFalse(root.path("store").asBoolean(true)); + // The /codex/responses endpoint rejects non-streaming with HTTP 400 + // "Stream must be set to true" — lock the flag in. + assertTrue(root.path("stream").asBoolean(false), + "stream must be true; codex/responses rejects non-streaming requests"); + assertTrue(root.path("instructions").asText("").contains("image_generation")); + + // Single user message carrying the prompt + JsonNode input = root.path("input"); + assertTrue(input.isArray()); + assertEquals(1, input.size()); + JsonNode msg = input.get(0); + assertEquals("user", msg.path("role").asText()); + assertEquals("a red panda", + msg.path("content").get(0).path("text").asText()); + + // Tool definition pinned to gpt-image-2 with the right knobs + JsonNode tools = root.path("tools"); + assertEquals(1, tools.size()); + JsonNode tool = tools.get(0); + assertEquals("image_generation", tool.path("type").asText()); + assertEquals("gpt-image-2", tool.path("model").asText()); + assertEquals("1024x1024", tool.path("size").asText()); + assertEquals("medium", tool.path("quality").asText()); + assertEquals("png", tool.path("output_format").asText()); + assertEquals("opaque", tool.path("background").asText()); + assertEquals(1, tool.path("partial_images").asInt()); + + // Forced tool_choice + JsonNode choice = root.path("tool_choice"); + assertEquals("allowed_tools", choice.path("type").asText()); + assertEquals("required", choice.path("mode").asText()); + assertEquals("image_generation", + choice.path("tools").get(0).path("type").asText()); + } + + @Test + @DisplayName("buildResponsesBody tolerates a null prompt (degrades to empty string)") + void buildResponsesBody_nullPromptSafe() throws Exception { + String body = provider.buildResponsesBody(null, "1024x1024", "low"); + JsonNode root = objectMapper.readTree(body); + assertEquals("", + root.path("input").get(0).path("content").get(0).path("text").asText()); + } + + @Test + @DisplayName("buildResponsesBody respects a configurable chat-host model override") + void buildResponsesBody_chatHostModelConfigurable() throws Exception { + setField(provider, "chatHostModel", "gpt-5.5"); + String body = provider.buildResponsesBody("hi", "1024x1024", "medium"); + assertEquals("gpt-5.5", objectMapper.readTree(body).path("model").asText()); + } + + // ==================== quality & size mapping ============================ + + @Test + @DisplayName("qualityForRequest reads tier from model id; falls back to default") + void qualityForRequest_tiersAndDefault() { + assertEquals("low", provider.qualityForRequest( + ImageGenerationRequest.builder().prompt("x").model("gpt-image-2-low").build())); + assertEquals("medium", provider.qualityForRequest( + ImageGenerationRequest.builder().prompt("x").model("gpt-image-2-medium").build())); + assertEquals("high", provider.qualityForRequest( + ImageGenerationRequest.builder().prompt("x").model("gpt-image-2-high").build())); + // unknown model id → fall back to configured default + assertEquals("medium", provider.qualityForRequest( + ImageGenerationRequest.builder().prompt("x").model("gpt-5.4").build())); + assertEquals("medium", provider.qualityForRequest( + ImageGenerationRequest.builder().prompt("x").build())); + } + + @Test + @DisplayName("normalizeSize honours explicit supported size, then aspect ratio, then defaults") + void normalizeSize_priorityOrder() { + assertEquals("1024x1024", provider.normalizeSize("1024x1024", "1:1")); + assertEquals("1536x1024", provider.normalizeSize("1536x1024", "1:1")); + assertEquals("1024x1536", provider.normalizeSize(null, "9:16")); + assertEquals("1536x1024", provider.normalizeSize(null, "16:9")); + assertEquals("1024x1024", provider.normalizeSize(null, null)); + // unsupported size → fall through to aspect ratio + assertEquals("1536x1024", provider.normalizeSize("9999x9999", "16:9")); + } + + // ==================== SSE parsing ======================================== + + @Test + @DisplayName("SSE parser returns final image from response.output_item.done") + void sseParser_returnsFinalImage() { + String body = + "event: response.image_generation_call.partial_image\n" + + "data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_b64\":\"PARTIAL\"}\n" + + "\n" + + "event: response.output_item.done\n" + + "data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"image_generation_call\",\"result\":\"FINAL\"}}\n" + + "\n"; + assertEquals("FINAL", provider.extractFinalImageFromSseBody(body)); + } + + @Test + @DisplayName("SSE parser falls back to the latest partial image if the final frame is missing") + void sseParser_fallsBackToPartial() { + String body = + "event: response.image_generation_call.partial_image\n" + + "data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_b64\":\"FIRST\"}\n" + + "\n" + + "event: response.image_generation_call.partial_image\n" + + "data: {\"type\":\"response.image_generation_call.partial_image\",\"partial_image_b64\":\"SECOND\"}\n" + + "\n"; + assertEquals("SECOND", provider.extractFinalImageFromSseBody(body)); + } + + @Test + @DisplayName("SSE parser also reads image from response.completed.output[]") + void sseParser_readsFromResponseCompleted() { + String body = + "event: response.completed\n" + + "data: {\"type\":\"response.completed\",\"response\":{\"output\":[{\"type\":\"image_generation_call\",\"result\":\"DONE\"}]}}\n" + + "\n"; + assertEquals("DONE", provider.extractFinalImageFromSseBody(body)); + } + + @Test + @DisplayName("SSE parser ignores [DONE] sentinels and unparseable frames") + void sseParser_ignoresNoiseFrames() { + String body = + ":heartbeat\n\n" + + "data: [DONE]\n\n" + + "data: not json at all\n\n" + + "event: response.output_item.done\n" + + "data: {\"type\":\"response.output_item.done\",\"item\":{\"type\":\"image_generation_call\",\"result\":\"REAL\"}}\n\n"; + assertEquals("REAL", provider.extractFinalImageFromSseBody(body)); + } + + @Test + @DisplayName("SSE parser returns null when there is no image in any frame") + void sseParser_returnsNullWhenNoImage() { + assertNull(provider.extractFinalImageFromSseBody("")); + assertNull(provider.extractFinalImageFromSseBody(null)); + assertNull(provider.extractFinalImageFromSseBody( + "event: response.created\ndata: {\"type\":\"response.created\"}\n\n")); + } + + // ==================== capability surface ================================= + + @Test + @DisplayName("detailedCapabilities exposes the three gpt-image-2 tiers and right sizes") + void detailedCapabilities_advertisesTiers() { + ImageProviderCapabilities caps = provider.detailedCapabilities(); + assertEquals("gpt-image-2-medium", caps.getDefaultModel()); + assertTrue(caps.getModels().containsAll( + java.util.List.of("gpt-image-2-low", "gpt-image-2-medium", "gpt-image-2-high"))); + assertTrue(caps.getSupportedSizes().contains("1536x1024")); + assertTrue(caps.getSupportedSizes().contains("1024x1536")); + assertEquals(1, caps.getMaxCount()); + } + + @Test + @DisplayName("provider id matches the existing OAuth provider id, label is descriptive") + void identityFields() { + assertEquals("openai-chatgpt", provider.id()); + assertTrue(provider.label().contains("ChatGPT")); + assertTrue(provider.requiresCredential()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageModelsTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageModelsTest.java new file mode 100644 index 00000000..e0eeb4f4 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageModelsTest.java @@ -0,0 +1,105 @@ +package vip.mate.tool.image.provider; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.ImageModelSpec; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Catalog-shape invariants on the DashScope image model registry. The point of + * these is not to assert specific model ids — those churn as Aliyun ships / + * deprecates families — but to enforce that whatever is registered is + * internally consistent: + *
    + *
  • Sync-transport models must hit the multimodal endpoint; async-transport + * models must hit the legacy image-generation endpoint.
  • + *
  • Edit-capable models must declare a positive {@code maxInputImages}.
  • + *
  • The {@code DEFAULT_EDIT_MODEL} must actually support {@link ImageCapability#IMAGE_EDIT}.
  • + *
  • Every model spec carries a non-empty endpoint, transport, and modes set.
  • + *
+ */ +@Tag("media-gen") +class DashScopeImageModelsTest { + + @Test + @DisplayName("every spec has non-null endpoint, transport, and at least one mode") + void everySpecIsWellFormed() { + Map all = DashScopeImageModels.all(); + assertFalse(all.isEmpty(), "catalog must not be empty"); + for (Map.Entry e : all.entrySet()) { + ImageModelSpec spec = e.getValue(); + assertEquals(e.getKey(), spec.id(), "map key must equal spec.id()"); + assertNotNull(spec.endpoint(), spec.id()); + assertFalse(spec.endpoint().isBlank(), spec.id()); + assertNotNull(spec.transport(), spec.id()); + assertNotNull(spec.modes(), spec.id()); + assertFalse(spec.modes().isEmpty(), spec.id()); + } + } + + @Test + @DisplayName("transport drives endpoint family (SYNC ⇒ multimodal-generation, ASYNC ⇒ image-generation)") + void transportMatchesEndpointFamily() { + for (ImageModelSpec spec : DashScopeImageModels.all().values()) { + switch (spec.transport()) { + case SYNC -> assertEquals(DashScopeImageModels.MULTIMODAL_ENDPOINT, spec.endpoint(), + "sync model " + spec.id() + " must use multimodal endpoint"); + case ASYNC -> assertEquals(DashScopeImageModels.LEGACY_ASYNC_ENDPOINT, spec.endpoint(), + "async model " + spec.id() + " must use legacy endpoint"); + } + } + } + + @Test + @DisplayName("edit-capable specs declare maxInputImages > 0") + void editCapableSpecsDeclareInputCapacity() { + for (ImageModelSpec spec : DashScopeImageModels.all().values()) { + if (spec.supportsEdit()) { + assertTrue(spec.maxInputImages() > 0, + "edit-capable model " + spec.id() + " has maxInputImages=" + spec.maxInputImages()); + } + } + } + + @Test + @DisplayName("DEFAULT_MODEL exists and supports text-to-image (the most common request)") + void defaultModelExistsAndGenerates() { + ImageModelSpec spec = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_MODEL); + assertNotNull(spec); + assertEquals(DashScopeImageModels.DEFAULT_MODEL, spec.id()); + assertTrue(spec.supportsGenerate(), + "default model must accept text-to-image requests"); + } + + @Test + @DisplayName("DEFAULT_EDIT_MODEL exists and actually supports image edit") + void defaultEditModelExistsAndEdits() { + ImageModelSpec spec = DashScopeImageModels.get(DashScopeImageModels.DEFAULT_EDIT_MODEL); + assertNotNull(spec); + assertTrue(spec.supportsEdit(), + "DEFAULT_EDIT_MODEL must declare IMAGE_EDIT capability"); + } + + @Test + @DisplayName("get(unknown) falls back to DEFAULT_MODEL rather than returning null") + void unknownModelFallsBackToDefault() { + ImageModelSpec spec = DashScopeImageModels.get("not-a-real-model-id"); + assertEquals(DashScopeImageModels.DEFAULT_MODEL, spec.id()); + } + + @Test + @DisplayName("get(null) and get(blank) fall back to DEFAULT_MODEL") + void nullOrBlankModelFallsBackToDefault() { + assertEquals(DashScopeImageModels.DEFAULT_MODEL, DashScopeImageModels.get(null).id()); + assertEquals(DashScopeImageModels.DEFAULT_MODEL, DashScopeImageModels.get("").id()); + assertEquals(DashScopeImageModels.DEFAULT_MODEL, DashScopeImageModels.get(" ").id()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageProviderRoutingTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageProviderRoutingTest.java new file mode 100644 index 00000000..68acc5ae --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/DashScopeImageProviderRoutingTest.java @@ -0,0 +1,93 @@ +package vip.mate.tool.image.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.tool.image.ImageGenerationRequest; +import vip.mate.tool.image.ImageModelSpec; +import vip.mate.tool.image.ImageReference; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Unit-level checks on the per-request model routing in + * {@link DashScopeImageProvider#resolveSpec(ImageGenerationRequest)}. The HTTP + * surface is excluded — that needs a mock server. The routing decision is the + * part that's easy to break and easy to verify cheaply. + */ +@Tag("media-gen") +class DashScopeImageProviderRoutingTest { + + private final DashScopeImageProvider provider = new DashScopeImageProvider(null, new ObjectMapper()); + + @Test + @DisplayName("text-to-image request with no model returns DEFAULT_MODEL") + void noModelNoInputs_resolvesDefault() { + ImageGenerationRequest req = ImageGenerationRequest.builder().prompt("hi").build(); + ImageModelSpec spec = provider.resolveSpec(req); + assertEquals(DashScopeImageModels.DEFAULT_MODEL, spec.id()); + } + + @Test + @DisplayName("text-to-image with explicit model id returns that exact spec") + void explicitModel_resolvesSameId() { + ImageGenerationRequest req = ImageGenerationRequest.builder() + .prompt("hi").model("z-image-turbo").build(); + ImageModelSpec spec = provider.resolveSpec(req); + assertEquals("z-image-turbo", spec.id()); + } + + @Test + @DisplayName("edit request with edit-capable model keeps that model") + void editCapableModel_keepsModel() { + ImageGenerationRequest req = ImageGenerationRequest.builder() + .prompt("change the background") + .model("qwen-image-edit") + .inputImages(List.of(new ImageReference(new byte[]{1}, "image/png", "x.png", "test"))) + .build(); + ImageModelSpec spec = provider.resolveSpec(req); + assertEquals("qwen-image-edit", spec.id()); + assertTrue(spec.supportsEdit()); + } + + @Test + @DisplayName("edit request with non-edit-capable model falls back to DEFAULT_EDIT_MODEL") + void editRequestOnNonEditModel_fallsBackToEditDefault() { + ImageGenerationRequest req = ImageGenerationRequest.builder() + .prompt("change the background") + .model("z-image-turbo") // text-to-image only + .inputImages(List.of(new ImageReference(new byte[]{1}, "image/png", "x.png", "test"))) + .build(); + ImageModelSpec spec = provider.resolveSpec(req); + assertEquals(DashScopeImageModels.DEFAULT_EDIT_MODEL, spec.id()); + assertTrue(spec.supportsEdit(), + "fallback target must actually support edits — that's the point of the fallback"); + assertNotEquals("z-image-turbo", spec.id()); + } + + @Test + @DisplayName("edit request with no model and inputs falls back to DEFAULT_EDIT_MODEL") + void editRequestNoModel_fallsBackToEditDefault() { + ImageGenerationRequest req = ImageGenerationRequest.builder() + .prompt("change the background") + .inputImages(List.of(new ImageReference(new byte[]{1}, "image/png", "x.png", "test"))) + .build(); + ImageModelSpec spec = provider.resolveSpec(req); + // DEFAULT_MODEL is a legacy text-only async model — edit request must not land there. + assertEquals(DashScopeImageModels.DEFAULT_EDIT_MODEL, spec.id()); + assertTrue(spec.supportsEdit()); + } + + @Test + @DisplayName("provider declares both TEXT_TO_IMAGE and IMAGE_EDIT capabilities at provider level") + void providerDeclaresBothCapabilities() { + var caps = provider.capabilities(); + assertTrue(caps.contains(vip.mate.tool.image.ImageCapability.TEXT_TO_IMAGE)); + assertTrue(caps.contains(vip.mate.tool.image.ImageCapability.IMAGE_EDIT)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/provider/MiniMaxImageProviderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/MiniMaxImageProviderTest.java new file mode 100644 index 00000000..d87bd94c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/MiniMaxImageProviderTest.java @@ -0,0 +1,50 @@ +package vip.mate.tool.image.provider; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.system.model.SystemSettingsDTO; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Region-routing pin for {@link MiniMaxImageProvider}. Image and video share + * the same {@code minimaxRegion} field on {@link SystemSettingsDTO} — + * verifying both providers land on the same host when region is set + * prevents the "image works in CN but video times out" footgun. + */ +@Tag("media-gen") +class MiniMaxImageProviderTest { + + @Test + @DisplayName("resolveBaseUrl: minimaxRegion='cn' → CN endpoint (matches video provider)") + void resolveBaseUrl_cn() { + SystemSettingsDTO cfg = new SystemSettingsDTO(); + cfg.setMinimaxRegion("cn"); + assertEquals(MiniMaxImageProvider.BASE_URL_CN, MiniMaxImageProvider.resolveBaseUrl(cfg)); + } + + @Test + @DisplayName("resolveBaseUrl: default / null / 'global' → Global endpoint") + void resolveBaseUrl_default() { + SystemSettingsDTO cfg = new SystemSettingsDTO(); + assertEquals(MiniMaxImageProvider.BASE_URL_GLOBAL, + MiniMaxImageProvider.resolveBaseUrl(cfg)); + cfg.setMinimaxRegion("global"); + assertEquals(MiniMaxImageProvider.BASE_URL_GLOBAL, + MiniMaxImageProvider.resolveBaseUrl(cfg)); + assertEquals(MiniMaxImageProvider.BASE_URL_GLOBAL, + MiniMaxImageProvider.resolveBaseUrl(null)); + } + + @Test + @DisplayName("Host constants match MiniMax's documented endpoints") + void hostsAreCanonical() { + // Pin string values so a typo (e.g. minimax.com vs minimaxi.com) fails + // the test before users notice in production. The Video provider's + // constants are package-private — pinning by literal here cross-checks + // the image provider without leaking visibility. + assertEquals("https://api.minimax.io", MiniMaxImageProvider.BASE_URL_GLOBAL); + assertEquals("https://api.minimaxi.com", MiniMaxImageProvider.BASE_URL_CN); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/provider/OpenAiImageProviderGptImage2Test.java b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/OpenAiImageProviderGptImage2Test.java new file mode 100644 index 00000000..101c3818 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/provider/OpenAiImageProviderGptImage2Test.java @@ -0,0 +1,143 @@ +package vip.mate.tool.image.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.tool.image.ImageProviderCapabilities; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@link OpenAiImageProvider} GPT-Image-2 wiring. + * + *

Inspired by hermes-agent's plugins/image_gen/openai/__init__.py — three + * virtual model IDs (gpt-image-2-low/medium/high) all map to API model + * {@code gpt-image-2} with a different {@code quality} parameter. The new + * size set is 1024x1024 / 1024x1536 / 1536x1024, distinct from DALL-E's + * 1024x1024 / 1024x1792 / 1792x1024. + * + *

Tests focus on the pure-logic helpers (capabilities catalog, tier→quality + * mapping, model dispatch detection, size normalization). HTTP submission is + * not exercised here — that requires either a live OPENAI_API_KEY or an HTTP + * mock framework. The split-out unit tests cover everything that isn't + * literally "did the network return 200". + */ +@Tag("media-gen") +class OpenAiImageProviderGptImage2Test { + + private OpenAiImageProvider newProvider() { + // ModelProviderService is only consulted inside submit(); the helper + // methods we exercise here don't touch it. null is safe. + return new OpenAiImageProvider(null, new ObjectMapper()); + } + + @Test + @DisplayName("detailedCapabilities lists all three gpt-image-2 tiers + DALL-E models") + void capabilities_listAllModels() { + ImageProviderCapabilities caps = newProvider().detailedCapabilities(); + + assertTrue(caps.getModels().contains("dall-e-3")); + assertTrue(caps.getModels().contains("dall-e-2")); + assertTrue(caps.getModels().contains("gpt-image-1")); + assertTrue(caps.getModels().contains("gpt-image-2-low"), + "gpt-image-2-low must be picker-visible"); + assertTrue(caps.getModels().contains("gpt-image-2-medium")); + assertTrue(caps.getModels().contains("gpt-image-2-high")); + + assertEquals("dall-e-3", caps.getDefaultModel(), + "Default stays dall-e-3 — gpt-image-2 is opt-in by selecting tier"); + } + + @Test + @DisplayName("detailedCapabilities supportedSizes covers both DALL-E and gpt-image-2 sizes") + void capabilities_unionOfSizes() { + ImageProviderCapabilities caps = newProvider().detailedCapabilities(); + + // DALL-E sizes + assertTrue(caps.getSupportedSizes().contains("1024x1024")); + assertTrue(caps.getSupportedSizes().contains("1024x1792")); + assertTrue(caps.getSupportedSizes().contains("1792x1024")); + + // gpt-image-2 sizes (NOT identical to DALL-E) + assertTrue(caps.getSupportedSizes().contains("1024x1536")); + assertTrue(caps.getSupportedSizes().contains("1536x1024")); + } + + @Test + @DisplayName("isGptImage2Tier identifies the three virtual IDs and rejects others") + void isGptImage2Tier_correctDispatch() { + assertTrue(OpenAiImageProvider.isGptImage2Tier("gpt-image-2-low")); + assertTrue(OpenAiImageProvider.isGptImage2Tier("gpt-image-2-medium")); + assertTrue(OpenAiImageProvider.isGptImage2Tier("gpt-image-2-high")); + + assertFalse(OpenAiImageProvider.isGptImage2Tier("gpt-image-2")); + assertFalse(OpenAiImageProvider.isGptImage2Tier("dall-e-3")); + assertFalse(OpenAiImageProvider.isGptImage2Tier("dall-e-2")); + assertFalse(OpenAiImageProvider.isGptImage2Tier("gpt-image-1")); + assertFalse(OpenAiImageProvider.isGptImage2Tier(null)); + assertFalse(OpenAiImageProvider.isGptImage2Tier("")); + } + + @Test + @DisplayName("qualityForTier maps each virtual ID to the right quality string") + void qualityForTier_correctMapping() { + assertEquals("low", OpenAiImageProvider.qualityForTier("gpt-image-2-low")); + assertEquals("medium", OpenAiImageProvider.qualityForTier("gpt-image-2-medium")); + assertEquals("high", OpenAiImageProvider.qualityForTier("gpt-image-2-high")); + + // Defensive: any unrecognised id falls back to medium (sane default; + // matches hermes-agent DEFAULT_MODEL = gpt-image-2-medium). + assertEquals("medium", OpenAiImageProvider.qualityForTier("anything-else")); + assertEquals("medium", OpenAiImageProvider.qualityForTier("")); + } + + @Test + @DisplayName("normalizeSize: gpt-image-2 path picks gpt-image-2 sizes from aspect ratio") + void normalizeSize_gptImage2_byAspectRatio() { + OpenAiImageProvider p = newProvider(); + + assertEquals("1024x1024", p.normalizeSize(null, "1:1", true)); + assertEquals("1024x1536", p.normalizeSize(null, "9:16", true), + "Portrait must map to gpt-image-2's 1024x1536, NOT dall-e's 1024x1792"); + assertEquals("1536x1024", p.normalizeSize(null, "16:9", true), + "Landscape must map to gpt-image-2's 1536x1024, NOT dall-e's 1792x1024"); + } + + @Test + @DisplayName("normalizeSize: dall-e path keeps original 1024x1792 / 1792x1024 sizes") + void normalizeSize_dallE_unchanged() { + OpenAiImageProvider p = newProvider(); + + assertEquals("1024x1024", p.normalizeSize(null, "1:1", false)); + assertEquals("1024x1792", p.normalizeSize(null, "9:16", false)); + assertEquals("1792x1024", p.normalizeSize(null, "16:9", false)); + } + + @Test + @DisplayName("normalizeSize: explicit size honored only when supported by selected model family") + void normalizeSize_explicitSizeRespectsModelFamily() { + OpenAiImageProvider p = newProvider(); + + // gpt-image-2 explicit size hit + assertEquals("1536x1024", p.normalizeSize("1536x1024", "1:1", true)); + // gpt-image-2 explicit size MISS (DALL-E size given to gpt-image-2 → fall back to aspect) + assertEquals("1024x1024", p.normalizeSize("1792x1024", "1:1", true)); + + // dall-e explicit size hit + assertEquals("1792x1024", p.normalizeSize("1792x1024", "1:1", false)); + // dall-e explicit size MISS (gpt-image-2 size given to dall-e → fall back to aspect) + assertEquals("1024x1024", p.normalizeSize("1536x1024", "1:1", false)); + } + + @Test + @DisplayName("normalizeSize: extra gpt-image-2 aspect-ratio aliases (3:4, 2:3, 4:3, 3:2) work") + void normalizeSize_gptImage2_extraAspectAliases() { + OpenAiImageProvider p = newProvider(); + // Per hermes-agent's spec: portrait aliases → 1024x1536, landscape → 1536x1024 + assertEquals("1024x1536", p.normalizeSize(null, "3:4", true)); + assertEquals("1024x1536", p.normalizeSize(null, "2:3", true)); + assertEquals("1536x1024", p.normalizeSize(null, "4:3", true)); + assertEquals("1536x1024", p.normalizeSize(null, "3:2", true)); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/vision/ImageVisionServiceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/vision/ImageVisionServiceTest.java new file mode 100644 index 00000000..28231bd5 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/vision/ImageVisionServiceTest.java @@ -0,0 +1,226 @@ +package vip.mate.tool.image.vision; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.exception.MateClawException; +import vip.mate.system.featureflag.FeatureFlagService; +import vip.mate.system.featureflag.FlagContext; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.system.service.SystemSettingService; +import vip.mate.tool.image.ImageCapability; +import vip.mate.wiki.metrics.WikiMetrics; +import vip.mate.wiki.model.WikiImageCaptionCacheEntity; +import vip.mate.wiki.service.WikiImageCaptionCacheService; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.eq; + +/** + * Unit tests for {@link ImageVisionService}. + * + *

Covers feature-flag short-circuit, cache-hit fast path, provider + * fallback chain (failure of higher-priority provider falls through to + * the next), all-failed case, and persist-after-success. + */ +class ImageVisionServiceTest { + + private WikiImageCaptionCacheService cacheService; + private SystemSettingService settingService; + private FeatureFlagService featureFlag; + private WikiMetrics metrics; + + @BeforeEach + void setUp() { + cacheService = mock(WikiImageCaptionCacheService.class); + settingService = mock(SystemSettingService.class); + featureFlag = mock(FeatureFlagService.class); + metrics = mock(WikiMetrics.class); + when(settingService.getSettings()).thenReturn(new SystemSettingsDTO()); + // Default: feature flag on + when(featureFlag.isEnabled("wiki.ocr.enabled")).thenReturn(true); + } + + @Test + @DisplayName("Disabled feature flag short-circuits with err.wiki.vision.disabled") + void disabledFlag_shortCircuits() { + when(featureFlag.isEnabled(anyString())).thenReturn(false); + ImageVisionService service = newService(List.of(stubProvider("p1", true, sampleResult("a")))); + + assertThatThrownBy(() -> service.caption(sampleRequest())) + .isInstanceOf(MateClawException.class) + .hasMessageContaining("disabled"); + } + + @Test + @DisplayName("Empty / null image bytes rejected with IllegalArgumentException") + void emptyImage_rejected() { + ImageVisionService service = newService(List.of()); + + assertThatThrownBy(() -> service.caption(null)) + .isInstanceOf(IllegalArgumentException.class); + + VisionRequest empty = VisionRequest.builder().imageBytes(new byte[0]).mimeType("image/png").build(); + assertThatThrownBy(() -> service.caption(empty)) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + @DisplayName("Cache hit returns immediately and skips provider chain") + void cacheHit_skipsProviders() { + WikiImageCaptionCacheEntity row = sampleCacheRow(); + when(cacheService.lookup(anyString())).thenReturn(Optional.of(row)); + + ImageVisionProvider p1 = stubProvider("p1", true, sampleResult("would-have-called")); + ImageVisionService service = newService(List.of(p1)); + + VisionResult result = service.caption(sampleRequest()); + + assertThat(result.getCaption()).isEqualTo(row.getCaption()); + assertThat(result.getProviderId()).isEqualTo(row.getProviderId()); + verify(p1, never()).caption(any(), any()); + verify(metrics).recordVisionCacheHit(true); + verify(cacheService, never()).persist(any()); + } + + @Test + @DisplayName("No available provider → err.wiki.vision.no_provider") + void noAvailableProvider_throws() { + when(cacheService.lookup(anyString())).thenReturn(Optional.empty()); + ImageVisionProvider p = stubProvider("p", false, null); + ImageVisionService service = newService(List.of(p)); + + assertThatThrownBy(() -> service.caption(sampleRequest())) + .isInstanceOf(MateClawException.class) + .hasMessageContaining("provider"); + } + + @Test + @DisplayName("First provider failure falls through to second in autoDetectOrder") + void firstFails_secondSucceeds() { + when(cacheService.lookup(anyString())).thenReturn(Optional.empty()); + ImageVisionProvider p1 = stubProvider("p1", true, null); // null result via throwing + when(p1.caption(any(), any())).thenThrow(new RuntimeException("rate-limited")); + when(p1.autoDetectOrder()).thenReturn(10); + + VisionResult win = sampleResult("from p2"); + ImageVisionProvider p2 = stubProvider("p2", true, win); + when(p2.autoDetectOrder()).thenReturn(20); + + ImageVisionService service = newService(List.of(p1, p2)); + + VisionResult result = service.caption(sampleRequest()); + + assertThat(result.getCaption()).isEqualTo("from p2"); + verify(p1).caption(any(), any()); + verify(p2).caption(any(), any()); + verify(cacheService).persist(any()); + verify(metrics).recordVisionCall(eq("p1"), eq(false), any()); + verify(metrics).recordVisionCall(eq("p2"), eq(true), any()); + } + + @Test + @DisplayName("All providers fail → err.wiki.vision.all_failed") + void allFail_throws() { + when(cacheService.lookup(anyString())).thenReturn(Optional.empty()); + ImageVisionProvider p1 = stubProvider("p1", true, null); + when(p1.caption(any(), any())).thenThrow(new RuntimeException("HTTP 500")); + ImageVisionService service = newService(List.of(p1)); + + assertThatThrownBy(() -> service.caption(sampleRequest())) + .isInstanceOf(MateClawException.class) + .hasMessageContaining("All image vision providers failed"); + verify(cacheService, never()).persist(any()); + } + + @Test + @DisplayName("Lower autoDetectOrder is tried first") + void orderingHonored() { + when(cacheService.lookup(anyString())).thenReturn(Optional.empty()); + VisionResult r1 = sampleResult("from p-low"); + ImageVisionProvider pLow = stubProvider("p-low", true, r1); + when(pLow.autoDetectOrder()).thenReturn(10); + + ImageVisionProvider pHigh = stubProvider("p-high", true, sampleResult("from p-high")); + when(pHigh.autoDetectOrder()).thenReturn(99); + + // Pass in reversed order to confirm internal sort. + ImageVisionService service = newService(List.of(pHigh, pLow)); + + VisionResult result = service.caption(sampleRequest()); + + assertThat(result.getCaption()).isEqualTo("from p-low"); + verify(pHigh, never()).caption(any(), any()); + } + + @Test + @DisplayName("Same image bytes always produce the same SHA-256 hex") + void sha256_stable() { + byte[] bytes = "hello world".getBytes(); + String a = ImageVisionService.sha256Hex(bytes); + String b = ImageVisionService.sha256Hex(bytes); + assertThat(a).isEqualTo(b).hasSize(64); + } + + // ==================== helpers ==================== + + private ImageVisionService newService(List providers) { + return new ImageVisionService(providers, cacheService, settingService, featureFlag, metrics); + } + + private static VisionRequest sampleRequest() { + return VisionRequest.builder() + .imageBytes(new byte[]{1, 2, 3, 4}) + .mimeType("image/png") + .build(); + } + + private static VisionResult sampleResult(String caption) { + return VisionResult.builder() + .caption(caption) + .providerId("test-provider") + .model("test-model") + .capturedAt(Instant.now()) + .durationMs(123L) + .build(); + } + + private static WikiImageCaptionCacheEntity sampleCacheRow() { + WikiImageCaptionCacheEntity row = new WikiImageCaptionCacheEntity(); + row.setImageSha256("0123456789abcdef".repeat(4)); + row.setCaption("cached caption"); + row.setCaptureModel("cached-model"); + row.setProviderId("cached-provider"); + row.setCapturedAt(LocalDateTime.now()); + row.setDurationMs(0L); + return row; + } + + private static ImageVisionProvider stubProvider(String id, boolean available, VisionResult result) { + ImageVisionProvider provider = mock(ImageVisionProvider.class); + when(provider.id()).thenReturn(id); + when(provider.label()).thenReturn(id); + when(provider.requiresCredential()).thenReturn(true); + when(provider.autoDetectOrder()).thenReturn(50); + when(provider.capabilities()).thenReturn(Set.of(ImageCapability.IMAGE_TO_TEXT)); + when(provider.isAvailable(any())).thenReturn(available); + if (result != null) { + when(provider.caption(any(), any())).thenReturn(result); + } + return provider; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/image/vision/provider/VisionProviderIdentityTest.java b/mateclaw-server/src/test/java/vip/mate/tool/image/vision/provider/VisionProviderIdentityTest.java new file mode 100644 index 00000000..f0cc5179 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/image/vision/provider/VisionProviderIdentityTest.java @@ -0,0 +1,119 @@ +package vip.mate.tool.image.vision.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.image.ImageCapability; +import vip.mate.tool.image.vision.ImageVisionProvider; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Identity + ordering contract for the OpenAI-compatible vision + * providers. Verifies each provider exposes a stable id, a sane + * autoDetectOrder, IMAGE_TO_TEXT capability, and that the auto-detect + * ordering across all three is monotonically increasing — operators + * relying on "DashScope wins when both are configured" depend on this. + */ +class VisionProviderIdentityTest { + + private final ModelProviderService modelProviderService = mock(ModelProviderService.class); + private final ObjectMapper objectMapper = new ObjectMapper(); + + private DashScopeVisionProvider dashScope() { + return new DashScopeVisionProvider(modelProviderService, objectMapper); + } + + private ZhipuVisionProvider zhipu() { + return new ZhipuVisionProvider(modelProviderService, objectMapper); + } + + private DoubaoVisionProvider doubao() { + return new DoubaoVisionProvider(modelProviderService, objectMapper); + } + + @Test + @DisplayName("DashScope provider keeps its public id and order") + void dashScopeIdentity() { + ImageVisionProvider p = dashScope(); + assertThat(p.id()).isEqualTo("dashscope-vision"); + assertThat(p.label()).isEqualTo("DashScope qwen-vl"); + assertThat(p.autoDetectOrder()).isEqualTo(10); + assertThat(p.requiresCredential()).isTrue(); + assertThat(p.capabilities()).contains(ImageCapability.IMAGE_TO_TEXT); + } + + @Test + @DisplayName("Zhipu provider exposes its own id, slot 20") + void zhipuIdentity() { + ImageVisionProvider p = zhipu(); + assertThat(p.id()).isEqualTo("zhipu-vision"); + assertThat(p.label()).isEqualTo("Zhipu GLM-V"); + assertThat(p.autoDetectOrder()).isEqualTo(20); + assertThat(p.capabilities()).contains(ImageCapability.IMAGE_TO_TEXT); + } + + @Test + @DisplayName("Doubao provider exposes its own id, slot 30") + void doubaoIdentity() { + ImageVisionProvider p = doubao(); + assertThat(p.id()).isEqualTo("doubao-vision"); + assertThat(p.label()).isEqualTo("Volcano Doubao Vision"); + assertThat(p.autoDetectOrder()).isEqualTo(30); + assertThat(p.capabilities()).contains(ImageCapability.IMAGE_TO_TEXT); + } + + @Test + @DisplayName("auto-detect ordering: DashScope < Zhipu < Doubao") + void orderingAcrossProviders() { + List orders = List.of( + dashScope().autoDetectOrder(), + zhipu().autoDetectOrder(), + doubao().autoDetectOrder()); + assertThat(orders).isSorted(); + assertThat(orders).doesNotHaveDuplicates(); + } + + @Test + @DisplayName("isAvailable: each provider checks its own model_provider key") + void availabilityChecksDelegate() { + SystemSettingsDTO settings = new SystemSettingsDTO(); + when(modelProviderService.isProviderConfigured(anyString())).thenReturn(false); + + assertThat(dashScope().isAvailable(settings)).isFalse(); + assertThat(zhipu().isAvailable(settings)).isFalse(); + assertThat(doubao().isAvailable(settings)).isFalse(); + + // Each provider must look up by the right provider_id + when(modelProviderService.isProviderConfigured("dashscope")).thenReturn(true); + assertThat(dashScope().isAvailable(settings)).isTrue(); + assertThat(zhipu().isAvailable(settings)).isFalse(); + assertThat(doubao().isAvailable(settings)).isFalse(); + + when(modelProviderService.isProviderConfigured("zhipu-cn")).thenReturn(true); + assertThat(zhipu().isAvailable(settings)).isTrue(); + assertThat(doubao().isAvailable(settings)).isFalse(); + + when(modelProviderService.isProviderConfigured("volcengine")).thenReturn(true); + assertThat(doubao().isAvailable(settings)).isTrue(); + } + + @Test + @DisplayName("isAvailable returns false when ModelProviderService throws — fail-soft") + void availabilityFailsSoft() { + SystemSettingsDTO settings = new SystemSettingsDTO(); + when(modelProviderService.isProviderConfigured(anyString())) + .thenThrow(new RuntimeException("db down")); + + assertThat(dashScope().isAvailable(settings)).isFalse(); + assertThat(zhipu().isAvailable(settings)).isFalse(); + assertThat(doubao().isAvailable(settings)).isFalse(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerSplitHttpUrlTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerSplitHttpUrlTest.java new file mode 100644 index 00000000..97a64610 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerSplitHttpUrlTest.java @@ -0,0 +1,125 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.tool.mcp.runtime.McpClientManager.HttpEndpointConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Verifies that {@link McpClientManager#splitHttpUrl(String, String)} produces + * the {@code baseUrl} / {@code endpoint} pair the underlying SDK builders + * expect, so a user-configured non-default path or query string is not + * silently dropped. + */ +class McpClientManagerSplitHttpUrlTest { + + @Test + @DisplayName("URL without path falls back to the transport's default endpoint") + void hostOnlyUsesDefaultEndpoint() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("Bare slash path is treated as no path") + void rootPathUsesDefaultEndpoint() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com/", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("Standard /mcp suffix round-trips") + void standardMcpSuffix() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com/mcp", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("Non-standard nested path is preserved as endpoint") + void nonStandardPathPreserved() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://api.example.com/api/v1/mcp", "/mcp"); + assertEquals("https://api.example.com", cfg.baseUrl()); + assertEquals("/api/v1/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("Query string is appended to the endpoint") + void queryStringPreserved() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com/mcp?token=abc", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp?token=abc", cfg.endpoint()); + } + + @Test + @DisplayName("Query string survives even when path is empty") + void queryStringWithoutPath() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com?token=abc", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp?token=abc", cfg.endpoint()); + } + + @Test + @DisplayName("Port and userinfo stay on the base URL") + void hostWithPort() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("http://localhost:8080/api/mcp", "/mcp"); + assertEquals("http://localhost:8080", cfg.baseUrl()); + assertEquals("/api/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("IPv6 authority is preserved") + void ipv6Host() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("http://[::1]:8080/mcp", "/mcp"); + assertEquals("http://[::1]:8080", cfg.baseUrl()); + assertEquals("/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("SSE default endpoint is honoured") + void sseDefaultEndpoint() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl("https://example.com", "/sse"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/sse", cfg.endpoint()); + } + + @Test + @DisplayName("Whitespace around URL is trimmed") + void trimsWhitespace() { + HttpEndpointConfig cfg = McpClientManager.splitHttpUrl(" https://example.com/mcp ", "/mcp"); + assertEquals("https://example.com", cfg.baseUrl()); + assertEquals("/mcp", cfg.endpoint()); + } + + @Test + @DisplayName("Null URL is rejected") + void nullUrlThrows() { + assertThrows(IllegalArgumentException.class, + () -> McpClientManager.splitHttpUrl(null, "/mcp")); + } + + @Test + @DisplayName("Empty URL is rejected") + void emptyUrlThrows() { + assertThrows(IllegalArgumentException.class, + () -> McpClientManager.splitHttpUrl(" ", "/mcp")); + } + + @Test + @DisplayName("Missing scheme is rejected") + void missingSchemeThrows() { + assertThrows(IllegalArgumentException.class, + () -> McpClientManager.splitHttpUrl("example.com/mcp", "/mcp")); + } + + @Test + @DisplayName("Malformed URL is rejected") + void malformedUrlThrows() { + assertThrows(IllegalArgumentException.class, + () -> McpClientManager.splitHttpUrl("http://exa mple.com/mcp", "/mcp")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerWrapTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerWrapTest.java new file mode 100644 index 00000000..4d839169 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpClientManagerWrapTest.java @@ -0,0 +1,156 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; + +import java.util.List; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Drives {@link McpClientManager#wrapServerCallbacks(long, ToolCallback[])} + * directly so the manager's collision-and-skip logic can be exercised + * without standing up a real MCP client. + */ +class McpClientManagerWrapTest { + + @Test + @DisplayName("two distinct raw callbacks both wrap and survive") + void twoDistinctCallbacksSurvive() { + ToolCallback a = stub("create_issue"); + ToolCallback b = stub("list_issues"); + + List wrapped = McpClientManager.wrapServerCallbacks(42L, + new ToolCallback[]{a, b}); + + assertEquals(2, wrapped.size()); + assertEquals(McpToolNameResolver.prefixedName(42L, "create_issue"), + wrapped.get(0).getToolDefinition().name()); + assertEquals(McpToolNameResolver.prefixedName(42L, "list_issues"), + wrapped.get(1).getToolDefinition().name()); + } + + @Test + @DisplayName("duplicate raw callback: only the first survives, second is skipped") + void duplicateRawSecondCallbackSkipped() { + // The previous Map shape would have looked up the + // first (bindable) decision for both callbacks, registering two + // wrapped callbacks under the same prefixed name. Lockstep + // alignment prevents that — the second should be dropped before + // wrapping happens. + ToolCallback first = stub("search"); + ToolCallback duplicate = stub("search"); + + List wrapped = McpClientManager.wrapServerCallbacks(42L, + new ToolCallback[]{first, duplicate}); + + assertEquals(1, wrapped.size()); + assertSame(((PrefixedNameToolCallback) wrapped.get(0)).getDelegate(), first); + } + + @Test + @DisplayName("hash-colliding raw pair: only the first survives") + void hashCollisionSecondCallbackSkipped() { + String[] pair = McpHashCollisionDetectorTest.hashCollidingPair(); + if (pair == null) { + // The detector test guarantees @BeforeAll populates the pair + // when this class runs alongside it; if it ran in isolation we + // recompute defensively. Either way the assertion below holds. + pair = findPair(); + } + ToolCallback first = stub(pair[0]); + ToolCallback collider = stub(pair[1]); + + List wrapped = McpClientManager.wrapServerCallbacks(42L, + new ToolCallback[]{first, collider}); + + assertEquals(1, wrapped.size()); + assertSame(((PrefixedNameToolCallback) wrapped.get(0)).getDelegate(), first); + } + + @Test + @DisplayName("blank raw is dropped without consuming a decision") + void blankRawDoesNotMisalignDecisions() { + ToolCallback good = stub("search"); + // DefaultToolDefinition's builder rejects blank names, so we build + // a hand-rolled ToolCallback whose ToolDefinition reports an empty + // string. The defensive blank-name handling in wrapServerCallbacks + // is exactly what protects against this kind of upstream surprise. + ToolCallback blank = new BlankNameCallback(); + ToolCallback alsoGood = stub("read_file"); + + List wrapped = McpClientManager.wrapServerCallbacks(42L, + new ToolCallback[]{good, blank, alsoGood}); + + // Both real callbacks survive; the blank entry is silently dropped + // and does NOT advance the decision pointer, otherwise alsoGood + // would have looked up search's bindable decision and wrapped under + // the wrong name. + assertEquals(2, wrapped.size()); + List names = wrapped.stream() + .map(cb -> cb.getToolDefinition().name()) + .collect(Collectors.toList()); + assertTrue(names.contains(McpToolNameResolver.prefixedName(42L, "search"))); + assertTrue(names.contains(McpToolNameResolver.prefixedName(42L, "read_file"))); + } + + /** Callback that surfaces a blank ToolDefinition.name() — exists only so + * the test can drive the defensive branch in {@code wrapServerCallbacks} + * that the upstream builder otherwise prevents. */ + private static final class BlankNameCallback implements ToolCallback { + private final ToolDefinition def = new ToolDefinition() { + @Override public String name() { return ""; } + @Override public String description() { return ""; } + @Override public String inputSchema() { return "{}"; } + }; + @Override public ToolDefinition getToolDefinition() { return def; } + @Override public ToolMetadata getToolMetadata() { return ToolCallback.super.getToolMetadata(); } + @Override public String call(String toolInput) { return ""; } + @Override public String call(String toolInput, ToolContext toolContext) { return ""; } + } + + @Test + @DisplayName("empty input returns an empty list") + void emptyInput() { + List wrapped = McpClientManager.wrapServerCallbacks(42L, new ToolCallback[0]); + assertEquals(0, wrapped.size()); + } + + private static String[] findPair() { + String anchor = "xxxxxxxxxxxxxxxxxxxx"; + java.util.Map seen = new java.util.HashMap<>(); + for (int i = 0; i < 1_000_000; i++) { + String raw = anchor + i; + String hash = McpToolNameResolver.hash6(raw); + String prior = seen.put(hash, raw); + if (prior != null) return new String[]{prior, raw}; + } + throw new IllegalStateException("hash distribution broken"); + } + + private static ToolCallback stub(String name) { + ToolDefinition def = DefaultToolDefinition.builder() + .name(name) + .description("") + .inputSchema("{}") + .build(); + return new ToolCallback() { + @Override + public ToolDefinition getToolDefinition() { return def; } + @Override + public ToolMetadata getToolMetadata() { return ToolCallback.super.getToolMetadata(); } + @Override + public String call(String toolInput) { return name + ":" + toolInput; } + @Override + public String call(String toolInput, ToolContext toolContext) { return call(toolInput); } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetectorTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetectorTest.java new file mode 100644 index 00000000..41f39b6c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpHashCollisionDetectorTest.java @@ -0,0 +1,128 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class McpHashCollisionDetectorTest { + + /** + * A pair of raw names with identical 20-char slug AND identical hash6 — + * found once at startup via birthday-style search. The constant prefix + * truncates the slug to {@code "xxxxxxxxxxxxxxxxxxxx"} so the only + * remaining variable in {@code prefixedName} is the hash, and on a + * 30-bit hash space the birthday paradox finds a collision in + * ~32k tries on average. + * + *

Failing fast at {@link BeforeAll} keeps the actual test honest — + * a hung search would surface as a build hang, not a silent skip. + */ + private static String[] HASH_COLLIDING_PAIR; + + @BeforeAll + static void findHashCollidingPair() { + String slugAnchor = "xxxxxxxxxxxxxxxxxxxx"; // exactly 20 chars → fills the slug budget + Map hashToRaw = new HashMap<>(); + for (int i = 0; i < 1_000_000; i++) { + String raw = slugAnchor + i; + String hash = McpToolNameResolver.hash6(raw); + String prior = hashToRaw.put(hash, raw); + if (prior != null && !prior.equals(raw)) { + HASH_COLLIDING_PAIR = new String[]{prior, raw}; + return; + } + } + // Astronomically unlikely; only happens if hash6's distribution is + // catastrophically bad (test serves as a smoke check on resolver too). + throw new IllegalStateException("No hash collision found in 1M tries — resolver hash distribution may be broken"); + } + + @Test + @DisplayName("distinct raw names that don't hash-collide are all bindable") + void noCollisionAllBindable() { + List decisions = + McpHashCollisionDetector.classify(42L, List.of("search", "read_file", "create_issue")); + assertEquals(3, decisions.size()); + for (McpHashCollisionDetector.Decision d : decisions) { + assertTrue(d.bindable(), "expected bindable for " + d.rawToolName()); + assertEquals(McpToolNameResolver.prefixedName(42L, d.rawToolName()), d.prefixedName()); + } + } + + @Test + @DisplayName("duplicate raw names within one server only bind once") + void duplicateRawNameSecondInstanceIsNotBindable() { + // MCP servers are not supposed to surface the same name twice, but be + // defensive — drop the second declaration with a clear reason. + List decisions = + McpHashCollisionDetector.classify(42L, List.of("search", "search")); + assertEquals(2, decisions.size()); + assertTrue(decisions.get(0).bindable()); + assertFalse(decisions.get(1).bindable()); + assertEquals("DUPLICATE_RAW_NAME", decisions.get(1).unavailableReason()); + } + + @Test + @DisplayName("blank or null raw names are dropped silently") + void blankRawNamesAreSkipped() { + List decisions = + McpHashCollisionDetector.classify(42L, + Arrays.asList("search", null, "", " ")); + assertEquals(1, decisions.size()); + assertEquals("search", decisions.get(0).rawToolName()); + } + + @Test + @DisplayName("hash collision: the second raw name is flagged with a reason carrying the prior raw") + void hashCollisionFlagsSecondEntry() { + assertNotNull(HASH_COLLIDING_PAIR, "@BeforeAll should have populated a colliding pair"); + String a = HASH_COLLIDING_PAIR[0]; + String b = HASH_COLLIDING_PAIR[1]; + + // Sanity: the pair really does collide on the prefixed name. + assertNotEquals(a, b); + assertEquals(McpToolNameResolver.prefixedName(42L, a), + McpToolNameResolver.prefixedName(42L, b)); + + List decisions = + McpHashCollisionDetector.classify(42L, List.of(a, b)); + assertEquals(2, decisions.size()); + assertTrue(decisions.get(0).bindable()); + assertEquals(a, decisions.get(0).rawToolName()); + assertFalse(decisions.get(1).bindable()); + assertTrue(decisions.get(1).unavailableReason().startsWith("HASH_COLLISION:"), + "got reason: " + decisions.get(1).unavailableReason()); + // The reason carries the prior raw so the operator can map back to + // the upstream tool to rename. + assertTrue(decisions.get(1).unavailableReason().contains(a)); + } + + /** Exposes the colliding pair to other tests in the same package. */ + static String[] hashCollidingPair() { + return HASH_COLLIDING_PAIR; + } + + @Test + @DisplayName("two raw names same on different servers do not collide (anchored to serverId)") + void crossServerNotACollision() { + List a = + McpHashCollisionDetector.classify(42L, List.of("search")); + List b = + McpHashCollisionDetector.classify(43L, List.of("search")); + assertTrue(a.get(0).bindable()); + assertTrue(b.get(0).bindable()); + assertNotEquals(a.get(0).prefixedName(), b.get(0).prefixedName()); + } + +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolCallbackProviderReturnDirectTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolCallbackProviderReturnDirectTest.java new file mode 100644 index 00000000..dc2a4631 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolCallbackProviderReturnDirectTest.java @@ -0,0 +1,174 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; + +import java.util.List; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Guards the cross-form returnDirect match — without this, an existing + * deployment with raw tool names in its returnDirect config would silently + * lose the direct-return wrapping after Lane 0 starts handing back + * prefix-wrapped callbacks. That regression would let sensitive payloads + * (HR / medical / etc.) flow back through the LLM context, so the test is + * load-bearing for the upgrade. + */ +class McpToolCallbackProviderReturnDirectTest { + + private McpClientManager clientManager; + + @BeforeEach + void setUp() { + clientManager = mock(McpClientManager.class); + } + + @Test + @DisplayName("legacy config (raw name): wrapped callback is treated as returnDirect") + void rawNameInConfigStillMatches() { + // Existing application.yml from before the prefix change: + // mateclaw.mcp.return-direct.tools: [query_employee_salary] + McpReturnDirectProperties props = newProps("query_employee_salary"); + + ToolCallback raw = stubCallback("query_employee_salary"); + ToolCallback prefixedWrap = new PrefixedNameToolCallback( + McpToolNameResolver.prefixedName(42L, "query_employee_salary"), raw); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(prefixedWrap)); + when(clientManager.getActiveCount()).thenReturn(1); + + McpToolCallbackProvider provider = new McpToolCallbackProvider(clientManager, props); + ToolCallback[] out = provider.getToolCallbacks(); + + assertEquals(1, out.length); + assertTrue(out[0] instanceof ReturnDirectMcpToolCallback, + "expected legacy raw-name match to wrap as ReturnDirectMcpToolCallback, got " + out[0].getClass()); + } + + @Test + @DisplayName("new config (prefixed name): wrapped callback is treated as returnDirect") + void prefixedNameInConfigMatches() { + String prefixed = McpToolNameResolver.prefixedName(42L, "query_employee_salary"); + McpReturnDirectProperties props = newProps(prefixed); + + ToolCallback raw = stubCallback("query_employee_salary"); + ToolCallback prefixedWrap = new PrefixedNameToolCallback(prefixed, raw); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(prefixedWrap)); + + McpToolCallbackProvider provider = new McpToolCallbackProvider(clientManager, props); + ToolCallback[] out = provider.getToolCallbacks(); + + assertEquals(1, out.length); + assertTrue(out[0] instanceof ReturnDirectMcpToolCallback); + } + + @Test + @DisplayName("non-matching name: callback is passed through, NOT wrapped") + void nonMatchingNameLeftAlone() { + McpReturnDirectProperties props = newProps("something_else"); + + ToolCallback raw = stubCallback("query_employee_salary"); + ToolCallback prefixedWrap = new PrefixedNameToolCallback( + McpToolNameResolver.prefixedName(42L, "query_employee_salary"), raw); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(prefixedWrap)); + + McpToolCallbackProvider provider = new McpToolCallbackProvider(clientManager, props); + ToolCallback[] out = provider.getToolCallbacks(); + + assertEquals(1, out.length); + assertFalse(out[0] instanceof ReturnDirectMcpToolCallback); + assertEquals(prefixedWrap, out[0]); + } + + @Test + @DisplayName("two servers expose the same raw name; raw config matches BOTH") + void rawNameInConfigMatchesAcrossServers() { + // Documented behavior of the legacy form: a raw token isolates + // every server that exposes that tool name. This is intentional — + // operators wanting per-server scoping switch to the prefixed form. + McpReturnDirectProperties props = newProps("read_medical_record"); + + ToolCallback rawA = stubCallback("read_medical_record"); + ToolCallback wrapA = new PrefixedNameToolCallback( + McpToolNameResolver.prefixedName(42L, "read_medical_record"), rawA); + ToolCallback rawB = stubCallback("read_medical_record"); + ToolCallback wrapB = new PrefixedNameToolCallback( + McpToolNameResolver.prefixedName(43L, "read_medical_record"), rawB); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(wrapA, wrapB)); + + ToolCallback[] out = new McpToolCallbackProvider(clientManager, props).getToolCallbacks(); + + assertEquals(2, out.length); + assertTrue(out[0] instanceof ReturnDirectMcpToolCallback); + assertTrue(out[1] instanceof ReturnDirectMcpToolCallback); + } + + @Test + @DisplayName("prefixed config of one server: only THAT server's callback wraps") + void prefixedNameOnlyMatchesScopedServer() { + String prefixedA = McpToolNameResolver.prefixedName(42L, "read_medical_record"); + McpReturnDirectProperties props = newProps(prefixedA); + + ToolCallback wrapA = new PrefixedNameToolCallback(prefixedA, stubCallback("read_medical_record")); + ToolCallback wrapB = new PrefixedNameToolCallback( + McpToolNameResolver.prefixedName(43L, "read_medical_record"), + stubCallback("read_medical_record")); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(wrapA, wrapB)); + + ToolCallback[] out = new McpToolCallbackProvider(clientManager, props).getToolCallbacks(); + + assertEquals(2, out.length); + assertTrue(out[0] instanceof ReturnDirectMcpToolCallback, + "scoped prefix should match server 42's callback"); + assertFalse(out[1] instanceof ReturnDirectMcpToolCallback, + "scoped prefix should NOT match server 43's callback"); + } + + @Test + @DisplayName("non-wrapped callback (no PrefixedNameToolCallback): match falls back to its single name") + void nonWrappedCallbackWithMatchingName() { + // Defensive: a callback might still flow through that isn't our + // wrapper (e.g. a unit-test path). The match must work on the + // callback's reported name without trying to extract a 'raw' that + // doesn't exist. + McpReturnDirectProperties props = newProps("plain_name"); + + ToolCallback plain = stubCallback("plain_name"); + when(clientManager.getAllToolCallbacks()).thenReturn(List.of(plain)); + + ToolCallback[] out = new McpToolCallbackProvider(clientManager, props).getToolCallbacks(); + assertEquals(1, out.length); + assertTrue(out[0] instanceof ReturnDirectMcpToolCallback); + } + + private static McpReturnDirectProperties newProps(String... toolNames) { + McpReturnDirectProperties p = new McpReturnDirectProperties(); + p.setTools(Set.of(toolNames)); + return p; + } + + private static ToolCallback stubCallback(String name) { + ToolDefinition def = DefaultToolDefinition.builder() + .name(name) + .description("") + .inputSchema("{}") + .build(); + return new ToolCallback() { + @Override public ToolDefinition getToolDefinition() { return def; } + @Override public ToolMetadata getToolMetadata() { return ToolCallback.super.getToolMetadata(); } + @Override public String call(String toolInput) { return ""; } + @Override public String call(String toolInput, ToolContext ctx) { return ""; } + }; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolNameResolverTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolNameResolverTest.java new file mode 100644 index 00000000..ae13341f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/McpToolNameResolverTest.java @@ -0,0 +1,122 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class McpToolNameResolverTest { + + @Test + @DisplayName("prefixedName follows mcp___ shape") + void prefixedNameShape() { + String name = McpToolNameResolver.prefixedName(42L, "create_issue"); + assertTrue(name.startsWith("mcp_42_create_issue_"), "got: " + name); + // hash6 occupies the last 6 chars; everything before the final '_' is + // the slug (not the raw name) prefixed by serverId. + String hash = name.substring(name.length() - 6); + assertEquals(6, hash.length()); + } + + @Test + @DisplayName("same raw name produces same prefixed name on the same server") + void deterministicForSameInput() { + String a = McpToolNameResolver.prefixedName(42L, "search"); + String b = McpToolNameResolver.prefixedName(42L, "search"); + assertEquals(a, b); + } + + @Test + @DisplayName("same raw name on different servers produces different prefixed names") + void differentServerYieldsDifferentName() { + String a = McpToolNameResolver.prefixedName(42L, "search"); + String b = McpToolNameResolver.prefixedName(43L, "search"); + assertNotEquals(a, b); + assertTrue(a.startsWith("mcp_42_")); + assertTrue(b.startsWith("mcp_43_")); + } + + @Test + @DisplayName("raw names that collapse to the same slug differ in the hash component") + void slugCollisionsAreDistinguishedByHash() { + // Without the hash, "a b" / "a_b" / "a/b" all slug to "a_b" and the + // single-string binding model would silently collide. + String a = McpToolNameResolver.prefixedName(42L, "a b"); + String b = McpToolNameResolver.prefixedName(42L, "a_b"); + String c = McpToolNameResolver.prefixedName(42L, "a/b"); + assertNotEquals(a, b); + assertNotEquals(b, c); + assertNotEquals(a, c); + assertTrue(a.startsWith("mcp_42_a_b_")); + assertTrue(b.startsWith("mcp_42_a_b_")); + assertTrue(c.startsWith("mcp_42_a_b_")); + } + + @Test + @DisplayName("non-ASCII raw names get a stable 'tool' slug placeholder") + void nonAsciiRawNameUsesPlaceholderSlug() { + String name = McpToolNameResolver.prefixedName(42L, "查询订单"); + assertTrue(name.startsWith("mcp_42_tool_"), "got: " + name); + } + + @Test + @DisplayName("slug is truncated to 20 chars even for very long raw names") + void slugTruncatedAtTwentyChars() { + String longRaw = "abcdefghijklmnopqrstuvwxyz0123456789"; // 36 chars + String name = McpToolNameResolver.prefixedName(42L, longRaw); + // shape: mcp_42__ + // verify slug portion is exactly 20 chars + int firstSep = name.indexOf('_', "mcp_".length()); + int lastSep = name.lastIndexOf('_'); + String slug = name.substring(firstSep + 1, lastSep); + assertEquals(20, slug.length()); + } + + @Test + @DisplayName("blank raw name throws IllegalArgumentException") + void blankRawNameRejected() { + assertThrows(IllegalArgumentException.class, + () -> McpToolNameResolver.prefixedName(42L, "")); + assertThrows(IllegalArgumentException.class, + () -> McpToolNameResolver.prefixedName(42L, null)); + } + + @Test + @DisplayName("parse round-trips serverId, slug, and hash6") + void parseRoundTrip() { + String name = McpToolNameResolver.prefixedName(42L, "create_issue"); + McpToolNameResolver.ParsedRef ref = McpToolNameResolver.parse(name); + assertNotNull(ref); + assertEquals(42L, ref.serverId()); + assertEquals("create_issue", ref.slug()); + assertEquals(6, ref.hash6().length()); + // hash6 of the same raw name reproduces — the cache reverse-lookup + // path depends on this property. + assertEquals(McpToolNameResolver.hash6("create_issue"), ref.hash6()); + } + + @Test + @DisplayName("parse returns null for non-MCP names") + void parseRejectsNonMcp() { + assertNull(McpToolNameResolver.parse(null)); + assertNull(McpToolNameResolver.parse("")); + assertNull(McpToolNameResolver.parse("web_search")); // builtin + assertNull(McpToolNameResolver.parse("mcp_")); // missing parts + assertNull(McpToolNameResolver.parse("mcp_abc_x_yz")); // serverId not numeric + assertNull(McpToolNameResolver.parse("mcp_42_search_xyz")); // hash too short + } + + @Test + @DisplayName("isMcpPrefixedName is a cheap routing check") + void isMcpPrefixedName() { + assertTrue(McpToolNameResolver.isMcpPrefixedName("mcp_42_search_aaaaaa")); + assertFalse(McpToolNameResolver.isMcpPrefixedName(null)); + assertFalse(McpToolNameResolver.isMcpPrefixedName("web_search")); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallbackTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallbackTest.java new file mode 100644 index 00000000..7eec03c3 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/PrefixedNameToolCallbackTest.java @@ -0,0 +1,125 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.DefaultToolDefinition; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.DefaultToolMetadata; +import org.springframework.ai.tool.metadata.ToolMetadata; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class PrefixedNameToolCallbackTest { + + @Test + @DisplayName("getToolDefinition().name() returns the prefixed name; description and schema pass through") + void nameOverriddenOthersPassThrough() { + ToolCallback inner = new RecordingCallback("search", "Search the web", "{\"type\":\"object\"}"); + PrefixedNameToolCallback wrapped = new PrefixedNameToolCallback("mcp_42_search_aaaaaa", inner); + + ToolDefinition td = wrapped.getToolDefinition(); + assertEquals("mcp_42_search_aaaaaa", td.name()); + assertEquals("Search the web", td.description()); + assertEquals("{\"type\":\"object\"}", td.inputSchema()); + } + + @Test + @DisplayName("call(toolInput) delegates to the inner callback unchanged") + void callDelegates() { + RecordingCallback inner = new RecordingCallback("search", "", "{}"); + PrefixedNameToolCallback wrapped = new PrefixedNameToolCallback("mcp_42_search_aaaaaa", inner); + + String result = wrapped.call("{\"q\":\"hello\"}"); + assertEquals("called:{\"q\":\"hello\"}", result); + assertEquals("{\"q\":\"hello\"}", inner.lastInput); + } + + @Test + @DisplayName("call(toolInput, ToolContext) delegates to the inner callback unchanged") + void callWithContextDelegates() { + RecordingCallback inner = new RecordingCallback("search", "", "{}"); + PrefixedNameToolCallback wrapped = new PrefixedNameToolCallback("mcp_42_search_aaaaaa", inner); + ToolContext ctx = new ToolContext(java.util.Map.of("k", "v")); + + String result = wrapped.call("{}", ctx); + assertEquals("called-with-ctx:{}", result); + assertSame(ctx, inner.lastContext); + } + + @Test + @DisplayName("getToolMetadata passes through the inner metadata") + void metadataPassesThrough() { + ToolMetadata meta = DefaultToolMetadata.builder().returnDirect(true).build(); + ToolCallback inner = new RecordingCallback("search", "", "{}", meta); + PrefixedNameToolCallback wrapped = new PrefixedNameToolCallback("mcp_42_search_aaaaaa", inner); + assertSame(meta, wrapped.getToolMetadata()); + } + + @Test + @DisplayName("getDelegate exposes the wrapped callback for downstream introspection") + void getDelegate() { + ToolCallback inner = new RecordingCallback("search", "", "{}"); + PrefixedNameToolCallback wrapped = new PrefixedNameToolCallback("mcp_42_search_aaaaaa", inner); + assertSame(inner, wrapped.getDelegate()); + } + + @Test + @DisplayName("blank prefixed name or null delegate is rejected") + void rejectsBadInputs() { + ToolCallback inner = new RecordingCallback("x", "", "{}"); + assertThrows(IllegalArgumentException.class, + () -> new PrefixedNameToolCallback(null, inner)); + assertThrows(IllegalArgumentException.class, + () -> new PrefixedNameToolCallback("", inner)); + assertThrows(IllegalArgumentException.class, + () -> new PrefixedNameToolCallback("mcp_x", null)); + } + + /** Simple ToolCallback fake to avoid pulling Mockito for these checks. */ + static final class RecordingCallback implements ToolCallback { + private final ToolDefinition definition; + private final ToolMetadata metadata; + String lastInput; + ToolContext lastContext; + + RecordingCallback(String name, String description, String inputSchema) { + this(name, description, inputSchema, null); + } + + RecordingCallback(String name, String description, String inputSchema, ToolMetadata metadata) { + this.definition = DefaultToolDefinition.builder() + .name(name) + .description(description) + .inputSchema(inputSchema) + .build(); + this.metadata = metadata; + } + + @Override + public ToolDefinition getToolDefinition() { + return definition; + } + + @Override + public ToolMetadata getToolMetadata() { + return metadata != null ? metadata : ToolCallback.super.getToolMetadata(); + } + + @Override + public String call(String toolInput) { + this.lastInput = toolInput; + return "called:" + toolInput; + } + + @Override + public String call(String toolInput, ToolContext toolContext) { + this.lastInput = toolInput; + this.lastContext = toolContext; + return "called-with-ctx:" + toolInput; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/ReturnDirectMcpToolCallbackTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/ReturnDirectMcpToolCallbackTest.java new file mode 100644 index 00000000..b9c1c733 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/runtime/ReturnDirectMcpToolCallbackTest.java @@ -0,0 +1,92 @@ +package vip.mate.tool.mcp.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.ai.chat.model.ToolContext; +import org.springframework.ai.tool.ToolCallback; +import org.springframework.ai.tool.definition.ToolDefinition; +import org.springframework.ai.tool.metadata.ToolMetadata; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-052 PR-4: verify the MCP returnDirect decorator only changes + * {@link ToolMetadata#returnDirect()} and delegates everything else. + */ +class ReturnDirectMcpToolCallbackTest { + + @Test + @DisplayName("decorator reports returnDirect=true while delegate stays false") + void overridesMetadataOnly() { + ToolCallback delegate = new RecordingDelegate(); + assertFalse(delegate.getToolMetadata().returnDirect(), + "sanity: bare delegate is not returnDirect"); + + ToolCallback wrapped = new ReturnDirectMcpToolCallback(delegate); + assertTrue(wrapped.getToolMetadata().returnDirect(), + "decorator must flip returnDirect to true"); + assertEquals(delegate.getToolDefinition().name(), wrapped.getToolDefinition().name(), + "tool definition name must be delegated unchanged"); + assertEquals(delegate.getToolDefinition().description(), wrapped.getToolDefinition().description(), + "tool definition description must be delegated unchanged"); + } + + @Test + @DisplayName("call(args) and call(args, ctx) both delegate") + void delegatesInvocations() { + RecordingDelegate delegate = new RecordingDelegate(); + ToolCallback wrapped = new ReturnDirectMcpToolCallback(delegate); + + assertEquals("called: x", wrapped.call("x")); + assertEquals(1, delegate.callCount); + + assertEquals("called-ctx: y", wrapped.call("y", null)); + assertEquals(1, delegate.callCtxCount); + } + + @Test + @DisplayName("null delegate is rejected at construction time") + void nullDelegateRejected() { + assertThrows(IllegalArgumentException.class, + () -> new ReturnDirectMcpToolCallback(null)); + } + + @Test + @DisplayName("McpReturnDirectProperties.isReturnDirect matches configured tool names only") + void propertiesMatchByName() { + McpReturnDirectProperties props = new McpReturnDirectProperties(); + props.setTools(java.util.Set.of("query_employee_salary", "read_medical_record")); + + assertTrue(props.isReturnDirect("query_employee_salary")); + assertTrue(props.isReturnDirect("read_medical_record")); + assertFalse(props.isReturnDirect("get_weather")); + assertFalse(props.isReturnDirect(null)); + assertFalse(props.isReturnDirect("")); + } + + private static final class RecordingDelegate implements ToolCallback { + int callCount; + int callCtxCount; + + @Override + public ToolDefinition getToolDefinition() { + return ToolDefinition.builder() + .name("recording_tool") + .description("test") + .inputSchema("{}") + .build(); + } + + @Override + public String call(String arguments) { + callCount++; + return "called: " + arguments; + } + + @Override + public String call(String arguments, ToolContext toolContext) { + callCtxCount++; + return "called-ctx: " + arguments; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/mcp/service/McpServerServiceListToolsTest.java b/mateclaw-server/src/test/java/vip/mate/tool/mcp/service/McpServerServiceListToolsTest.java new file mode 100644 index 00000000..4a46e86e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/mcp/service/McpServerServiceListToolsTest.java @@ -0,0 +1,138 @@ +package vip.mate.tool.mcp.service; + +import io.modelcontextprotocol.spec.McpSchema; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import vip.mate.exception.MateClawException; +import vip.mate.tool.mcp.model.McpServerEntity; +import vip.mate.tool.mcp.model.McpToolDescriptor; +import vip.mate.tool.mcp.repository.McpServerMapper; +import vip.mate.tool.mcp.runtime.McpClientManager; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Covers {@link McpServerService#listToolsByServer(Long)}, + * the new endpoint that lets the admin UI see what tools an MCP server + * has actually surfaced to the runtime. + * + *

Critical contracts under test: + *

    + *
  • Existence check must happen first — a deleted server id must + * surface as a {@code MateClawException("err.mcp.not_found")} which + * the global handler maps to HTTP 200 + {@code code=500} (project's + * "HTTP 200 + biz code" convention; see McpServerController javadoc). + * The point is that "no tools" must not be confused with "no such server".
  • + *
  • An existing-but-empty cache returns {@code []}, not an error + * (server may be disconnected, in error state, or simply have no + * tools — UI should render "no tools yet" not an error toast).
  • + *
  • Field mapping from the SDK record to the DTO is verbatim — name, + * description, inputSchema all pass through.
  • + *
+ */ +@ExtendWith(MockitoExtension.class) +class McpServerServiceListToolsTest { + + @Mock + private McpServerMapper mcpServerMapper; + + @Mock + private McpClientManager mcpClientManager; + + @InjectMocks + private McpServerService service; + + private static McpServerEntity server(Long id) { + McpServerEntity e = new McpServerEntity(); + e.setId(id); + e.setName("test-server-" + id); + return e; + } + + @Test + @DisplayName("missing server id throws MateClawException — distinguishes not-found from empty-tools") + void missingServerThrows() { + when(mcpServerMapper.selectById(99L)).thenReturn(null); + + assertThrows(MateClawException.class, + () -> service.listToolsByServer(99L)); + + // Don't even consult the cache for a non-existent server. + verify(mcpClientManager, never()).getServerTools(99L); + } + + @Test + @DisplayName("empty tools cache returns [] — not an error") + void emptyCacheReturnsEmptyList() { + when(mcpServerMapper.selectById(7L)).thenReturn(server(7L)); + when(mcpClientManager.getServerTools(7L)).thenReturn(List.of()); + + List result = service.listToolsByServer(7L); + + assertTrue(result.isEmpty()); + } + + /** Tool record signature (mcp-core 1.1.0): name, title, description, + * inputSchema, outputSchema (Map), annotations, meta (Map). Tests pass + * null for the fields they don't exercise — the SDK accepts that. */ + private static McpSchema.Tool tool(String name, String description, McpSchema.JsonSchema inputSchema) { + return new McpSchema.Tool(name, null, description, inputSchema, null, null, null); + } + + /** Convenience for an "object" JSON schema with the given properties map. */ + private static McpSchema.JsonSchema objectSchema(Map properties) { + return new McpSchema.JsonSchema("object", properties, null, null, null, null); + } + + @Test + @DisplayName("populated cache maps every Tool record verbatim into the DTO") + void populatedCacheMappedVerbatim() { + when(mcpServerMapper.selectById(7L)).thenReturn(server(7L)); + McpSchema.JsonSchema echoSchema = objectSchema(Map.of( + "text", Map.of("type", "string"))); + McpSchema.JsonSchema sumSchema = objectSchema(Map.of( + "a", Map.of("type", "number"), + "b", Map.of("type", "number"))); + when(mcpClientManager.getServerTools(7L)).thenReturn(List.of( + tool("echo", "Echoes the input back", echoSchema), + tool("sum", "Adds two numbers", sumSchema) + )); + + List result = service.listToolsByServer(7L); + + assertEquals(2, result.size()); + assertEquals("echo", result.get(0).name()); + assertEquals("Echoes the input back", result.get(0).description()); + assertEquals(echoSchema, result.get(0).inputSchema()); + assertEquals("sum", result.get(1).name()); + assertEquals(sumSchema, result.get(1).inputSchema()); + } + + @Test + @DisplayName("tools with null description still flow through the mapping") + void nullDescriptionPreserved() { + when(mcpServerMapper.selectById(7L)).thenReturn(server(7L)); + when(mcpClientManager.getServerTools(7L)).thenReturn(List.of( + tool("ping", null, objectSchema(Map.of())) + )); + + List result = service.listToolsByServer(7L); + + assertEquals("ping", result.get(0).name()); + // null description survives; DTO @JsonInclude(NON_NULL) drops it from + // the wire payload but the Java value is preserved through the mapping. + assertTrue(result.get(0).description() == null); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/service/AvailableToolServiceTest.java b/mateclaw-server/src/test/java/vip/mate/tool/service/AvailableToolServiceTest.java new file mode 100644 index 00000000..db891774 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/service/AvailableToolServiceTest.java @@ -0,0 +1,224 @@ +package vip.mate.tool.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.tool.mcp.model.McpServerEntity; +import vip.mate.tool.mcp.runtime.McpToolNameResolver; +import vip.mate.tool.mcp.service.McpServerService; +import vip.mate.tool.model.AvailableToolDTO; +import vip.mate.tool.model.ToolEntity; +// imports above intentionally minimal; java.util.* used inline where needed + +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class AvailableToolServiceTest { + + private ToolService toolService; + private McpServerService mcpServerService; + private AvailableToolService service; + + @BeforeEach + void setUp() { + toolService = mock(ToolService.class); + mcpServerService = mock(McpServerService.class); + service = new AvailableToolService(toolService, mcpServerService); + when(toolService.listEnabledTools()).thenReturn(List.of()); + when(mcpServerService.listEnabled()).thenReturn(List.of()); + } + + @Test + @DisplayName("listAvailable mixes builtin and MCP tools") + void mixesBuiltinAndMcp() { + when(toolService.listEnabledTools()).thenReturn(List.of(builtin("web_search", "Search the web"))); + when(mcpServerService.listEnabled()).thenReturn(List.of(connectedServer(42L, "github", "create_issue"))); + + List out = service.listAvailable(); + + assertEquals(2, out.size()); + Set sources = out.stream().map(AvailableToolDTO::getSource).collect(Collectors.toSet()); + assertEquals(Set.of("builtin", "mcp"), sources); + } + + @Test + @DisplayName("MCP entry name equals McpToolNameResolver.prefixedName(serverId, raw)") + void mcpNameMatchesResolver() { + when(mcpServerService.listEnabled()).thenReturn(List.of(connectedServer(42L, "github", "create_issue"))); + + AvailableToolDTO mcp = service.listAvailable().get(0); + + assertEquals(McpToolNameResolver.prefixedName(42L, "create_issue"), mcp.getName()); + assertEquals("create_issue", mcp.getRawName()); + assertEquals("mcp:42", mcp.getGroupId()); + assertEquals("MCP · github", mcp.getGroup()); + assertTrue(mcp.isAvailable()); + assertFalse(mcp.isStale()); + } + + @Test + @DisplayName("disconnected MCP server marks tools stale but keeps them in the response") + void staleFlagSetWhenDisconnected() { + McpServerEntity disconnected = connectedServer(42L, "github", "create_issue"); + disconnected.setLastStatus("disconnected"); + when(mcpServerService.listEnabled()).thenReturn(List.of(disconnected)); + + List out = service.listAvailable(); + + assertEquals(1, out.size()); + assertTrue(out.get(0).isStale()); + // stale entries are still bindable from the picker's perspective — + // runtime will silently filter them when the callback isn't there. + assertTrue(out.get(0).isAvailable()); + } + + @Test + @DisplayName("two MCP servers exposing the same raw name produce distinct prefixed names, both bindable") + void crossServerSameRawIsNotACollision() { + when(mcpServerService.listEnabled()).thenReturn(List.of( + connectedServer(42L, "github", "search"), + connectedServer(43L, "filesystem", "search"))); + + List out = service.listAvailable(); + + assertEquals(2, out.size()); + Set names = out.stream().map(AvailableToolDTO::getName).collect(Collectors.toSet()); + assertTrue(names.contains(McpToolNameResolver.prefixedName(42L, "search"))); + assertTrue(names.contains(McpToolNameResolver.prefixedName(43L, "search"))); + assertEquals(2, names.size()); + for (AvailableToolDTO dto : out) { + assertTrue(dto.isAvailable(), "expected bindable, got: " + dto); + } + } + + @Test + @DisplayName("duplicate raw names within a server flag the second entry as unavailable") + void duplicateRawNameSecondMarkedUnavailable() { + // Two cached entries with the same raw name — pretend the upstream + // surfaces a duplicate (defensive): the picker should disable the + // second occurrence so the user can't bind a name that resolves to + // nothing at runtime. + McpServerEntity server = serverWithCacheJson(42L, "github", + "[{\"name\":\"search\",\"description\":\"\",\"inputSchema\":{}}," + + "{\"name\":\"search\",\"description\":\"\",\"inputSchema\":{}}]"); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + List out = service.listAvailable(); + + assertEquals(2, out.size()); + assertTrue(out.get(0).isAvailable()); + assertFalse(out.get(1).isAvailable()); + assertEquals("DUPLICATE_RAW_NAME", out.get(1).getUnavailableReason()); + // Two rows share the same prefixed `name`; rowId must differ so + // the Vue picker doesn't reuse DOM state across them. + assertNotEquals(out.get(0).getRowId(), out.get(1).getRowId(), + "rowId must distinguish duplicate-raw entries"); + } + + @Test + @DisplayName("hash-colliding raw pair: second entry is unavailable with HASH_COLLISION reason") + void hashCollisionSecondMarkedUnavailable() { + // Pair pre-mined by birthday search — same prefixed name, different raw. + String[] pair = findHashCollidingPair(42L); + String cacheJson = "[" + + "{\"name\":\"" + pair[0] + "\",\"description\":\"\",\"inputSchema\":{}}," + + "{\"name\":\"" + pair[1] + "\",\"description\":\"\",\"inputSchema\":{}}" + + "]"; + McpServerEntity server = serverWithCacheJson(42L, "github", cacheJson); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + List out = service.listAvailable(); + + assertEquals(2, out.size()); + // Both rows carry the same prefixed name (that's the whole point of + // a hash collision) but only the first is bindable. + assertEquals(out.get(0).getName(), out.get(1).getName()); + assertTrue(out.get(0).isAvailable()); + assertFalse(out.get(1).isAvailable()); + assertNotNull(out.get(1).getUnavailableReason()); + assertTrue(out.get(1).getUnavailableReason().startsWith("HASH_COLLISION:"), + "got reason: " + out.get(1).getUnavailableReason()); + // rowId must differ even though name is identical. + assertNotEquals(out.get(0).getRowId(), out.get(1).getRowId()); + } + + /** Birthday-search a colliding raw-name pair (same slug + same hash6). */ + private static String[] findHashCollidingPair(long serverId) { + String anchor = "xxxxxxxxxxxxxxxxxxxx"; // 20-char slug filler + java.util.Map seen = new java.util.HashMap<>(); + for (int i = 0; i < 1_000_000; i++) { + String raw = anchor + i; + String hash = McpToolNameResolver.hash6(raw); + String prior = seen.put(hash, raw); + if (prior != null) { + // sanity: confirm the FULL prefixed name is identical + if (McpToolNameResolver.prefixedName(serverId, prior) + .equals(McpToolNameResolver.prefixedName(serverId, raw))) { + return new String[]{prior, raw}; + } + } + } + throw new IllegalStateException("Could not find a colliding pair in 1M tries"); + } + + @Test + @DisplayName("MCP server with empty cache contributes nothing to the picker") + void emptyCacheContributesNothing() { + McpServerEntity server = connectedServer(42L, "github"); + server.setToolsCacheJson("[]"); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + List out = service.listAvailable(); + assertEquals(0, out.size()); + } + + @Test + @DisplayName("malformed cache JSON does not 500 the picker; the server contributes nothing") + void malformedCacheGracefullySkipped() { + McpServerEntity server = connectedServer(42L, "github"); + server.setToolsCacheJson("{not valid json}"); + when(mcpServerService.listEnabled()).thenReturn(List.of(server)); + + List out = service.listAvailable(); + assertNotNull(out); + assertEquals(0, out.size()); + } + + private static ToolEntity builtin(String name, String description) { + ToolEntity t = new ToolEntity(); + t.setName(name); + t.setDescription(description); + t.setEnabled(true); + return t; + } + + private static McpServerEntity connectedServer(long id, String name, String... rawTools) { + StringBuilder sb = new StringBuilder("["); + for (int i = 0; i < rawTools.length; i++) { + if (i > 0) sb.append(","); + sb.append("{\"name\":\"").append(rawTools[i]) + .append("\",\"description\":\"\",\"inputSchema\":{}}"); + } + sb.append("]"); + return serverWithCacheJson(id, name, sb.toString()); + } + + private static McpServerEntity serverWithCacheJson(long id, String name, String cacheJson) { + McpServerEntity s = new McpServerEntity(); + s.setId(id); + s.setName(name); + s.setEnabled(true); + s.setLastStatus("connected"); + s.setToolsCacheJson(cacheJson); + return s; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/video/provider/DashScopeVideoProviderRoutingTest.java b/mateclaw-server/src/test/java/vip/mate/tool/video/provider/DashScopeVideoProviderRoutingTest.java new file mode 100644 index 00000000..722bbcf7 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/video/provider/DashScopeVideoProviderRoutingTest.java @@ -0,0 +1,134 @@ +package vip.mate.tool.video.provider; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.tool.video.VideoCapability; +import vip.mate.tool.video.VideoGenerationRequest; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Pinpoints the routing decisions in {@link DashScopeVideoProvider}: the + * model id picks both the endpoint family and the JSON body shape (legacy + * {@code img_url} flat input vs unified {@code media[]} array). HTTP + * submission is not exercised here. + */ +@Tag("media-gen") +class DashScopeVideoProviderRoutingTest { + + private final DashScopeVideoProvider provider = + new DashScopeVideoProvider(null, new ObjectMapper()); + + @Test + @DisplayName("legacy text-to-video model: LEGACY body shape, video-generation/generation endpoint") + void legacyT2v_routesToLegacyShape() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("a cat playing piano") + .model("wan2.5-t2v-turbo") + .mode(VideoCapability.GENERATE) + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + assertEquals(DashScopeVideoProvider.BodyShape.LEGACY, spec.bodyShape()); + assertTrue(spec.endpoint().endsWith("/services/aigc/video-generation/generation")); + } + + @Test + @DisplayName("unified text-to-video model: UNIFIED body shape, video-synthesis endpoint") + void unifiedT2v_routesToUnifiedShape() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("a sunset over the sea") + .model("wan2.7-t2v-2026-04-25") + .mode(VideoCapability.GENERATE) + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + assertEquals(DashScopeVideoProvider.BodyShape.UNIFIED, spec.bodyShape()); + assertTrue(spec.endpoint().endsWith("/services/aigc/video-generation/video-synthesis")); + } + + @Test + @DisplayName("happyhorse t2v: routed to UNIFIED endpoint family") + void happyhorse_routesToUnifiedShape() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("a horse running on a beach") + .model("happyhorse-1.0-t2v") + .mode(VideoCapability.GENERATE) + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + assertEquals(DashScopeVideoProvider.BodyShape.UNIFIED, spec.bodyShape()); + assertTrue(spec.endpoint().endsWith("/services/aigc/video-generation/video-synthesis")); + } + + @Test + @DisplayName("legacy body: input.img_url is set when image url present, parameters.size keyed") + void legacyBody_includesImgUrlAndSizeKey() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("walking forward") + .model("wan2.5-i2v-turbo") + .mode(VideoCapability.IMAGE_TO_VIDEO) + .imageUrl("https://cdn.example.com/cover.png") + .aspectRatio("16:9") + .durationSeconds(5) + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + JsonNode body = provider.buildRequestBody(req, spec); + + assertEquals("wan2.5-i2v-turbo", body.path("model").asText()); + assertEquals("https://cdn.example.com/cover.png", body.path("input").path("img_url").asText()); + assertFalse(body.path("input").has("media"), + "legacy shape must not include the unified media[] array"); + // Size uses the legacy '*' separator + assertEquals("1280*720", body.path("parameters").path("size").asText()); + assertEquals("5", body.path("parameters").path("duration").asText()); + } + + @Test + @DisplayName("unified body: input.media[] is set with first_frame; parameters.resolution + ratio keyed") + void unifiedBody_usesMediaArrayAndResolution() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("the camera pans right") + .model("wan2.7-i2v-2026-04-25") + .mode(VideoCapability.IMAGE_TO_VIDEO) + .imageUrl("https://cdn.example.com/cover.png") + .aspectRatio("16:9") + .durationSeconds(8) + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + JsonNode body = provider.buildRequestBody(req, spec); + + assertEquals("wan2.7-i2v-2026-04-25", body.path("model").asText()); + // Unified shape uses media[] not img_url + assertFalse(body.path("input").has("img_url")); + JsonNode media = body.path("input").path("media"); + assertTrue(media.isArray() && media.size() == 1); + assertEquals("first_frame", media.get(0).path("type").asText()); + assertEquals("https://cdn.example.com/cover.png", media.get(0).path("url").asText()); + + // Size lives in parameters.resolution + parameters.ratio + assertFalse(body.path("parameters").has("size"), + "unified shape uses resolution/ratio, not the legacy size key"); + assertEquals("720P", body.path("parameters").path("resolution").asText()); + assertEquals("16:9", body.path("parameters").path("ratio").asText()); + // Duration is an integer in unified shape (legacy was a string) + assertEquals(8, body.path("parameters").path("duration").asInt()); + } + + @Test + @DisplayName("unified body: text-only request omits media[] (no first_frame to send)") + void unifiedBody_textOnlyOmitsMedia() { + VideoGenerationRequest req = VideoGenerationRequest.builder() + .prompt("a horse runs") + .model("happyhorse-1.0-t2v") + .mode(VideoCapability.GENERATE) + .aspectRatio("16:9") + .build(); + DashScopeVideoProvider.ModelSpec spec = provider.resolveSpec(req); + JsonNode body = provider.buildRequestBody(req, spec); + assertFalse(body.path("input").has("media"), + "text-to-video must not synthesize an empty first_frame"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/tool/video/provider/MiniMaxVideoProviderTest.java b/mateclaw-server/src/test/java/vip/mate/tool/video/provider/MiniMaxVideoProviderTest.java new file mode 100644 index 00000000..a85e693c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tool/video/provider/MiniMaxVideoProviderTest.java @@ -0,0 +1,100 @@ +package vip.mate.tool.video.provider; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tool.video.VideoCapability; +import vip.mate.tool.video.VideoProviderCapabilities; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Verifies the two pure-logic surfaces of {@link MiniMaxVideoProvider} that + * shouldn't require a live API: region routing and the published model + * catalog. Network paths (submit / poll / file-resolve) need wiremock or + * live fixtures and are out of scope here. + */ +@Tag("media-gen") +class MiniMaxVideoProviderTest { + + private final MiniMaxVideoProvider provider = new MiniMaxVideoProvider(new ObjectMapper()); + + @Test + @DisplayName("resolveBaseUrl: minimaxRegion='cn' (any case) → CN endpoint") + void resolveBaseUrl_cn() { + // CN MiniMax accounts can't reach api.minimax.io — region routing is + // not optional for that user segment. + SystemSettingsDTO cfg = new SystemSettingsDTO(); + cfg.setMinimaxRegion("cn"); + assertEquals(MiniMaxVideoProvider.BASE_URL_CN, MiniMaxVideoProvider.resolveBaseUrl(cfg)); + + cfg.setMinimaxRegion("CN"); + assertEquals(MiniMaxVideoProvider.BASE_URL_CN, MiniMaxVideoProvider.resolveBaseUrl(cfg), + "Region match must be case-insensitive"); + } + + @Test + @DisplayName("resolveBaseUrl: default / explicit global / null → Global endpoint") + void resolveBaseUrl_globalFallbacks() { + // Defaults must NOT silently route to CN — operators outside mainland + // CN must work without setting any region. + SystemSettingsDTO cfg = new SystemSettingsDTO(); + assertEquals(MiniMaxVideoProvider.BASE_URL_GLOBAL, + MiniMaxVideoProvider.resolveBaseUrl(cfg)); + cfg.setMinimaxRegion("global"); + assertEquals(MiniMaxVideoProvider.BASE_URL_GLOBAL, + MiniMaxVideoProvider.resolveBaseUrl(cfg)); + // Defensive: null config → still global, no NPE. + assertEquals(MiniMaxVideoProvider.BASE_URL_GLOBAL, + MiniMaxVideoProvider.resolveBaseUrl(null)); + } + + @Test + @DisplayName("Catalog: 6 models declared (3 T2V + 3 I2V) matching openclaw") + void detailedCapabilities_listsAllModels() { + // Sync with openclaw extensions/minimax/provider-models.ts. Adding a + // model here without verifying MiniMax actually serves it would lead + // to opaque 404s — the public catalog is the source of truth. + VideoProviderCapabilities caps = provider.detailedCapabilities(); + assertTrue(caps.getModels().contains("MiniMax-Hailuo-2.3")); + assertTrue(caps.getModels().contains("MiniMax-Hailuo-2.3-Fast")); + assertTrue(caps.getModels().contains("MiniMax-Hailuo-02"), + "Hailuo-02 was missing before this change — keep pinned to detect regressions"); + assertTrue(caps.getModels().contains("I2V-01-Director")); + assertTrue(caps.getModels().contains("I2V-01-live")); + assertTrue(caps.getModels().contains("I2V-01")); + assertEquals(6, caps.getModels().size(), + "Adding a model? Update this assertion + wire it through openclaw to confirm the API serves it"); + } + + @Test + @DisplayName("Default model stays MiniMax-Hailuo-2.3 (most-used T2V)") + void detailedCapabilities_defaultModel() { + // Default model is what users hit when they don't explicitly pick. + // Changing this changes user behavior — pin it. + assertEquals("MiniMax-Hailuo-2.3", provider.detailedCapabilities().getDefaultModel()); + } + + @Test + @DisplayName("Capabilities: TEXT_TO_VIDEO + IMAGE_TO_VIDEO both declared") + void capabilities_includesBoth() { + // I2V-01-* models live in the catalog but the provider also has to + // advertise the capability flag, otherwise the dispatcher won't route + // image-input requests here. + var caps = provider.capabilities(); + assertTrue(caps.contains(VideoCapability.GENERATE)); + assertTrue(caps.contains(VideoCapability.IMAGE_TO_VIDEO)); + } + + @Test + @DisplayName("Host constants match MiniMax's documented endpoints") + void hostsAreCanonical() { + // Pin string values so a typo (api.minimax.com vs api.minimaxi.com) + // is caught at test time, not via opaque DNS errors in production. + assertEquals("https://api.minimax.io", MiniMaxVideoProvider.BASE_URL_GLOBAL); + assertEquals("https://api.minimaxi.com", MiniMaxVideoProvider.BASE_URL_CN); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/AgentLifecycleTriggerTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/AgentLifecycleTriggerTest.java new file mode 100644 index 00000000..7f99a098 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/AgentLifecycleTriggerTest.java @@ -0,0 +1,120 @@ +package vip.mate.trigger; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.agent.event.AgentLifecycleEvent; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.service.TriggerService; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.runtime.StubAgentInvoker; +import vip.mate.workflow.runtime.StubAgentInvokerConfig; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Confirms agent_lifecycle is wired as a real event source. The agent + * module publishes an {@link AgentLifecycleEvent} when an agent is + * spawned / enabled / disabled / terminated, the trigger bridge maps + * it into an agent_lifecycle envelope, and a matching trigger fires + * its target workflow. + * + *

The test publishes the event directly via the publisher rather + * than driving full agent-create CRUD — that's the contract the agent + * module commits to, and skipping the controller keeps the test + * focused on bridge wiring. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:agent_lifecycle_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none", + "mateclaw.workflow.trigger.async-dispatch=false" +}) +@Import({StubAgentInvokerConfig.class, TriggerDispatcherWorkflowTest.StubGraphLoaderConfig.class}) +class AgentLifecycleTriggerTest { + + @Autowired private ApplicationEventPublisher publisher; + @Autowired private TriggerService triggerService; + @Autowired private WorkflowRunMapper runMapper; + @Autowired private TriggerDispatcherWorkflowTest.StubGraphLoader stubGraphLoader; + @Autowired private StubAgentInvoker stubInvoker; + + @Test + @DisplayName("agent_lifecycle trigger fires when the matching phase + agent is published.") + void agentLifecycleRoutesToWorkflow() { + long workspace = 8800L; + long downstream = 8810L; + long agentId = 4242L; + + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok"); + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("on-agent-spawn"); + t.setPatternType("agent_lifecycle"); + t.setPatternJson("{\"agentId\":" + agentId + ",\"phase\":\"spawned\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new AgentLifecycleEvent( + workspace, agentId, "greeter", "spawned", System.currentTimeMillis())); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertEquals(1, runs.size(), + "agent_lifecycle event should have triggered exactly one workflow run"); + assertEquals("succeeded", runs.get(0).getState()); + } + + @Test + @DisplayName("agent_lifecycle trigger keyed on a different phase stays dormant.") + void wrongPhaseDoesNotMisfire() { + long workspace = 8900L; + long downstream = 8910L; + long agentId = 4243L; + + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("only-on-terminate"); + t.setPatternType("agent_lifecycle"); + t.setPatternJson("{\"agentId\":" + agentId + ",\"phase\":\"terminated\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new AgentLifecycleEvent( + workspace, agentId, "greeter", "spawned", System.currentTimeMillis())); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertTrue(runs.isEmpty(), + "phase mismatch should leave the trigger dormant"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/ChannelMessageTriggerTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/ChannelMessageTriggerTest.java new file mode 100644 index 00000000..efe1964d --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/ChannelMessageTriggerTest.java @@ -0,0 +1,186 @@ +package vip.mate.trigger; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.channel.event.ChannelMessageReceivedEvent; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.service.TriggerService; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.runtime.StubAgentInvoker; +import vip.mate.workflow.runtime.StubAgentInvokerConfig; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Confirms channel_message + content_match are wired as real event + * sources: when the channel module publishes a + * {@link ChannelMessageReceivedEvent}, the trigger bridge forwards it + * into the ingest pipeline and a matching trigger fires its target + * workflow. + * + *

The test publishes the event directly via + * {@link ApplicationEventPublisher} rather than building a full channel + * adapter — that's the contract the channel router commits to, and + * skipping the adapter keeps the test focused on the bridge wiring. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:channel_trigger_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none", + "mateclaw.workflow.trigger.async-dispatch=false" +}) +@Import({StubAgentInvokerConfig.class, TriggerDispatcherWorkflowTest.StubGraphLoaderConfig.class}) +class ChannelMessageTriggerTest { + + @Autowired private ApplicationEventPublisher publisher; + @Autowired private TriggerService triggerService; + @Autowired private WorkflowRunMapper runMapper; + @Autowired private TriggerDispatcherWorkflowTest.StubGraphLoader stubGraphLoader; + @Autowired private StubAgentInvoker stubInvoker; + + @Test + @DisplayName("channel_message trigger fires its target workflow on a matching channelType.") + void channelMessageRoutesToWorkflow() { + long workspace = 7700L; + long downstream = 7710L; + + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok"); + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("on-feishu"); + t.setPatternType("channel_message"); + // narrow to a specific channelType — the matcher reads channelType + // out of envelope.data, which the bridge populates from the event. + t.setPatternJson("{\"channelType\":\"feishu\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new ChannelMessageReceivedEvent( + workspace, "feishu", "msg-1", "alice", "Alice", "chat-1", "hello")); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertEquals(1, runs.size(), "channel_message envelope should have triggered exactly one run"); + assertEquals("succeeded", runs.get(0).getState()); + assertTrue(runs.get(0).getTriggeredBy() != null + && runs.get(0).getTriggeredBy().startsWith("trigger:"), + "downstream run should be triggered_by trigger:* — got " + + runs.get(0).getTriggeredBy()); + } + + @Test + @DisplayName("channel_message trigger keyed on a different channelType stays dormant.") + void wrongChannelTypeDoesNotMisfire() { + long workspace = 7800L; + long downstream = 7810L; + + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + stubInvoker.respond("greeter", "ok"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("only-dingtalk"); + t.setPatternType("channel_message"); + t.setPatternJson("{\"channelType\":\"dingtalk\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new ChannelMessageReceivedEvent( + workspace, "feishu", "msg-2", "bob", "Bob", "chat-2", "hello")); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertTrue(runs.isEmpty(), + "channelType mismatch should leave the trigger dormant"); + } + + @Test + @DisplayName("content_match trigger fires when the message body contains the configured substring.") + void contentMatchRoutesToWorkflow() { + long workspace = 7900L; + long downstream = 7910L; + + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok"); + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"chained\"}]}"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("on-order-keyword"); + t.setPatternType("content_match"); + t.setPatternJson("{\"substring\":\"order\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new ChannelMessageReceivedEvent( + workspace, "feishu", "msg-3", "alice", "Alice", "chat-3", "Place an Order, please")); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertEquals(1, runs.size(), + "content_match should fire when the substring is present in the message"); + } + + @Test + @DisplayName("content_match trigger does NOT fire when the substring is missing.") + void contentMatchSkipsWhenSubstringAbsent() { + long workspace = 8000L; + long downstream = 8010L; + + stubGraphLoader.reset(); + stubGraphLoader.bind(downstream, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspace); + t.setName("only-order"); + t.setPatternType("content_match"); + t.setPatternJson("{\"substring\":\"order\"}"); + t.setTargetType("workflow"); + t.setTargetId(downstream); + t.setEnabled(true); + triggerService.create(t); + + publisher.publishEvent(new ChannelMessageReceivedEvent( + workspace, "feishu", "msg-4", "alice", "Alice", "chat-4", "completely unrelated text")); + + List runs = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, downstream)); + assertTrue(runs.isEmpty(), + "missing substring should leave the content_match trigger dormant"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/TriggerDispatcherWorkflowTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerDispatcherWorkflowTest.java new file mode 100644 index 00000000..993c696c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerDispatcherWorkflowTest.java @@ -0,0 +1,192 @@ +package vip.mate.trigger; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.trigger.dispatch.TriggerDispatcher; +import vip.mate.trigger.dispatch.WorkflowGraphLoader; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.repository.TriggerMapper; +import vip.mate.trigger.scheduler.TriggerScheduler; +import vip.mate.trigger.service.TriggerService; +import vip.mate.workflow.compiler.WorkflowParser; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.runtime.WorkflowRunResult; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Drives the trigger dispatch path end-to-end against a stub workflow + * loader and a stub agent invoker: a fired trigger should produce exactly + * one {@code mate_workflow_run} row whose triggered_by column points back + * at the trigger id, and the rendered payload template should land in the + * run's initial inputs. + * + *

Also exercises the lamport-coordination path on the scheduler: a + * fire dispatched with a stale captured version is silently dropped (the + * scheduler self-cancels), no workflow run row appears, and the + * registration is cleared. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:trigger_dispatch_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none", + "mateclaw.workflow.trigger.async-dispatch=false" +}) +@Import({vip.mate.workflow.runtime.StubAgentInvokerConfig.class, + TriggerDispatcherWorkflowTest.StubGraphLoaderConfig.class}) +class TriggerDispatcherWorkflowTest { + + @Autowired private TriggerService triggerService; + @Autowired private TriggerMapper triggerMapper; + @Autowired private TriggerScheduler scheduler; + @Autowired private TriggerDispatcher dispatcher; + @Autowired private WorkflowRunMapper runMapper; + @Autowired private vip.mate.workflow.runtime.StubAgentInvoker stubInvoker; + @Autowired private StubGraphLoader stubGraphLoader; + + @Test + @DisplayName("Dispatching a cron trigger creates a workflow run with payload-rendered inputs.") + void dispatchProducesWorkflowRun() { + stubInvoker.reset(); + stubInvoker.respond("greeter", "hello world"); + stubGraphLoader.reset(); + stubGraphLoader.bind(7000L, 11L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi {{ inputs.who }}\"}]}"); + + TriggerEntity trigger = triggerService.create(cronTrigger( + "hello-cron", "0 0 * * * *", 7000L, + "{\"who\":\"{{ event.who }}\"}")); + + vip.mate.trigger.dispatch.DispatchResult result = dispatcher.dispatch(trigger, + Map.of("who", "alice")); + assertNotNull(result); + assertEquals(vip.mate.trigger.dispatch.DispatchResult.Kind.FIRED, result.kind()); + assertEquals("hi alice", stubInvoker.lastPromptFor("greeter")); + + WorkflowRunEntity runRow = runMapper.selectById(result.runId()); + assertNotNull(runRow); + assertEquals(7000L, runRow.getWorkflowId()); + assertEquals(11L, runRow.getRevisionId()); + assertEquals("trigger:" + trigger.getId(), runRow.getTriggeredBy()); + } + + @Test + @DisplayName("Dispatching a workflow with no published revision skips fire and records nothing.") + void missingRevisionSkipsRun() { + stubGraphLoader.reset(); + stubGraphLoader.bindMissing(8001L); + + TriggerEntity trigger = triggerService.create(cronTrigger( + "ghost", "0 0 * * * *", 8001L, null)); + + vip.mate.trigger.dispatch.DispatchResult result = dispatcher.dispatch(trigger, Map.of()); + assertNotNull(result); + assertEquals(vip.mate.trigger.dispatch.DispatchResult.Kind.SKIPPED, result.kind(), + "missing revision should yield a SKIPPED outcome, not silent null"); + + List runRows = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, 8001L)); + assertTrue(runRows.isEmpty(), "no workflow run row should be inserted"); + } + + @Test + @DisplayName("A fire whose captured pattern_version trails the live row self-cancels.") + void staleCapturedVersionSelfCancels() { + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok"); + stubGraphLoader.reset(); + stubGraphLoader.bind(9000L, 21L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"hi\"}]}"); + + TriggerEntity trigger = triggerService.create(cronTrigger( + "lamport", "0 0 * * * *", 9000L, null)); + long triggerId = trigger.getId(); + assertTrue(scheduler.isRegistered(triggerId)); + + // Bump the row's pattern_version directly so the in-flight scheduled + // task's captured value is now stale. + TriggerEntity row = triggerMapper.selectById(triggerId); + row.setPatternVersion(row.getPatternVersion() + 5); + triggerMapper.updateById(row); + + // Capture the original version 1; live is now 6 → fire should drop. + scheduler.fireForTest(triggerId, 1L); + + // No new run row created. + List runRows = runMapper.selectList( + new LambdaQueryWrapper().eq(WorkflowRunEntity::getWorkflowId, 9000L)); + assertTrue(runRows.isEmpty(), "stale lamport must drop the fire silently"); + // And the registration should be cleared so a peer with the latest version + // can take over. + assertTrue(!scheduler.isRegistered(triggerId), "scheduler should self-cancel stale registration"); + } + + private static TriggerEntity cronTrigger(String name, String cron, long workflowId, String payloadTpl) { + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(99L); + t.setName(name); + t.setPatternType("cron"); + t.setPatternJson("{\"cron\":\"" + cron + "\"}"); + t.setTargetType("workflow"); + t.setTargetId(workflowId); + t.setPayloadTemplate(payloadTpl); + t.setEnabled(true); + return t; + } + + @TestConfiguration + static class StubGraphLoaderConfig { + @Bean + @Primary + StubGraphLoader stubGraphLoader(WorkflowParser parser) { + return new StubGraphLoader(parser); + } + } + + static class StubGraphLoader implements WorkflowGraphLoader { + private final WorkflowParser parser; + private final java.util.Map graphs = new java.util.concurrent.ConcurrentHashMap<>(); + private final java.util.Set missing = java.util.concurrent.ConcurrentHashMap.newKeySet(); + + StubGraphLoader(WorkflowParser parser) { this.parser = parser; } + + void reset() { graphs.clear(); missing.clear(); } + + void bind(long workflowId, long revisionId, String json) { + WorkflowGraph g = parser.parse(json); + graphs.put(workflowId, new Loaded(g, revisionId)); + } + + void bindMissing(long workflowId) { missing.add(workflowId); } + + @Override + public Loaded load(long workflowId) { + if (missing.contains(workflowId)) return Loaded.missing(); + return graphs.getOrDefault(workflowId, Loaded.missing()); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/TriggerEventIngestServiceTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerEventIngestServiceTest.java new file mode 100644 index 00000000..01a652be --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerEventIngestServiceTest.java @@ -0,0 +1,209 @@ +package vip.mate.trigger; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.context.TestConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.trigger.dispatch.WorkflowGraphLoader; +import vip.mate.trigger.ingest.BotSelfFilter; +import vip.mate.trigger.ingest.TriggerEventEnvelope; +import vip.mate.trigger.ingest.TriggerEventIngestService; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.repository.TriggerMapper; +import vip.mate.trigger.service.TriggerService; +import vip.mate.workflow.compiler.WorkflowParser; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.runtime.StubAgentInvoker; +import vip.mate.workflow.runtime.StubAgentInvokerConfig; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Drives the four-stage ingest pipeline end-to-end against H2 with stub + * agent invocation and stub workflow graph loading: the dedup window + * collapses repeated events, the per-trigger sliding rate limit drops + * over-cap events, the bot-self filter shields against echo loops, and + * a clean event produces exactly one workflow run. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:trigger_ingest_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none", + "mateclaw.workflow.trigger.async-dispatch=false" +}) +@Import({StubAgentInvokerConfig.class, + TriggerDispatcherWorkflowTest.StubGraphLoaderConfig.class, + TriggerEventIngestServiceTest.SwitchableBotFilterConfig.class}) +class TriggerEventIngestServiceTest { + + @Autowired private TriggerService triggerService; + @Autowired private TriggerMapper triggerMapper; + @Autowired private TriggerEventIngestService ingest; + @Autowired private WorkflowRunMapper runMapper; + @Autowired private TriggerDispatcherWorkflowTest.StubGraphLoader stubGraphLoader; + @Autowired private StubAgentInvoker stubInvoker; + @Autowired private SwitchableBotFilter botFilter; + + // Each test uses its own (workspaceId, patternType=webhook) pair so the + // ingest's selectList only returns the trigger this test owns. webhook + // is the pass-through pattern documented in TriggerPatternMatcher; we + // can't reuse synthetic types like "evt.clean" anymore because the + // matcher correctly fails closed on unknown pattern types now. + + @Test + @DisplayName("A clean event for one matching trigger produces one workflow run.") + void cleanEventFiresOnce() { + long ws = 91000L; + TriggerEntity t = createTrigger(ws, "hook", 9100L, "webhook", 60, 60); + bindGraph(9100L); + stubInvoker.respond("greeter", "ok"); + + List results = ingest.ingest(envelope( + ws, "evt-1", "u-1", "webhook")); + assertEquals(1, results.size()); + assertTrue(results.get(0).fired()); + assertEquals(t.getId(), results.get(0).triggerId()); + + List runs = runMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, 9100L)); + assertEquals(1, runs.size()); + } + + @Test + @DisplayName("Dedup window collapses repeated events with the same eventId.") + void duplicateEventIdIsDropped() { + long ws = 92000L; + createTrigger(ws, "dedup", 9200L, "webhook", 60, 60); + bindGraph(9200L); + stubInvoker.respond("greeter", "ok"); + + var first = ingest.ingest(envelope(ws, "evt-dup", "u", "webhook")); + var second = ingest.ingest(envelope(ws, "evt-dup", "u", "webhook")); + + assertTrue(first.get(0).fired()); + assertFalse(second.get(0).fired()); + assertEquals(TriggerEventIngestService.Reason.DUPLICATE, second.get(0).droppedReason()); + + List runs = runMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, 9200L)); + assertEquals(1, runs.size()); + } + + @Test + @DisplayName("Sliding rate limit drops events past the per-minute cap.") + void rateLimitedEventsAreDropped() { + long ws = 93000L; + createTrigger(ws, "burst", 9300L, "webhook", /* rate */ 2, 60); + bindGraph(9300L); + stubInvoker.respond("greeter", "ok"); + + var r1 = ingest.ingest(envelope(ws, "evt-1", "u", "webhook")); + var r2 = ingest.ingest(envelope(ws, "evt-2", "u", "webhook")); + var r3 = ingest.ingest(envelope(ws, "evt-3", "u", "webhook")); + + assertTrue(r1.get(0).fired()); + assertTrue(r2.get(0).fired()); + assertFalse(r3.get(0).fired()); + assertEquals(TriggerEventIngestService.Reason.RATE_LIMITED, r3.get(0).droppedReason()); + } + + @Test + @DisplayName("Bot-self events are dropped before any DB or dispatch work happens.") + void botSelfFilterDropsEcho() { + long ws = 94000L; + createTrigger(ws, "echo", 9400L, "webhook", 60, 60); + bindGraph(9400L); + botFilter.flagAsBot("bot-account"); + + var results = ingest.ingest(envelope(ws, "evt-1", "bot-account", "webhook")); + assertEquals(1, results.size()); + assertFalse(results.get(0).fired()); + assertEquals(TriggerEventIngestService.Reason.BOT_SELF, results.get(0).droppedReason()); + + List runs = runMapper.selectList(new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, 9400L)); + assertTrue(runs.isEmpty(), "no run row for bot-self event"); + } + + @Test + @DisplayName("Triggers exhausted on max_fires drop further events without dispatch.") + void exhaustedTriggerStopsFiring() { + long ws = 95000L; + TriggerEntity t = createTrigger(ws, "oneshot", 9500L, "webhook", 60, 60); + bindGraph(9500L); + TriggerEntity row = triggerMapper.selectById(t.getId()); + row.setMaxFires(1L); + row.setFireCount(1L); + triggerMapper.updateById(row); + + var results = ingest.ingest(envelope(ws, "evt-late", "u", "webhook")); + assertEquals(TriggerEventIngestService.Reason.EXHAUSTED, results.get(0).droppedReason()); + } + + private TriggerEntity createTrigger(long workspaceId, String name, long workflowId, + String patternType, int ratePerMin, int dedupWindowSecs) { + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(workspaceId); + t.setName(name); + t.setPatternType(patternType); + t.setPatternJson("{}"); + t.setTargetType("workflow"); + t.setTargetId(workflowId); + t.setEnabled(true); + t.setRateLimitPerMin(ratePerMin); + t.setDedupWindowSecs(dedupWindowSecs); + t.setBotSelfFilter(true); + return triggerService.create(t); + } + + private void bindGraph(long workflowId) { + stubInvoker.reset(); + stubGraphLoader.reset(); + stubGraphLoader.bind(workflowId, 1L, + "{\"steps\":[{\"name\":\"a\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"go\"}]}"); + } + + private static TriggerEventEnvelope envelope(long workspaceId, String eventId, + String senderId, String patternType) { + return new TriggerEventEnvelope(workspaceId, patternType, eventId, senderId, + Map.of("hello", "world")); + } + + @TestConfiguration + static class SwitchableBotFilterConfig { + @Bean + @Primary + SwitchableBotFilter switchableBotFilter() { return new SwitchableBotFilter(); } + } + + static class SwitchableBotFilter implements BotSelfFilter { + private final Set bots = new CopyOnWriteArraySet<>(); + + void flagAsBot(String senderId) { bots.add(senderId); } + + @Override + public boolean isBotSelf(long workspaceId, String senderId) { + return bots.contains(senderId); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/TriggerServiceLifecycleTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerServiceLifecycleTest.java new file mode 100644 index 00000000..ca8bc26e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/TriggerServiceLifecycleTest.java @@ -0,0 +1,109 @@ +package vip.mate.trigger; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.repository.TriggerMapper; +import vip.mate.trigger.scheduler.TriggerScheduler; +import vip.mate.trigger.service.TriggerService; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertNull; + +/** + * Covers the lamport / scheduler-sync invariants of {@link TriggerService}: + * pattern_version must bump on every cron expression / pattern type change + * and on every enable→disable transition; the scheduler must mirror the + * row's enabled state. The tests rely on the scheduler's package-private + * {@code isRegistered} accessor instead of waiting for an actual cron tick. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:trigger_lifecycle_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none" +}) +class TriggerServiceLifecycleTest { + + @Autowired private TriggerService triggerService; + @Autowired private TriggerMapper triggerMapper; + @Autowired private TriggerScheduler scheduler; + + @Test + @DisplayName("create() persists a v1 trigger and registers it with the scheduler when enabled.") + void createRegistersEnabled() { + TriggerEntity t = newCronTrigger("hourly", "0 0 * * * *", true); + TriggerEntity saved = triggerService.create(t); + assertEquals(1L, saved.getPatternVersion()); + assertTrue(scheduler.isRegistered(saved.getId())); + + // Disabled trigger row persists but does not occupy a scheduled slot. + TriggerEntity disabled = triggerService.create(newCronTrigger("dormant", "0 0 1 * * *", false)); + assertEquals(1L, disabled.getPatternVersion()); + assertFalse(scheduler.isRegistered(disabled.getId())); + } + + @Test + @DisplayName("update() bumps pattern_version when the cron expression changes.") + void updateBumpsLamportOnPatternChange() { + TriggerEntity created = triggerService.create(newCronTrigger("flex", "0 0 * * * *", true)); + long firstVersion = created.getPatternVersion(); + + created.setPatternJson("{\"cron\":\"0 30 * * * *\"}"); + TriggerEntity updated = triggerService.update(created); + assertEquals(firstVersion + 1, updated.getPatternVersion()); + + // No-op update does not bump the lamport. + TriggerEntity reloaded = triggerMapper.selectById(updated.getId()); + TriggerEntity touched = triggerService.update(reloaded); + assertEquals(updated.getPatternVersion(), touched.getPatternVersion()); + } + + @Test + @DisplayName("update() flipping enabled toggles scheduler registration and bumps lamport.") + void enableTransitionTogglesSchedulerAndBumpsLamport() { + TriggerEntity created = triggerService.create(newCronTrigger("toggle", "0 0 * * * *", true)); + long version = created.getPatternVersion(); + + created.setEnabled(false); + TriggerEntity disabled = triggerService.update(created); + assertEquals(version + 1, disabled.getPatternVersion()); + assertFalse(scheduler.isRegistered(disabled.getId())); + + disabled.setEnabled(true); + TriggerEntity reEnabled = triggerService.update(disabled); + assertEquals(version + 2, reEnabled.getPatternVersion()); + assertTrue(scheduler.isRegistered(reEnabled.getId())); + } + + @Test + @DisplayName("delete() removes both the row and the scheduler registration.") + void deleteUnregistersAndRemovesRow() { + TriggerEntity created = triggerService.create(newCronTrigger("ephemeral", "0 0 * * * *", true)); + long id = created.getId(); + triggerService.delete(id); + assertFalse(scheduler.isRegistered(id)); + assertNull(triggerMapper.selectById(id)); + } + + private static TriggerEntity newCronTrigger(String name, String cron, boolean enabled) { + TriggerEntity t = new TriggerEntity(); + t.setWorkspaceId(99L); + t.setName(name); + t.setPatternType("cron"); + t.setPatternJson("{\"cron\":\"" + cron + "\"}"); + t.setTargetType("workflow"); + t.setTargetId(42L); + t.setEnabled(enabled); + return t; + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/WorkflowCompletionTriggerTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/WorkflowCompletionTriggerTest.java new file mode 100644 index 00000000..bbff0e10 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/WorkflowCompletionTriggerTest.java @@ -0,0 +1,151 @@ +package vip.mate.trigger; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.context.annotation.Import; +import org.springframework.test.context.TestPropertySource; +import vip.mate.MateClawApplication; +import vip.mate.trigger.model.TriggerEntity; +import vip.mate.trigger.service.TriggerService; +import vip.mate.workflow.compiler.WorkflowParser; +import vip.mate.workflow.compiler.ir.WorkflowGraph; +import vip.mate.workflow.model.WorkflowRunEntity; +import vip.mate.workflow.repository.WorkflowRunMapper; +import vip.mate.workflow.runtime.StubAgentInvoker; +import vip.mate.workflow.runtime.StubAgentInvokerConfig; +import vip.mate.workflow.runtime.WorkflowRunRequest; +import vip.mate.workflow.runtime.WorkflowRunResult; +import vip.mate.workflow.runtime.WorkflowRunner; + +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Confirms the workflow_completion event source is genuinely wired — + * a workflow run reaching a terminal state must publish a Spring event + * that the trigger module's bridge converts into a TriggerEventEnvelope + * and pushes through the ingest pipeline. Without this end-to-end + * confirmation the runtime decision could regress quietly. + * + *

Setup: a "downstream" trigger keyed on workflow_completion fires a + * second workflow when the first one succeeds. The chain runs + * synchronously in the same JVM thread so by the time the upstream + * runner.run returns, the downstream run row should also exist. + */ +@SpringBootTest( + classes = MateClawApplication.class, + webEnvironment = SpringBootTest.WebEnvironment.NONE +) +@TestPropertySource(properties = { + "spring.datasource.url=jdbc:h2:mem:wf_completion_${random.uuid};MODE=MySQL;DATABASE_TO_LOWER=TRUE;CASE_INSENSITIVE_IDENTIFIERS=TRUE;DB_CLOSE_DELAY=-1", + "spring.ai.dashscope.api-key=test-key", + "spring.main.web-application-type=none", + "mateclaw.workflow.trigger.async-dispatch=false" +}) +@Import({StubAgentInvokerConfig.class, TriggerDispatcherWorkflowTest.StubGraphLoaderConfig.class}) +class WorkflowCompletionTriggerTest { + + @Autowired private WorkflowRunner runner; + @Autowired private WorkflowParser parser; + @Autowired private TriggerService triggerService; + @Autowired private WorkflowRunMapper runMapper; + @Autowired private TriggerDispatcherWorkflowTest.StubGraphLoader stubGraphLoader; + @Autowired private StubAgentInvoker stubInvoker; + + @Test + @DisplayName("A succeeded workflow run fans out via workflow_completion to a downstream trigger.") + void completionEventChainsToDownstreamWorkflow() { + long upstreamWf = 7100L; + long downstreamWf = 7200L; + long workspace = 510L; + + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok-upstream"); + stubInvoker.respond("downstream", "ok-downstream"); + + // Bind the downstream graph so the trigger dispatcher has something + // to compile when the completion event fires. + stubGraphLoader.reset(); + stubGraphLoader.bind(downstreamWf, 1L, + "{\"steps\":[{\"name\":\"d\",\"agentName\":\"downstream\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"chained\"}]}"); + + // Wire a trigger that fires on workflow_completion of the upstream + // workflow. The matcher narrows by sourceWorkflowId so it only fires + // for the run we're about to start. + TriggerEntity trig = new TriggerEntity(); + trig.setWorkspaceId(workspace); + trig.setName("downstream-on-upstream"); + trig.setPatternType("workflow_completion"); + trig.setPatternJson("{\"sourceWorkflowId\":" + upstreamWf + ",\"stateFilter\":\"completed\"}"); + trig.setTargetType("workflow"); + trig.setTargetId(downstreamWf); + trig.setEnabled(true); + triggerService.create(trig); + + // Run the upstream workflow. Bind a graph for runner.run; we use + // parser.parse since this test doesn't go through publish. + WorkflowGraph graph = parser.parse( + "{\"steps\":[{\"name\":\"u\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"go\"}]}"); + + WorkflowRunResult upstream = runner.run(graph, + new WorkflowRunRequest(upstreamWf, 1L, workspace, "manual", Map.of())); + assertEquals("succeeded", upstream.state()); + + // The completion event should have caused the downstream workflow + // to run synchronously. Look for its run row. + List downstreamRuns = runMapper.selectList( + new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, downstreamWf)); + assertTrue(!downstreamRuns.isEmpty(), + "completion event should have triggered a downstream run"); + assertEquals("succeeded", downstreamRuns.get(0).getState()); + // The runner stamps triggered_by with "trigger:{id}" — confirm the + // chain was traced through the trigger module, not invoked directly. + assertTrue(downstreamRuns.get(0).getTriggeredBy() != null + && downstreamRuns.get(0).getTriggeredBy().startsWith("trigger:"), + "downstream run should be triggered_by trigger:* — got " + + downstreamRuns.get(0).getTriggeredBy()); + } + + @Test + @DisplayName("A workflow_completion trigger with mismatched sourceWorkflowId stays dormant.") + void completionEventDoesNotMisfireForOtherWorkflows() { + long upstreamWf = 7300L; + long otherWf = 7400L; + long workspace = 520L; + + stubInvoker.reset(); + stubInvoker.respond("greeter", "ok"); + + // Trigger keyed on a DIFFERENT workflow id — it must not fire when + // upstreamWf completes. + TriggerEntity trig = new TriggerEntity(); + trig.setWorkspaceId(workspace); + trig.setName("only-other"); + trig.setPatternType("workflow_completion"); + trig.setPatternJson("{\"sourceWorkflowId\":" + otherWf + "}"); + trig.setTargetType("workflow"); + trig.setTargetId(otherWf); + trig.setEnabled(true); + triggerService.create(trig); + + WorkflowGraph graph = parser.parse( + "{\"steps\":[{\"name\":\"u\",\"agentName\":\"greeter\"," + + "\"mode\":{\"type\":\"sequential\"},\"promptTemplate\":\"go\"}]}"); + runner.run(graph, new WorkflowRunRequest(upstreamWf, 1L, workspace, "manual", Map.of())); + + List otherRuns = runMapper.selectList( + new LambdaQueryWrapper() + .eq(WorkflowRunEntity::getWorkflowId, otherWf)); + assertTrue(otherRuns.isEmpty(), + "trigger keyed on a different sourceWorkflowId must not fire"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/trigger/ingest/TriggerPatternMatcherTest.java b/mateclaw-server/src/test/java/vip/mate/trigger/ingest/TriggerPatternMatcherTest.java new file mode 100644 index 00000000..6bd1b358 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/trigger/ingest/TriggerPatternMatcherTest.java @@ -0,0 +1,140 @@ +package vip.mate.trigger.ingest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.trigger.model.TriggerEntity; + +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Plain JUnit coverage for {@link TriggerPatternMatcher}: the matcher is a + * pure function of (trigger row, envelope) and pulls no Spring beans, so + * tests stay POJO-only and run in milliseconds. + * + *

The shape of these cases enforces the design intent: cron is + * scheduler-driven (never fires from ingest), webhook is opaque + * pass-through, and unknown pattern types fail closed instead of + * fan-firing every workspace trigger. + */ +class TriggerPatternMatcherTest { + + private final TriggerPatternMatcher matcher = new TriggerPatternMatcher(new ObjectMapper()); + + @Test + @DisplayName("Cron patterns never match an inbound envelope — they fire from the scheduler.") + void cronAlwaysReturnsFalse() { + TriggerEntity t = trigger("cron", "{\"cron\":\"0 * * * * *\"}"); + TriggerEventEnvelope env = envelope("cron", Map.of()); + assertFalse(matcher.matches(t, env)); + } + + @Test + @DisplayName("Webhook patterns are pass-through; the secret check happens at the HTTP entry.") + void webhookAlwaysReturnsTrue() { + TriggerEntity t = trigger("webhook", "{}"); + TriggerEventEnvelope env = envelope("webhook", Map.of()); + assertTrue(matcher.matches(t, env)); + } + + @Test + @DisplayName("channel_message narrows to channelType / senderEquals when present.") + void channelMessageNarrowsByChannelType() { + TriggerEntity t = trigger("channel_message", "{\"channelType\":\"feishu\"}"); + // channelType lives in envelope.data() — the controller stuffs it + // there because the envelope record itself is generic. + assertTrue(matcher.matches(t, envelope("channel_message", Map.of("channelType", "feishu")))); + assertFalse(matcher.matches(t, envelope("channel_message", Map.of("channelType", "telegram")))); + assertFalse(matcher.matches(t, envelope("channel_message", Map.of()))); + } + + @Test + @DisplayName("channel_message narrows to senderEquals when present.") + void channelMessageNarrowsBySender() { + TriggerEntity t = trigger("channel_message", "{\"senderEquals\":\"alice\"}"); + assertTrue(matcher.matches(t, envelope("channel_message", "alice", Map.of()))); + assertFalse(matcher.matches(t, envelope("channel_message", "bob", Map.of()))); + } + + @Test + @DisplayName("content_match needs a non-blank substring or it refuses to fire.") + void contentMatchRefusesBlankSubstring() { + TriggerEntity blank = trigger("content_match", "{}"); + assertFalse(matcher.matches(blank, + envelope("content_match", Map.of("content", "anything")))); + + TriggerEntity needle = trigger("content_match", "{\"substring\":\"order\"}"); + assertTrue(matcher.matches(needle, + envelope("content_match", Map.of("content", "Place an Order, please")))); + assertFalse(matcher.matches(needle, + envelope("content_match", Map.of("content", "no relevant text")))); + } + + @Test + @DisplayName("workflow_completion can narrow to source and state.") + void workflowCompletionNarrows() { + // The runner emits state="succeeded"; the pattern's stateFilter + // accepts either the runner's vocabulary ("succeeded") or the + // ergonomic alias "completed" — both should match a succeeded run. + TriggerEntity t = trigger("workflow_completion", + "{\"sourceWorkflowId\":42,\"stateFilter\":\"completed\"}"); + assertTrue(matcher.matches(t, envelope("workflow_completion", + Map.of("sourceWorkflowId", 42L, "state", "succeeded")))); + assertFalse(matcher.matches(t, envelope("workflow_completion", + Map.of("sourceWorkflowId", 42L, "state", "failed")))); + assertFalse(matcher.matches(t, envelope("workflow_completion", + Map.of("sourceWorkflowId", 99L, "state", "succeeded")))); + + // stateFilter="failed" matches the runner's literal "failed" state. + TriggerEntity onFail = trigger("workflow_completion", + "{\"sourceWorkflowId\":42,\"stateFilter\":\"failed\"}"); + assertTrue(matcher.matches(onFail, envelope("workflow_completion", + Map.of("sourceWorkflowId", 42L, "state", "failed")))); + assertFalse(matcher.matches(onFail, envelope("workflow_completion", + Map.of("sourceWorkflowId", 42L, "state", "succeeded")))); + } + + @Test + @DisplayName("Unknown pattern types fail closed — must not fan-fire across the workspace.") + void unknownPatternFailsClosed() { + TriggerEntity t = trigger("does-not-exist", "{}"); + assertFalse(matcher.matches(t, envelope("does-not-exist", Map.of()))); + } + + @Test + @DisplayName("Malformed pattern_json is treated as empty constraints, never a throw.") + void malformedPatternJsonDoesNotThrow() { + // channel_message with empty constraints is intentionally permissive + // (matches any channel) — the test verifies no exception escapes, + // not the boolean. + TriggerEntity permissive = trigger("channel_message", "{ this is not json"); + assertTrue(matcher.matches(permissive, envelope("channel_message", + Map.of("channelType", "feishu")))); + + // content_match without a substring refuses to fire — proves the + // empty-constraint envelope still goes through the type-specific + // gate instead of being silently treated as a wildcard. + TriggerEntity strict = trigger("content_match", "{ this is not json"); + assertFalse(matcher.matches(strict, envelope("content_match", + Map.of("content", "hello")))); + } + + private static TriggerEntity trigger(String type, String json) { + TriggerEntity t = new TriggerEntity(); + t.setId(1L); + t.setPatternType(type); + t.setPatternJson(json); + return t; + } + + private static TriggerEventEnvelope envelope(String type, Map data) { + return envelope(type, "u1", data); + } + + private static TriggerEventEnvelope envelope(String type, String senderId, Map data) { + return new TriggerEventEnvelope(99L, type, "evt-" + System.nanoTime(), senderId, data); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/controller/WikiHotCacheControllerTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/controller/WikiHotCacheControllerTest.java new file mode 100644 index 00000000..e5d6e7ce --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/controller/WikiHotCacheControllerTest.java @@ -0,0 +1,103 @@ +package vip.mate.wiki.controller; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.common.result.R; +import vip.mate.wiki.hotcache.HotCacheUpdateReason; +import vip.mate.wiki.hotcache.HotCacheUpdateScheduler; +import vip.mate.wiki.hotcache.WikiHotCacheService; +import vip.mate.wiki.model.WikiHotCacheEntity; + +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Plain controller tests — pure behavioral verification, no MockMvc. + * Spring wiring is exercised by WikiHotCacheProviderE2ETest; here we + * focus on the controller's logic and call shape. + */ +class WikiHotCacheControllerTest { + + private WikiHotCacheService service; + private HotCacheUpdateScheduler scheduler; + private WikiHotCacheController controller; + + @BeforeEach + void setUp() { + service = mock(WikiHotCacheService.class); + scheduler = mock(HotCacheUpdateScheduler.class); + controller = new WikiHotCacheController(service, scheduler); + } + + @Test + @DisplayName("GET returns the row when one exists") + void get_present() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setContent("body"); + when(service.findByKb(7L)).thenReturn(Optional.of(row)); + + R resp = controller.get(7L); + + assertThat(resp.getData()).isNotNull(); + assertThat(resp.getData().getKbId()).isEqualTo(7L); + assertThat(resp.getData().getContent()).isEqualTo("body"); + } + + @Test + @DisplayName("GET returns ok with null data when no row") + void get_missing() { + when(service.findByKb(7L)).thenReturn(Optional.empty()); + + R resp = controller.get(7L); + + // ok envelope, null payload — operators distinguish "never built" vs "error" + assertThat(resp.getData()).isNull(); + } + + @Test + @DisplayName("regenerate schedules a MANUAL rebuild and returns ok") + void regenerate_schedules() { + controller.regenerate(7L); + + verify(scheduler).scheduleRebuild(7L, HotCacheUpdateReason.MANUAL); + } + + @Test + @DisplayName("regenerate response carries no payload (ack only)") + void regenerate_responseShape() { + R resp = controller.regenerate(7L); + assertThat(resp.getData()).isNull(); + } + + @Test + @DisplayName("reset soft-deletes the row when one exists") + void reset_existing() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setId(99L); + row.setKbId(7L); + when(service.findByKb(7L)).thenReturn(Optional.of(row)); + + controller.reset(7L); + + verify(service).softDelete(99L); + } + + @Test + @DisplayName("reset is a no-op when no row to delete") + void reset_missing() { + when(service.findByKb(7L)).thenReturn(Optional.empty()); + + controller.reset(7L); + + verify(service, never()).softDelete(anyLong()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheEventListenerTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheEventListenerTest.java new file mode 100644 index 00000000..2736857e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheEventListenerTest.java @@ -0,0 +1,82 @@ +package vip.mate.wiki.hotcache; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.memory.event.ConversationCompletedEvent; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.service.WikiKnowledgeBaseService; + +import java.util.List; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class HotCacheEventListenerTest { + + private HotCacheUpdateScheduler scheduler; + private WikiKnowledgeBaseService kbService; + private HotCacheEventListener listener; + + @BeforeEach + void setUp() { + scheduler = mock(HotCacheUpdateScheduler.class); + kbService = mock(WikiKnowledgeBaseService.class); + listener = new HotCacheEventListener(scheduler, kbService); + } + + private static WikiKnowledgeBaseEntity kb(Long id) { + WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity(); + kb.setId(id); + kb.setName("kb-" + id); + return kb; + } + + private static ConversationCompletedEvent event(Long agentId) { + return new ConversationCompletedEvent(agentId, "conv-1", "hi", "hello", 2, "web"); + } + + @Test + @DisplayName("agent has KBs → schedule rebuild for the first one with reason CONVERSATION_END") + void schedulesForPrimaryKb() { + when(kbService.listByAgentId(7L)).thenReturn(List.of(kb(100L), kb(200L))); + + listener.onConversationEnd(event(7L)); + + verify(scheduler).scheduleRebuild(100L, HotCacheUpdateReason.CONVERSATION_END); + verify(scheduler, never()).scheduleRebuild(eq(200L), any()); + } + + @Test + @DisplayName("agent has no KBs → no rebuild scheduled") + void noKbs_noOp() { + when(kbService.listByAgentId(7L)).thenReturn(List.of()); + + listener.onConversationEnd(event(7L)); + + verify(scheduler, never()).scheduleRebuild(any(), any()); + } + + @Test + @DisplayName("null agentId → no rebuild scheduled, no KB lookup") + void nullAgent_noOp() { + listener.onConversationEnd(event(null)); + + verify(scheduler, never()).scheduleRebuild(any(), any()); + verify(kbService, never()).listByAgentId(any()); + } + + @Test + @DisplayName("kbService throws → no rebuild scheduled, exception swallowed") + void resolverThrows_noOp() { + when(kbService.listByAgentId(eq(7L))).thenThrow(new RuntimeException("db down")); + + listener.onConversationEnd(event(7L)); + + verify(scheduler, never()).scheduleRebuild(any(), any()); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheRebuildPromptBuilderTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheRebuildPromptBuilderTest.java new file mode 100644 index 00000000..d494da1f --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheRebuildPromptBuilderTest.java @@ -0,0 +1,123 @@ +package vip.mate.wiki.hotcache; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.agent.prompt.PromptLoader; +import vip.mate.wiki.model.WikiPageEntity; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class HotCacheRebuildPromptBuilderTest { + + private HotCacheRebuildPromptBuilder builder; + + @BeforeEach + void setUp() { + PromptLoader.clearCache(); + HotCacheProperties props = new HotCacheProperties(); + builder = new HotCacheRebuildPromptBuilder(props); + } + + private static WikiPageEntity page(String slug, String title) { + WikiPageEntity p = new WikiPageEntity(); + p.setSlug(slug); + p.setTitle(title); + return p; + } + + @Test + @DisplayName("system prompt loads + reads as the rebuilder role document") + void systemPromptLoads() { + String system = builder.buildSystem(); + assertThat(system).contains("hot cache rebuilder"); + assertThat(system).contains("## Last Updated"); + assertThat(system).contains("## Key Recent Facts"); + assertThat(system).contains("## Recent Changes"); + assertThat(system).contains("## Active Threads"); + } + + @Test + @DisplayName("user prompt substitutes all placeholders with provided inputs") + void userPromptSubstitutes() { + String user = builder.buildUser( + "previous body content", + "## 2026-05-02 ingest\n- 18:30 — uploaded paper", + List.of(page("redlock", "RedLock"), page("paxos", "Paxos")), + List.of(page("distributed-locks", "Distributed Locks"))); + + assertThat(user).contains("previous body content"); + assertThat(user).contains("18:30 — uploaded paper"); + assertThat(user).contains("- [[redlock]] RedLock"); + assertThat(user).contains("- [[paxos]] Paxos"); + assertThat(user).contains("- [[distributed-locks]] Distributed Locks"); + // ISO timestamp injected — not asserting exact value, just shape + assertThat(user).matches("(?s).*\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}.*"); + // No leftover placeholder tokens + assertThat(user).doesNotContain("{previous_content}"); + assertThat(user).doesNotContain("{log_excerpt}"); + assertThat(user).doesNotContain("{recent_creates}"); + assertThat(user).doesNotContain("{recent_updates}"); + assertThat(user).doesNotContain("{iso_timestamp}"); + assertThat(user).doesNotContain("{recent_window}"); + } + + @Test + @DisplayName("blank or null sections render as (none)") + void blankSections() { + String user = builder.buildUser(null, "", List.of(), List.of()); + + // Each "(none)" appears once per missing section; we just check the + // marker is present rather than counting. + assertThat(user).contains("(none)"); + // Every placeholder still resolved. + assertThat(user).doesNotContain("{"); + } + + @Test + @DisplayName("oversized previous content is abbreviated to the configured cap") + void abbreviatesPreviousContent() { + HotCacheProperties tightProps = new HotCacheProperties(); + tightProps.setPreviousContentCap(50); + HotCacheRebuildPromptBuilder tight = new HotCacheRebuildPromptBuilder(tightProps); + + String huge = "x".repeat(500); + String user = tight.buildUser(huge, null, List.of(), List.of()); + + assertThat(user).contains("…"); + // Substring "xxxx…" — at least 49 x's then ellipsis (cap=50 → 49 x + …) + assertThat(user).contains("x".repeat(49) + "…"); + assertThat(user).doesNotContain("x".repeat(60)); + } + + @Test + @DisplayName("oversized log excerpt is abbreviated to the configured cap") + void abbreviatesLogExcerpt() { + HotCacheProperties tightProps = new HotCacheProperties(); + tightProps.setLogExcerptCap(40); + HotCacheRebuildPromptBuilder tight = new HotCacheRebuildPromptBuilder(tightProps); + + String log = "y".repeat(500); + String user = tight.buildUser(null, log, List.of(), List.of()); + + assertThat(user).contains("…"); + assertThat(user).doesNotContain("y".repeat(60)); + } + + @Test + @DisplayName("missing slug or title falls back gracefully without NPE") + void missingPageFields() { + WikiPageEntity slugless = new WikiPageEntity(); + slugless.setTitle("title-only"); + + WikiPageEntity titleless = new WikiPageEntity(); + titleless.setSlug("slug-only"); + + String user = builder.buildUser(null, null, List.of(slugless, titleless), List.of()); + + assertThat(user).contains("title-only"); + assertThat(user).contains("slug-only"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheUpdateSchedulerTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheUpdateSchedulerTest.java new file mode 100644 index 00000000..c2f446ea --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/HotCacheUpdateSchedulerTest.java @@ -0,0 +1,106 @@ +package vip.mate.wiki.hotcache; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.wiki.model.WikiHotCacheEntity; + +import java.time.Duration; +import java.time.LocalDateTime; +import java.util.Optional; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class HotCacheUpdateSchedulerTest { + + private WikiHotCacheService cacheService; + private WikiHotCacheUpdater updater; + private HotCacheProperties props; + private HotCacheUpdateScheduler scheduler; + + @BeforeEach + void setUp() { + cacheService = mock(WikiHotCacheService.class); + updater = mock(WikiHotCacheUpdater.class); + props = new HotCacheProperties(); + props.setDebounce(Duration.ofMinutes(5)); + scheduler = new HotCacheUpdateScheduler(props, cacheService, updater); + } + + @Test + @DisplayName("blocking call: no existing row → rebuild fires once") + void firstRebuild() { + when(cacheService.findByKb(7L)).thenReturn(Optional.empty()); + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.CONVERSATION_END); + verify(updater).rebuild(7L, HotCacheUpdateReason.CONVERSATION_END); + } + + @Test + @DisplayName("debounce: rebuild started 1 minute ago + window=5min → next call skipped") + void withinDebounce_skipped() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setLastRebuildStartedAt(LocalDateTime.now().minus(Duration.ofMinutes(1))); + when(cacheService.findByKb(7L)).thenReturn(Optional.of(row)); + + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.CONVERSATION_END); + + verify(updater, never()).rebuild(anyLong(), any()); + } + + @Test + @DisplayName("debounce: rebuild started 6 minutes ago + window=5min → next call passes") + void outsideDebounce_passes() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setLastRebuildStartedAt(LocalDateTime.now().minus(Duration.ofMinutes(6))); + when(cacheService.findByKb(7L)).thenReturn(Optional.of(row)); + + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.CONVERSATION_END); + + verify(updater).rebuild(7L, HotCacheUpdateReason.CONVERSATION_END); + } + + @Test + @DisplayName("MANUAL reason bypasses debounce") + void manualBypassesDebounce() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setLastRebuildStartedAt(LocalDateTime.now()); // just now + when(cacheService.findByKb(7L)).thenReturn(Optional.of(row)); + + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.MANUAL); + + verify(updater).rebuild(7L, HotCacheUpdateReason.MANUAL); + } + + @Test + @DisplayName("null kbId is a no-op") + void nullKbId() { + scheduler.rebuildNowBlocking(null, HotCacheUpdateReason.MANUAL); + verify(updater, never()).rebuild(any(), any()); + } + + @Test + @DisplayName("updater throws → caught, lock released for next call") + void updaterThrows_lockReleased() { + when(cacheService.findByKb(eq(7L))).thenReturn(Optional.empty()); + org.mockito.Mockito.doThrow(new RuntimeException("boom")) + .when(updater).rebuild(eq(7L), any()); + + // Must not throw + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.MANUAL); + + // Lock released — second call goes through + org.mockito.Mockito.reset(updater); + scheduler.rebuildNowBlocking(7L, HotCacheUpdateReason.MANUAL); + verify(updater, times(1)).rebuild(7L, HotCacheUpdateReason.MANUAL); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderE2ETest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderE2ETest.java new file mode 100644 index 00000000..4eff4e2c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderE2ETest.java @@ -0,0 +1,179 @@ +package vip.mate.wiki.hotcache; + +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.annotation.DirtiesContext; +import vip.mate.memory.spi.MemoryManager; +import vip.mate.memory.spi.MemoryProvider; +import vip.mate.system.featureflag.FeatureFlagEntity; +import vip.mate.system.featureflag.FeatureFlagService; +import vip.mate.system.featureflag.repository.FeatureFlagMapper; +import vip.mate.wiki.model.WikiHotCacheEntity; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.repository.WikiHotCacheMapper; +import vip.mate.wiki.repository.WikiKnowledgeBaseMapper; + +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Spring-context end-to-end smoke for the hot-cache injection chain. + * + *

Boots the full Spring Boot context with the H2 + Flyway test profile so + * the V82 migration runs on the in-memory DB; then verifies the hot-cache + * row → {@link WikiHotCacheProvider} → {@link MemoryManager} chain end to + * end, exercising {@link MemoryManager#buildSystemPromptBlock} (the same + * call agent-build performs at session start). + * + *

This catches wiring failures that pure mock-based unit tests miss: + * the bean discovery, the mapper round-trip, the feature-flag cache + * refresh, and the new migration column shape. + */ +@SpringBootTest( + webEnvironment = SpringBootTest.WebEnvironment.NONE, + properties = { + "spring.flyway.enabled=true", + "spring.flyway.locations=classpath:db/migration/h2", + "mateclaw.feature-flag.refresh-ms=999999" + } +) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +class WikiHotCacheProviderE2ETest { + + private static final String FLAG = "wiki.hot_cache.enabled"; + + @Autowired private MemoryManager memoryManager; + @Autowired private List allProviders; + @Autowired private WikiHotCacheProvider hotCacheProvider; + @Autowired private WikiHotCacheMapper hotCacheMapper; + @Autowired private WikiKnowledgeBaseMapper kbMapper; + @Autowired private FeatureFlagService featureFlagService; + @Autowired private FeatureFlagMapper featureFlagMapper; + + private Long agentId; + private Long kbId; + + @AfterEach + void cleanup() { + // Test data lives in the H2 file unless we wipe it; @DirtiesContext on + // the base class scrubs Spring state but not DB rows. + if (kbId != null) kbMapper.deleteById(kbId); + hotCacheMapper.delete(new LambdaQueryWrapper()); + // Reset flag to its seed default (off) for the next test. + setFlag(false); + } + + @Test + @DisplayName("WikiHotCacheProvider is discovered + present in MemoryManager's provider list") + void providerIsRegistered() { + assertThat(hotCacheProvider).isNotNull(); + assertThat(allProviders) + .extracting(MemoryProvider::id) + .contains("wiki_hot_cache"); + // Spring autowires List in registration order; MemoryManager + // applies its own enabled-filter/sort. We assert the bean made it into + // Spring's container at minimum. + } + + @Test + @DisplayName("flag off → MemoryManager.buildSystemPromptBlock excludes the hot cache section") + void flagOff_omitsHotCache() { + seedAgentAndKb(); + seedHotCacheRow(); + setFlag(false); + + String block = memoryManager.buildSystemPromptBlock(agentId); + + assertThat(block).doesNotContain("Recent Wiki Activity"); + assertThat(block).doesNotContain("smoke-test-fact"); + } + + @Test + @DisplayName("flag on + hot cache row exists → injected into MemoryManager output") + void flagOn_injectsHotCache() { + seedAgentAndKb(); + seedHotCacheRow(); + setFlag(true); + + String block = memoryManager.buildSystemPromptBlock(agentId); + + assertThat(block).contains("# Recent Wiki Activity"); + assertThat(block).contains("smoke-test-fact"); + } + + @Test + @DisplayName("flag on + KB has no hot cache row → block is empty for that section") + void flagOn_noRow_skipsSection() { + seedAgentAndKb(); + // intentionally no seedHotCacheRow() + setFlag(true); + + String block = memoryManager.buildSystemPromptBlock(agentId); + + assertThat(block).doesNotContain("Recent Wiki Activity"); + } + + @Test + @DisplayName("provider read API returns the same body the SQL row holds") + void readApi_roundTrip() { + seedAgentAndKb(); + seedHotCacheRow(); + + Optional row = hotCacheProvider.id() == null + ? Optional.empty() + : hotCacheMapper.selectList( + new LambdaQueryWrapper().eq(WikiHotCacheEntity::getKbId, kbId)) + .stream().findFirst(); + + assertThat(row).isPresent(); + assertThat(row.get().getContent()).contains("smoke-test-fact"); + } + + // ==================== helpers ==================== + + /** Inserts a KB owned by a synthetic agent so listByAgentId returns it. */ + private void seedAgentAndKb() { + // Use a high agentId we're unlikely to collide with seed data. Agents + // are referenced via foreign key on the KB row but not strictly + // enforced at the DB level (seed data has agent_id NULL too). + agentId = 9_999_001L; + + WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity(); + kb.setName("hot-cache-smoke-kb"); + kb.setAgentId(agentId); + kbMapper.insert(kb); + kbId = kb.getId(); + assertThat(kbId).isNotNull(); + } + + private void seedHotCacheRow() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(kbId); + row.setContent("## Last Updated\nsmoke-test-fact\n"); + row.setContentHash("test-hash"); + row.setLastUpdated(LocalDateTime.now()); + row.setUpdateReason("MANUAL"); + row.setRebuildCount(1L); + row.setDeleted(0); + hotCacheMapper.insert(row); + } + + private void setFlag(boolean enabled) { + FeatureFlagEntity flag = featureFlagMapper.selectOne( + new LambdaQueryWrapper() + .eq(FeatureFlagEntity::getFlagKey, FLAG)); + assertThat(flag) + .as("V78 seed should have inserted %s", FLAG) + .isNotNull(); + flag.setEnabled(enabled); + featureFlagMapper.updateById(flag); + featureFlagService.invalidate(); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderTest.java new file mode 100644 index 00000000..919f9e7c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheProviderTest.java @@ -0,0 +1,154 @@ +package vip.mate.wiki.hotcache; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.system.featureflag.FeatureFlagService; +import vip.mate.wiki.model.WikiKnowledgeBaseEntity; +import vip.mate.wiki.service.WikiKnowledgeBaseService; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class WikiHotCacheProviderTest { + + private WikiHotCacheService cacheService; + private WikiKnowledgeBaseService kbService; + private FeatureFlagService featureFlagService; + private WikiHotCacheProvider provider; + + @BeforeEach + void setUp() { + cacheService = mock(WikiHotCacheService.class); + kbService = mock(WikiKnowledgeBaseService.class); + featureFlagService = mock(FeatureFlagService.class); + provider = new WikiHotCacheProvider(cacheService, kbService, featureFlagService); + + when(featureFlagService.isEnabled("wiki.hot_cache.enabled")).thenReturn(true); + } + + private static WikiKnowledgeBaseEntity kb(Long id, String name) { + WikiKnowledgeBaseEntity kb = new WikiKnowledgeBaseEntity(); + kb.setId(id); + kb.setName(name); + return kb; + } + + @Test + @DisplayName("flag off → empty block") + void flagOff() { + when(featureFlagService.isEnabled("wiki.hot_cache.enabled")).thenReturn(false); + assertThat(provider.systemPromptBlock(7L)).isEmpty(); + } + + @Test + @DisplayName("null agentId → empty block") + void nullAgent() { + assertThat(provider.systemPromptBlock(null)).isEmpty(); + } + + @Test + @DisplayName("agent has no KBs → empty block") + void noKbs() { + when(kbService.listByAgentId(7L)).thenReturn(List.of()); + assertThat(provider.systemPromptBlock(7L)).isEmpty(); + } + + @Test + @DisplayName("KB present but no hot cache row → empty block") + void kbWithoutCache() { + when(kbService.listByAgentId(7L)).thenReturn(List.of(kb(100L, "Engineering"))); + when(cacheService.getContentOrNull(100L)).thenReturn(null); + assertThat(provider.systemPromptBlock(7L)).isEmpty(); + } + + @Test + @DisplayName("KB present with blank cache content → empty block") + void kbWithBlankCache() { + when(kbService.listByAgentId(7L)).thenReturn(List.of(kb(100L, "Engineering"))); + when(cacheService.getContentOrNull(100L)).thenReturn(" "); + assertThat(provider.systemPromptBlock(7L)).isEmpty(); + } + + @Test + @DisplayName("single KB with cache → header + body") + void singleKb() { + when(kbService.listByAgentId(7L)).thenReturn(List.of(kb(100L, "Engineering"))); + when(cacheService.getContentOrNull(100L)).thenReturn("## Last Updated\nfoo"); + + String block = provider.systemPromptBlock(7L); + + assertThat(block).startsWith("# Recent Wiki Activity\n\n"); + assertThat(block).contains("## Last Updated\nfoo"); + // Single KB: no per-KB heading + assertThat(block).doesNotContain("## Engineering"); + } + + @Test + @DisplayName("two KBs with cache → header + first body + second KB heading + body") + void twoKbs() { + when(kbService.listByAgentId(7L)).thenReturn(List.of( + kb(100L, "Engineering"), kb(200L, "Product"))); + when(cacheService.getContentOrNull(100L)).thenReturn("eng-body"); + when(cacheService.getContentOrNull(200L)).thenReturn("prod-body"); + + String block = provider.systemPromptBlock(7L); + + assertThat(block).startsWith("# Recent Wiki Activity\n\n"); + assertThat(block).contains("eng-body"); + assertThat(block).contains("\n\n## Product\n\nprod-body"); + } + + @Test + @DisplayName("three KBs → only first two contribute (prompt budget)") + void capsAtTwo() { + when(kbService.listByAgentId(7L)).thenReturn(List.of( + kb(100L, "Engineering"), kb(200L, "Product"), kb(300L, "Marketing"))); + when(cacheService.getContentOrNull(100L)).thenReturn("eng-body"); + when(cacheService.getContentOrNull(200L)).thenReturn("prod-body"); + when(cacheService.getContentOrNull(300L)).thenReturn("mkt-body"); + + String block = provider.systemPromptBlock(7L); + + assertThat(block).contains("eng-body"); + assertThat(block).contains("prod-body"); + assertThat(block).doesNotContain("mkt-body"); + assertThat(block).doesNotContain("## Marketing"); + } + + @Test + @DisplayName("first KB has no cache → second KB still contributes as the leader") + void skipsKbWithoutCache() { + when(kbService.listByAgentId(7L)).thenReturn(List.of( + kb(100L, "Engineering"), kb(200L, "Product"))); + when(cacheService.getContentOrNull(100L)).thenReturn(null); + when(cacheService.getContentOrNull(200L)).thenReturn("prod-body"); + + String block = provider.systemPromptBlock(7L); + + // Product is the only contributor → it gets the leading "Recent Wiki + // Activity" header, not a per-KB sub-heading. + assertThat(block).startsWith("# Recent Wiki Activity\n\n"); + assertThat(block).contains("prod-body"); + assertThat(block).doesNotContain("## Engineering"); + assertThat(block).doesNotContain("## Product"); + } + + @Test + @DisplayName("kbService throws → empty block, no propagation") + void kbServiceFails() { + when(kbService.listByAgentId(eq(7L))).thenThrow(new RuntimeException("db down")); + assertThat(provider.systemPromptBlock(7L)).isEmpty(); + } + + @Test + @DisplayName("id and order are stable") + void identity() { + assertThat(provider.id()).isEqualTo("wiki_hot_cache"); + assertThat(provider.order()).isEqualTo(30); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheServiceTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheServiceTest.java new file mode 100644 index 00000000..9afcd84c --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheServiceTest.java @@ -0,0 +1,129 @@ +package vip.mate.wiki.hotcache; + +import com.baomidou.mybatisplus.core.conditions.Wrapper; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.wiki.model.WikiHotCacheEntity; +import vip.mate.wiki.repository.WikiHotCacheMapper; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class WikiHotCacheServiceTest { + + private WikiHotCacheMapper mapper; + private WikiHotCacheService service; + + @BeforeEach + void setUp() { + mapper = mock(WikiHotCacheMapper.class); + service = new WikiHotCacheService(mapper); + } + + @Test + @DisplayName("findByKb returns the row when one exists") + void findByKb_returnsRow() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setContent("# Last Updated\nsomething"); + when(mapper.selectOne(any())).thenReturn(row); + + assertThat(service.findByKb(7L)).hasValueSatisfying(e -> { + assertThat(e.getKbId()).isEqualTo(7L); + assertThat(e.getContent()).contains("Last Updated"); + }); + } + + @Test + @DisplayName("findByKb returns empty when no row") + void findByKb_empty() { + when(mapper.selectOne(any())).thenReturn(null); + assertThat(service.findByKb(7L)).isEmpty(); + } + + @Test + @DisplayName("findByKb short-circuits on null kbId") + void findByKb_nullId() { + assertThat(service.findByKb(null)).isEmpty(); + verify(mapper, never()).selectOne(any(Wrapper.class)); + } + + @Test + @DisplayName("getContentOrNull unwraps body") + void getContentOrNull_present() { + WikiHotCacheEntity row = new WikiHotCacheEntity(); + row.setKbId(7L); + row.setContent("body"); + when(mapper.selectOne(any())).thenReturn(row); + + assertThat(service.getContentOrNull(7L)).isEqualTo("body"); + } + + @Test + @DisplayName("getContentOrNull returns null when row missing") + void getContentOrNull_missing() { + when(mapper.selectOne(any())).thenReturn(null); + assertThat(service.getContentOrNull(7L)).isNull(); + } + + @Test + @DisplayName("softDelete delegates to mapper.deleteById (logical delete)") + void softDelete_delegates() { + service.softDelete(42L); + verify(mapper).deleteById(42L); + } + + @Test + @DisplayName("softDelete short-circuits on null id") + void softDelete_nullId() { + service.softDelete(null); + verify(mapper, never()).deleteById((java.io.Serializable) any()); + } + + @Test + @DisplayName("HotCacheContent renders markdown with all sections") + void content_rendersMarkdown() { + HotCacheContent content = HotCacheContent.builder() + .updatedAt(java.time.Instant.parse("2026-05-02T08:30:00Z")) + .lastUpdatedSummary("ingested 3 papers on RedLock") + .keyRecentFacts(java.util.List.of( + "RedLock has known safety issues under network partition", + "Internal Redis 7.4 release notes confirm scheduled deprecation in 8.0")) + .recentChanges(java.util.List.of( + "Created: [[redlock-safety-analysis]]", + "Updated: [[distributed-locks]]")) + .activeThreads(java.util.List.of( + "Open question: should we recommend ZooKeeper for new services?")) + .build(); + + String md = content.toMarkdown(); + + assertThat(md).contains("type: meta"); + assertThat(md).contains("updated: 2026-05-02T08:30:00Z"); + assertThat(md).contains("## Last Updated\ningested 3 papers on RedLock"); + assertThat(md).contains("## Key Recent Facts\n- RedLock has known safety issues"); + assertThat(md).contains("## Recent Changes\n- Created: [[redlock-safety-analysis]]"); + assertThat(md).contains("## Active Threads\n- Open question: should we recommend ZooKeeper"); + } + + @Test + @DisplayName("HotCacheContent renders (none) for empty sections") + void content_emptySections() { + HotCacheContent content = HotCacheContent.builder() + .updatedAt(java.time.Instant.parse("2026-05-02T08:30:00Z")) + .lastUpdatedSummary("") + .build(); + + String md = content.toMarkdown(); + + assertThat(md).contains("## Last Updated\n(no recent activity)"); + assertThat(md).contains("## Key Recent Facts\n(none)"); + assertThat(md).contains("## Recent Changes\n(none)"); + assertThat(md).contains("## Active Threads\n(none)"); + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheUpdaterTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheUpdaterTest.java new file mode 100644 index 00000000..c8c30a91 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/hotcache/WikiHotCacheUpdaterTest.java @@ -0,0 +1,269 @@ +package vip.mate.wiki.hotcache; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.springframework.ai.chat.messages.AssistantMessage; +import org.springframework.ai.chat.model.ChatModel; +import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.model.Generation; +import org.springframework.ai.chat.prompt.Prompt; +import vip.mate.agent.AgentGraphBuilder; +import vip.mate.llm.model.ModelConfigEntity; +import vip.mate.llm.service.ModelConfigService; +import vip.mate.system.featureflag.FeatureFlagService; +import vip.mate.wiki.job.WikiModelRoutingService; +import vip.mate.wiki.metrics.WikiMetrics; +import vip.mate.wiki.model.WikiHotCacheEntity; +import vip.mate.wiki.model.WikiPageEntity; +import vip.mate.wiki.repository.WikiHotCacheMapper; +import vip.mate.wiki.service.WikiPageService; + +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +class WikiHotCacheUpdaterTest { + + private WikiHotCacheService cacheService; + private WikiHotCacheMapper mapper; + private HotCacheRebuildPromptBuilder promptBuilder; + private HotCacheProperties props; + private WikiModelRoutingService modelRoutingService; + private ModelConfigService modelConfigService; + private AgentGraphBuilder agentGraphBuilder; + private FeatureFlagService featureFlagService; + private WikiMetrics metrics; + private WikiPageService pageService; + private ChatModel chatModel; + + private WikiHotCacheUpdater updater; + + @BeforeEach + void setUp() { + cacheService = mock(WikiHotCacheService.class); + mapper = mock(WikiHotCacheMapper.class); + promptBuilder = mock(HotCacheRebuildPromptBuilder.class); + props = new HotCacheProperties(); + modelRoutingService = mock(WikiModelRoutingService.class); + modelConfigService = mock(ModelConfigService.class); + agentGraphBuilder = mock(AgentGraphBuilder.class); + featureFlagService = mock(FeatureFlagService.class); + metrics = mock(WikiMetrics.class); + pageService = mock(WikiPageService.class); + chatModel = mock(ChatModel.class); + + // Default: flag on, model resolution OK, prompts return stable strings + when(featureFlagService.isEnabledForKb(eq("wiki.hot_cache.enabled"), anyLong())).thenReturn(true); + when(modelRoutingService.selectModelId(anyLong(), anyString(), any())).thenReturn(42L); + ModelConfigEntity modelCfg = new ModelConfigEntity(); + modelCfg.setId(42L); + when(modelConfigService.getModel(42L)).thenReturn(modelCfg); + when(agentGraphBuilder.buildRuntimeChatModel(any(), any())).thenReturn(chatModel); + when(promptBuilder.buildSystem()).thenReturn("system-prompt"); + when(promptBuilder.buildUser(any(), any(), any(), any())).thenReturn("user-prompt"); + + updater = new WikiHotCacheUpdater(cacheService, mapper, promptBuilder, props, + modelRoutingService, modelConfigService, agentGraphBuilder, featureFlagService, + metrics, pageService); + } + + private static WikiPageEntity page(Long id) { + WikiPageEntity p = new WikiPageEntity(); + p.setId(id); + p.setSlug("p" + id); + p.setTitle("Page " + id); + return p; + } + + private void stubLlm(String body) { + Generation g = new Generation(new AssistantMessage(body)); + ChatResponse resp = new ChatResponse(List.of(g)); + when(chatModel.call(any(Prompt.class))).thenReturn(resp); + } + + private void stubRecentActivity() { + when(pageService.findRecentCreated(anyLong(), any(), anyInt())).thenReturn(List.of(page(1L))); + when(pageService.findRecentUpdated(anyLong(), any(), anyInt())).thenReturn(List.of(page(2L))); + when(pageService.getBySlug(anyLong(), anyString())).thenReturn(null); + } + + @Test + @DisplayName("flag off → returns silently, no LLM call, no DB write") + void flagOff() { + when(featureFlagService.isEnabledForKb(eq("wiki.hot_cache.enabled"), anyLong())).thenReturn(false); + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + verify(chatModel, never()).call(any(Prompt.class)); + verify(mapper, never()).insert(any(WikiHotCacheEntity.class)); + verify(mapper, never()).updateById(any(WikiHotCacheEntity.class)); + } + + @Test + @DisplayName("no recent activity → skip rebuild, clear started_at marker if present") + void noRecentActivity_skips() { + when(pageService.findRecentCreated(anyLong(), any(), anyInt())).thenReturn(List.of()); + when(pageService.findRecentUpdated(anyLong(), any(), anyInt())).thenReturn(List.of()); + when(pageService.getBySlug(anyLong(), anyString())).thenReturn(null); + + WikiHotCacheEntity existing = new WikiHotCacheEntity(); + existing.setId(99L); + existing.setKbId(7L); + existing.setLastRebuildStartedAt(java.time.LocalDateTime.now()); + when(cacheService.findByKb(7L)).thenReturn(Optional.of(existing)); + + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + verify(chatModel, never()).call(any(Prompt.class)); + // started_at cleared via updateById on the same row + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper, times(2)).updateById(captor.capture()); + // First call (markRebuildStarted) sets started_at; second (clearRebuildMarker) nulls it. + assertThat(captor.getAllValues().get(1).getLastRebuildStartedAt()).isNull(); + } + + @Test + @DisplayName("no chat model resolvable → records error, no LLM call, no body write") + void noChatModel() { + stubRecentActivity(); + when(modelConfigService.getModel(anyLong())).thenReturn(null); + when(cacheService.findByKb(7L)).thenReturn(Optional.empty()); + + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + verify(chatModel, never()).call(any(Prompt.class)); + } + + @Test + @DisplayName("happy path: LLM returns body → row inserted with content, hash, reason") + void happyPath_insert() { + stubRecentActivity(); + stubLlm("## Last Updated\nfresh snapshot"); + when(cacheService.findByKb(7L)).thenReturn(Optional.empty()); + + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper).insert(captor.capture()); + WikiHotCacheEntity inserted = captor.getValue(); + assertThat(inserted.getKbId()).isEqualTo(7L); + assertThat(inserted.getContent()).contains("fresh snapshot"); + assertThat(inserted.getContentHash()).hasSize(64); + assertThat(inserted.getUpdateReason()).isEqualTo("MANUAL"); + assertThat(inserted.getRebuildCount()).isEqualTo(1L); + assertThat(inserted.getLastRebuildError()).isNull(); + verify(metrics).recordCompileStage(eq("hot-cache-rebuild"), eq(7L), any()); + } + + @Test + @DisplayName("body unchanged: row updated but content/hash/count unchanged, reason refreshed") + void unchangedBody_skipsContentWrite() { + stubRecentActivity(); + stubLlm("## Last Updated\nidentical body"); + + // Pre-existing row with the same hash as we'd compute + WikiHotCacheEntity existing = new WikiHotCacheEntity(); + existing.setId(99L); + existing.setKbId(7L); + existing.setContent("## Last Updated\nidentical body"); + existing.setContentHash(sha256("## Last Updated\nidentical body")); + existing.setRebuildCount(5L); + // findByKb is called multiple times in the path; same Optional value works + when(cacheService.findByKb(7L)).thenReturn(Optional.of(existing)); + + updater.rebuild(7L, HotCacheUpdateReason.COMPILE_DONE); + + // The final updateById in persistRebuild leaves content + hash + count untouched + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper, times(2)).updateById(captor.capture()); + WikiHotCacheEntity finalState = captor.getAllValues().get(captor.getAllValues().size() - 1); + assertThat(finalState.getRebuildCount()).isEqualTo(5L); + assertThat(finalState.getContent()).isEqualTo("## Last Updated\nidentical body"); + assertThat(finalState.getUpdateReason()).isEqualTo("COMPILE_DONE"); + } + + @Test + @DisplayName("LLM returns blank body → recorded as failure, no body write") + void blankResponse_recordsFailure() { + stubRecentActivity(); + stubLlm(" "); + WikiHotCacheEntity existing = new WikiHotCacheEntity(); + existing.setId(99L); + existing.setKbId(7L); + when(cacheService.findByKb(7L)).thenReturn(Optional.of(existing)); + + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper, times(2)).updateById(captor.capture()); + WikiHotCacheEntity finalState = captor.getAllValues().get(captor.getAllValues().size() - 1); + assertThat(finalState.getLastRebuildError()).isEqualTo("LLM returned empty body"); + } + + @Test + @DisplayName("LLM call throws → recorded as failure, exception swallowed") + void llmException_swallowed() { + stubRecentActivity(); + when(chatModel.call(any(Prompt.class))).thenThrow(new RuntimeException("model timeout")); + WikiHotCacheEntity existing = new WikiHotCacheEntity(); + existing.setId(99L); + existing.setKbId(7L); + when(cacheService.findByKb(7L)).thenReturn(Optional.of(existing)); + + // Must NOT throw + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper, times(2)).updateById(captor.capture()); + WikiHotCacheEntity finalState = captor.getAllValues().get(captor.getAllValues().size() - 1); + assertThat(finalState.getLastRebuildError()).contains("model timeout"); + } + + @Test + @DisplayName("oversize LLM body is truncated to maxChars") + void truncatesOversizeBody() { + stubRecentActivity(); + String huge = "z".repeat(props.getMaxChars() + 500); + stubLlm(huge); + when(cacheService.findByKb(7L)).thenReturn(Optional.empty()); + + updater.rebuild(7L, HotCacheUpdateReason.MANUAL); + + ArgumentCaptor captor = ArgumentCaptor.forClass(WikiHotCacheEntity.class); + verify(mapper).insert(captor.capture()); + WikiHotCacheEntity row = captor.getValue(); + assertThat(row.getContent()).hasSize(props.getMaxChars()); + assertThat(row.getContent()).endsWith("…"); + } + + @Test + @DisplayName("null kbId is a no-op") + void nullKbId() { + updater.rebuild(null, HotCacheUpdateReason.MANUAL); + verify(featureFlagService, never()).isEnabledForKb(anyString(), anyLong()); + } + + private static String sha256(String s) { + try { + byte[] digest = java.security.MessageDigest.getInstance("SHA-256") + .digest(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(64); + for (byte b : digest) sb.append(String.format("%02x", b)); + return sb.toString(); + } catch (Exception e) { + return "no-hash"; + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/metrics/WikiMetricsTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/metrics/WikiMetricsTest.java new file mode 100644 index 00000000..19037ea9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/metrics/WikiMetricsTest.java @@ -0,0 +1,175 @@ +package vip.mate.wiki.metrics; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.ObjectProvider; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Unit tests for {@link WikiMetrics}. + * + *

Covers three regimes: + *

    + *
  • Registry available: meters are registered with correct tags
  • + *
  • Registry absent: all methods become no-ops, never throw
  • + *
  • {@link WikiTimerSample}: try-with-resources records elapsed time
  • + *
+ */ +class WikiMetricsTest { + + @Test + @DisplayName("recordCompileStage registers timer with stage and kb_id tags") + void recordCompileStage_registersTaggedTimer() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + metrics.recordCompileStage("summary", 42L, Duration.ofMillis(150)); + + var timer = registry.find("wiki.compile.stage") + .tag("stage", "summary") + .tag("kb_id", "42") + .timer(); + assertThat(timer).isNotNull(); + assertThat(timer.count()).isEqualTo(1); + } + + @Test + @DisplayName("recordCompileCache emits hit/miss counter and tokens_saved") + void recordCompileCache_emitsBothCounters() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + metrics.recordCompileCache(true, 800); + metrics.recordCompileCache(false, 0); + + assertThat(registry.find("wiki.compile.cache.outcome").tag("outcome", "hit").counter().count()) + .isEqualTo(1); + assertThat(registry.find("wiki.compile.cache.outcome").tag("outcome", "miss").counter().count()) + .isEqualTo(1); + assertThat(registry.find("wiki.compile.cache.tokens_saved").counter().count()) + .isEqualTo(800); + } + + @Test + @DisplayName("recordRetrieval tags by mode and increments result counter") + void recordRetrieval_tagsByMode() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + metrics.recordRetrieval("hybrid", Duration.ofMillis(50), 5); + metrics.recordRetrieval("hybrid", Duration.ofMillis(80), 3); + + var timer = registry.find("wiki.retrieval.duration").tag("mode", "hybrid").timer(); + assertThat(timer.count()).isEqualTo(2); + assertThat(registry.find("wiki.retrieval.results").tag("mode", "hybrid").counter().count()) + .isEqualTo(8); + } + + @Test + @DisplayName("recordVisionCall tags by provider and outcome") + void recordVisionCall_tagsByProviderAndOutcome() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + metrics.recordVisionCall("dashscope-vision", true, Duration.ofMillis(2000)); + metrics.recordVisionCall("dashscope-vision", false, Duration.ofMillis(500)); + + assertThat(registry.find("wiki.vision.call") + .tag("provider", "dashscope-vision") + .tag("outcome", "success").timer().count()).isEqualTo(1); + assertThat(registry.find("wiki.vision.call") + .tag("provider", "dashscope-vision") + .tag("outcome", "failure").timer().count()).isEqualTo(1); + } + + @Test + @DisplayName("Without MeterRegistry available, all methods are silent no-ops") + void noRegistry_allMethodsNoOp() { + @SuppressWarnings("unchecked") + ObjectProvider empty = mock(ObjectProvider.class); + when(empty.getIfAvailable()).thenReturn(null); + + WikiMetrics metrics = new WikiMetrics(empty); + + // None of these may throw. + metrics.recordCompileStage("summary", 1L, Duration.ZERO); + metrics.recordCompileCache(true, 100); + metrics.recordRelationCompute(1L, 50, Duration.ZERO); + metrics.recordRelationCacheHit(true); + metrics.recordRetrieval("hybrid", Duration.ZERO, 5); + metrics.recordVisionCall("p", true, Duration.ZERO); + metrics.recordVisionCacheHit(false); + + // Sample close should also be silent. + try (var sample = metrics.startTimer("wiki.test.foo", "kb_id", "1")) { + // no-op + } + } + + @Test + @DisplayName("startTimer records elapsed time on close, with tags applied") + void timerSample_recordsOnClose() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + try (var sample = metrics.startTimer("wiki.test.foo", "kb_id", "7")) { + sleepMillis(5); + } + + var timer = registry.find("wiki.test.foo").tag("kb_id", "7").timer(); + assertThat(timer).isNotNull(); + assertThat(timer.count()).isEqualTo(1); + assertThat(timer.totalTime(java.util.concurrent.TimeUnit.MILLISECONDS)).isGreaterThanOrEqualTo(1); + } + + @Test + @DisplayName("Calling close twice on a sample does not double-record") + void timerSample_idempotentClose() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + WikiTimerSample sample = metrics.startTimer("wiki.test.idempotent"); + sample.close(); + sample.close(); + + assertThat(registry.find("wiki.test.idempotent").timer().count()).isEqualTo(1); + } + + @Test + @DisplayName("Same meter name + tags is registered only once across calls") + void meterCacheReusesRegistration() { + SimpleMeterRegistry registry = new SimpleMeterRegistry(); + WikiMetrics metrics = new WikiMetrics(provider(registry)); + + for (int i = 0; i < 100; i++) { + metrics.recordCompileStage("summary", 1L, Duration.ofMillis(1)); + } + + // 100 records on a single meter, not 100 separate meters. + assertThat(registry.getMeters().stream() + .filter(m -> m.getId().getName().equals("wiki.compile.stage")) + .count()).isEqualTo(1); + } + + @SuppressWarnings("unchecked") + private static ObjectProvider provider(MeterRegistry r) { + ObjectProvider p = mock(ObjectProvider.class); + when(p.getIfAvailable()).thenReturn(r); + return p; + } + + private static void sleepMillis(long millis) { + try { + Thread.sleep(millis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/wiki/service/DocumentPreprocessServiceTest.java b/mateclaw-server/src/test/java/vip/mate/wiki/service/DocumentPreprocessServiceTest.java new file mode 100644 index 00000000..8fb4ef19 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/wiki/service/DocumentPreprocessServiceTest.java @@ -0,0 +1,121 @@ +package vip.mate.wiki.service; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import vip.mate.wiki.WikiProperties; +import vip.mate.wiki.dto.WikiChunkDraft; +import vip.mate.wiki.model.WikiRawMaterialEntity; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * RFC-051 PR-1c: pin the preprocessor's metadata extraction so future PR-1c + * extensions (Tika, smarter chunkers) don't silently drop page numbers or + * heading breadcrumbs. + */ +class DocumentPreprocessServiceTest { + + private DocumentPreprocessService service; + private WikiContentNormalizer normalizer; + + /** Single-window chunker: each test asserts at chunk[0]. */ + private static final DocumentPreprocessService.Chunker WHOLE_AS_ONE_CHUNK = + text -> List.of(new int[]{0, text.length()}); + + @BeforeEach + void setUp() { + normalizer = new WikiContentNormalizer(); + service = new DocumentPreprocessService(normalizer, new WikiProperties()); + } + + private WikiRawMaterialEntity raw(String type) { + WikiRawMaterialEntity r = new WikiRawMaterialEntity(); + r.setSourceType(type); + return r; + } + + @Test + @DisplayName("markdown headings produce header_breadcrumb and source_section") + void markdownHeadingsBecomeBreadcrumb() { + String text = "# Intro\nWelcome.\n## Setup\nDo this.\n### Linux\nDetails follow."; + DocumentPreprocessService.Chunker chunker = t -> { + int linuxIdx = t.indexOf("Details"); + return List.of(new int[]{linuxIdx, t.length()}); + }; + List drafts = service.preprocess(raw("markdown"), text, chunker); + assertEquals(1, drafts.size()); + WikiChunkDraft d = drafts.get(0); + assertEquals("Intro / Setup / Linux", d.headerBreadcrumb()); + assertEquals("Linux", d.sourceSection()); + } + + @Test + @DisplayName("PDF page markers map chunk to its enclosing page number") + void pdfPageMarkers() { + String text = "--- Page 1 ---\nFirst page body.\n--- Page 2 ---\nSecond page body here."; + DocumentPreprocessService.Chunker chunker = t -> { + int second = t.indexOf("Second page body"); + return List.of(new int[]{second, t.length()}); + }; + List drafts = service.preprocess(raw("pdf"), text, chunker); + assertEquals(1, drafts.size()); + assertEquals(2, drafts.get(0).pageNumber()); + } + + @Test + @DisplayName("token_count uses ceil(charCount / 4) for every chunk") + void tokenCountHeuristic() { + String text = "a".repeat(17); // 17 chars → 5 tokens + List drafts = service.preprocess(raw("text"), text, WHOLE_AS_ONE_CHUNK); + assertEquals(1, drafts.size()); + assertEquals(5, drafts.get(0).tokenCount()); + } + + @Test + @DisplayName("chunk before any heading has null breadcrumb") + void noHeadingsAboveChunk() { + String text = "Plain paragraph with no headings at all."; + List drafts = service.preprocess(raw("text"), text, WHOLE_AS_ONE_CHUNK); + assertEquals(1, drafts.size()); + assertNull(drafts.get(0).headerBreadcrumb()); + assertNull(drafts.get(0).sourceSection()); + assertNull(drafts.get(0).pageNumber()); + } + + @Test + @DisplayName("blank input yields no drafts") + void blankInputIsEmpty() { + assertTrue(service.preprocess(raw("text"), "", WHOLE_AS_ONE_CHUNK).isEmpty()); + assertTrue(service.preprocess(raw("text"), null, WHOLE_AS_ONE_CHUNK).isEmpty()); + } + + @Test + @DisplayName("HTML normalization strips nav/footer/script and emits headings on their own lines") + void htmlNormalizationCleansNoise() { + String html = "" + + "

Title

Body para.

" + + "
copy
"; + String normalized = normalizer.normalize("html", html); + assertFalse(normalized.contains("alert"), " + + diff --git a/mateclaw-ui/src/components/chat/RecoverableModelBanner.vue b/mateclaw-ui/src/components/chat/RecoverableModelBanner.vue new file mode 100644 index 00000000..84f26c18 --- /dev/null +++ b/mateclaw-ui/src/components/chat/RecoverableModelBanner.vue @@ -0,0 +1,78 @@ + + + + + diff --git a/mateclaw-ui/src/components/chat/StreamLoadingBar.vue b/mateclaw-ui/src/components/chat/StreamLoadingBar.vue index bb831e31..ad1d4047 100644 --- a/mateclaw-ui/src/components/chat/StreamLoadingBar.vue +++ b/mateclaw-ui/src/components/chat/StreamLoadingBar.vue @@ -36,6 +36,28 @@ interface LifecycleStage { since: number } +/** Latest compact_status SSE event from useChat. Renders an inline chip + * so the user can see that the pause is the window manager compacting + * history, not a network stall. */ +interface CompactStatus { + status: 'start' | 'pair_safe' | 'summarize' | 'done' | 'skipped' | 'failed' + preTokens?: number + postTokens?: number + messagesIn?: number + messagesSummarized?: number + tailKept?: number + toolResultsSpilled?: number + reason?: string + anchored?: boolean + fromCache?: boolean + movedFrom?: number + movedTo?: number + summaryBudget?: number + trigger?: string + fallbackKept?: number + timestamp?: number +} + interface Props { isLoading: boolean toolCount?: number @@ -53,6 +75,8 @@ interface Props { hasQueued?: boolean /** Fine-grained pre-token stage. Preferred over `phase` while no token has arrived. */ lifecycleStage?: LifecycleStage | null + /** Latest compact_status event. When non-null and not 'done', the bar shows compaction copy. */ + compactStatus?: CompactStatus | null } const props = withDefaults(defineProps(), { @@ -66,6 +90,7 @@ const props = withDefaults(defineProps(), { runningToolName: '', hasQueued: false, lifecycleStage: null, + compactStatus: null, }) const { t } = useI18n() @@ -119,7 +144,51 @@ const inPreTokenWindow = computed(() => { return !!ls && ls.stage !== 'streaming' }) +/** + * Compaction copy. Takes priority over both lifecycleStage and phase + * while the compactor is mid-pass (any status except done/skipped/failed) + * because the user cares more about "we paused to compact" than the + * underlying llm-request lifecycle. After done/skipped/failed we let the + * regular phase text take over — the chip's transient hint suffices. + */ +const compactStatusText = computed(() => { + const cs = props.compactStatus + if (!cs) return '' + switch (cs.status) { + case 'start': + return cs.preTokens + ? t('chat.compactStartWithTokens', { tokens: formatTokens(cs.preTokens) }) + : t('chat.compactStart') + case 'pair_safe': + return t('chat.compactPairSafe') + case 'summarize': { + const n = cs.messagesSummarized ?? cs.messagesIn ?? 0 + return n > 0 + ? t('chat.compactSummarizeWithCount', { count: n }) + : t('chat.compactSummarize') + } + default: + return '' + } +}) + +const isCompactActive = computed(() => { + const s = props.compactStatus?.status + return s === 'start' || s === 'pair_safe' || s === 'summarize' +}) + +function formatTokens(n: number): string { + if (n >= 1000) return `${(n / 1000).toFixed(1)}k tokens` + return `${n} tokens` +} + const statusText = computed(() => { + // Compaction copy wins while a pass is in flight. Done/skipped/failed + // fall through to the regular phase text so the chip releases focus. + if (isCompactActive.value) { + const cText = compactStatusText.value + if (cText) return cText + } // Prefer fine-grained pre-token text when no first delta has arrived yet. if (inPreTokenWindow.value && props.lifecycleStage) { const key = lifecycleI18nMap[props.lifecycleStage.stage] diff --git a/mateclaw-ui/src/components/common/ModelPicker.vue b/mateclaw-ui/src/components/common/ModelPicker.vue new file mode 100644 index 00000000..3c97abac --- /dev/null +++ b/mateclaw-ui/src/components/common/ModelPicker.vue @@ -0,0 +1,408 @@ + + + + + diff --git a/mateclaw-ui/src/components/skill/PreflightInstallDialog.vue b/mateclaw-ui/src/components/skill/PreflightInstallDialog.vue index 98519241..e1408a9e 100644 --- a/mateclaw-ui/src/components/skill/PreflightInstallDialog.vue +++ b/mateclaw-ui/src/components/skill/PreflightInstallDialog.vue @@ -79,6 +79,7 @@ import { computed, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' import { ElMessage } from 'element-plus' import { skillApi } from '@/api/index' +import { copyToClipboard } from '@/utils/clipboard' interface RequirementStatus { key: string @@ -146,21 +147,7 @@ function handleClose() { async function copy(cmd: string) { try { - if (navigator.clipboard) { - await navigator.clipboard.writeText(cmd) - } else { - // Fallback for clipboard-API-disabled contexts (eg http://). The - // textarea trick is broadly supported and avoids a noisy permission - // failure on the production-ish workflow. - const ta = document.createElement('textarea') - ta.value = cmd - ta.style.position = 'fixed' - ta.style.left = '-9999px' - document.body.appendChild(ta) - ta.select() - document.execCommand('copy') - document.body.removeChild(ta) - } + await copyToClipboard(cmd) ElMessage.success(t('common.copied')) } catch { ElMessage.warning(t('common.copyFailed')) diff --git a/mateclaw-ui/src/components/skill/SkillSecretsPanel.vue b/mateclaw-ui/src/components/skill/SkillSecretsPanel.vue new file mode 100644 index 00000000..350f15e2 --- /dev/null +++ b/mateclaw-ui/src/components/skill/SkillSecretsPanel.vue @@ -0,0 +1,433 @@ + + + + + diff --git a/mateclaw-ui/src/components/workflow/CreateWorkflowDialog.vue b/mateclaw-ui/src/components/workflow/CreateWorkflowDialog.vue new file mode 100644 index 00000000..b11bee02 --- /dev/null +++ b/mateclaw-ui/src/components/workflow/CreateWorkflowDialog.vue @@ -0,0 +1,244 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiKBCard.vue b/mateclaw-ui/src/views/Wiki/components/WikiKBCard.vue new file mode 100644 index 00000000..bcfb7621 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiKBCard.vue @@ -0,0 +1,177 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiLibrary.vue b/mateclaw-ui/src/views/Wiki/components/WikiLibrary.vue new file mode 100644 index 00000000..b59de633 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiLibrary.vue @@ -0,0 +1,254 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiPageSidebar.vue b/mateclaw-ui/src/views/Wiki/components/WikiPageSidebar.vue new file mode 100644 index 00000000..a35c6ee1 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiPageSidebar.vue @@ -0,0 +1,599 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue new file mode 100644 index 00000000..6faea039 --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiWorkspace.vue @@ -0,0 +1,124 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/components/WikiWorkspaceHeader.vue b/mateclaw-ui/src/views/Wiki/components/WikiWorkspaceHeader.vue new file mode 100644 index 00000000..ccb7b39b --- /dev/null +++ b/mateclaw-ui/src/views/Wiki/components/WikiWorkspaceHeader.vue @@ -0,0 +1,148 @@ + + + + + diff --git a/mateclaw-ui/src/views/Wiki/index.vue b/mateclaw-ui/src/views/Wiki/index.vue index 23a1089e..8e017f9d 100644 --- a/mateclaw-ui/src/views/Wiki/index.vue +++ b/mateclaw-ui/src/views/Wiki/index.vue @@ -2,297 +2,21 @@
-
-
-
{{ t('wiki.kicker') }}
-

{{ t('nav.wiki') }}

-

{{ t('wiki.desc') }}

-
- -
- -
- -
- - - - - - -
- - -
-
- - - -

{{ t('wiki.selectKB') }}

-
- -
-
- -
- -
- -
- -
- -
- - - -

{{ t('wiki.selectPage') }}

-
-
- -
- -
- -
- -
- -
- -
-
-
-
+ +
-