diff --git a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java index 9400ecd8..b140e069 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/ChannelManager.java @@ -92,6 +92,14 @@ public class ChannelManager { */ private final vip.mate.channel.media.GeneratedFileScrubber generatedFileScrubber; + /** + * Feishu CardKit streaming-card manager. Drives + * {@link vip.mate.channel.feishu.FeishuChannelAdapter}'s + * {@code processStream} so the receiver sees text appearing + * character-by-character instead of waiting for the full reply. + */ + private final vip.mate.channel.feishu.FeishuStreamingCardManager feishuStreamingCardManager; + /** * Distributed leader election. Channels whose adapter reports * {@link ChannelAdapter#requiresSingleLeader()} are gated on a lease so @@ -1158,7 +1166,7 @@ public class ChannelManager { case "web" -> new WebChannelAdapter(channel, messageRouter, objectMapper); case "dingtalk" -> new DingTalkChannelAdapter(channel, messageRouter, objectMapper, generatedFileCache); case "feishu" -> new FeishuChannelAdapter(channel, messageRouter, objectMapper, - feishuMediaUploader, generatedFileScrubber); + feishuMediaUploader, generatedFileScrubber, feishuStreamingCardManager); case "telegram" -> new TelegramChannelAdapter(channel, messageRouter, objectMapper); case "discord" -> new DiscordChannelAdapter(channel, messageRouter, objectMapper); case "wecom" -> new WeComChannelAdapter(channel, messageRouter, objectMapper, diff --git a/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java index c39d64ca..1a0faa5b 100644 --- a/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java +++ b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuChannelAdapter.java @@ -5,10 +5,13 @@ import com.lark.oapi.event.EventDispatcher; import com.lark.oapi.service.im.ImService; import com.lark.oapi.service.im.v1.model.P2MessageReceiveV1; import lombok.extern.slf4j.Slf4j; +import reactor.core.publisher.Flux; +import vip.mate.agent.AgentService.StreamDelta; import vip.mate.channel.AbstractChannelAdapter; import vip.mate.channel.ChannelMessage; import vip.mate.channel.ChannelMessageRouter; import vip.mate.channel.ExponentialBackoff; +import vip.mate.channel.StreamingChannelAdapter; import vip.mate.channel.media.GeneratedFileScrubber; import vip.mate.channel.media.MediaSource; import vip.mate.channel.media.MediaUploadException; @@ -66,7 +69,7 @@ import java.util.concurrent.TimeUnit; * @author MateClaw Team */ @Slf4j -public class FeishuChannelAdapter extends AbstractChannelAdapter { +public class FeishuChannelAdapter extends AbstractChannelAdapter implements StreamingChannelAdapter { public static final String CHANNEL_TYPE = "feishu"; @@ -128,10 +131,13 @@ public class FeishuChannelAdapter extends AbstractChannelAdapter { /** Scrubs {@code /api/v1/files/generated/{id}} URLs into native attachments. Nullable for legacy callers. */ private final GeneratedFileScrubber generatedFileScrubber; + /** CardKit streaming-card manager. Nullable for legacy callers / tests. */ + private final FeishuStreamingCardManager streamingCardManager; + public FeishuChannelAdapter(ChannelEntity channelEntity, ChannelMessageRouter messageRouter, ObjectMapper objectMapper) { - this(channelEntity, messageRouter, objectMapper, null, null); + this(channelEntity, messageRouter, objectMapper, null, null, null); } public FeishuChannelAdapter(ChannelEntity channelEntity, @@ -139,9 +145,19 @@ public class FeishuChannelAdapter extends AbstractChannelAdapter { ObjectMapper objectMapper, FeishuMediaUploader mediaUploader, GeneratedFileScrubber generatedFileScrubber) { + this(channelEntity, messageRouter, objectMapper, mediaUploader, generatedFileScrubber, null); + } + + public FeishuChannelAdapter(ChannelEntity channelEntity, + ChannelMessageRouter messageRouter, + ObjectMapper objectMapper, + FeishuMediaUploader mediaUploader, + GeneratedFileScrubber generatedFileScrubber, + FeishuStreamingCardManager streamingCardManager) { super(channelEntity, messageRouter, objectMapper); this.mediaUploader = mediaUploader; this.generatedFileScrubber = generatedFileScrubber; + this.streamingCardManager = streamingCardManager; // Feishu WebSocket reconnect: 2s→4s→8s→16s→30s, infinite retry this.backoff = new ExponentialBackoff(2000, 30000, 2.0, -1); } @@ -1437,6 +1453,136 @@ public class FeishuChannelAdapter extends AbstractChannelAdapter { */ static final int MAX_TEXT_MESSAGE_CHARS = 4000; + // ==================== StreamingChannelAdapter ==================== + + /** + * Stream consumption strategy: + * + */ + @Override + public String processStream(Flux stream, ChannelMessage message, String conversationId) { + if (!isCardStreamingEnabled() || streamingCardManager == null + || channelEntity == null) { + return processStreamAsText(stream, message); + } + + String receiveId = pickReceiveId(message); + if (receiveId == null) { + log.warn("[feishu-stream] No usable receive id on message; falling back to text mode"); + return processStreamAsText(stream, message); + } + String receiveIdType = resolveReceiveIdType(receiveId); + + String sessionKey = streamingCardManager.createAndDeliver( + channelEntity.getId(), receiveIdType, receiveId, null); + if (sessionKey == null) { + log.info("[feishu-stream] Card create/deliver failed; falling back to text mode"); + return processStreamAsText(stream, message); + } + + StringBuilder accumulator = new StringBuilder(); + try { + stream.doOnNext(delta -> { + if (delta.content() != null) { + accumulator.append(delta.content()); + streamingCardManager.appendContent(sessionKey, delta.content(), false); + } + }) + .doOnError(err -> { + log.error("[feishu-stream] stream error: sessionKey={}, err={}", + sessionKey, err.getMessage()); + streamingCardManager.failCard(sessionKey, err.getMessage()); + }) + .blockLast(Duration.ofMinutes(5)); + + String finalContent = accumulator.toString(); + if (finalContent.isBlank()) { + finalContent = "（无回复内容）"; + } + streamingCardManager.finishCard(sessionKey, finalContent); + log.info("[feishu-stream] Card streaming completed: sessionKey={}, contentLen={}", + sessionKey, finalContent.length()); + return finalContent; + + } catch (Exception e) { + log.error("[feishu-stream] Card streaming failed: sessionKey={}, err={}", + sessionKey, e.getMessage(), e); + streamingCardManager.failCard(sessionKey, e.getMessage()); + + // Tag returned content with the "[错误] " prefix so + // ChannelMessageRouter.isErrorReply flips status='error' on the + // persisted row and BaseAgent.sanitizeForLlm filters it out of + // the next turn's history. Without this, partial streaming + // output (e.g. LLM 400'd mid-stream) would re-enter the prompt + // as a valid assistant turn and re-trigger the same 400. + String partial = accumulator.toString(); + String errorPrefix = "[错误] Feishu CardKit streaming failed: " + e.getMessage(); + if (!partial.isBlank()) { + return errorPrefix + "\n\n（已生成的部分内容，已忽略）\n" + partial; + } + throw new RuntimeException(errorPrefix, e); + } + } + + /** + * Streaming fallback — accumulate all deltas, then send through the + * existing {@link #sendMessage} path so the message goes out as a + * regular text bubble (auto-upgraded to a non-streaming card by the + * existing {@code card_format} logic when content looks card-worthy). + */ + private String processStreamAsText(Flux stream, ChannelMessage message) { + StringBuilder accumulator = new StringBuilder(); + stream.doOnNext(delta -> { + if (delta.content() != null) { + accumulator.append(delta.content()); + } + }) + .blockLast(Duration.ofMinutes(5)); + String finalContent = accumulator.toString(); + if (!finalContent.isBlank()) { + String replyTarget = message.getReplyToken() != null + ? message.getReplyToken() + : (message.getChatId() != null ? message.getChatId() : message.getSenderId()); + if (replyTarget != null) { + sendMessage(replyTarget, finalContent); + } + } + return finalContent; + } + + /** Resolve the best id to receive a streaming card — prefer reply token, then chat, then sender. */ + private static String pickReceiveId(ChannelMessage message) { + if (message == null) return null; + if (message.getReplyToken() != null && !message.getReplyToken().isBlank()) { + return message.getReplyToken(); + } + if (message.getChatId() != null && !message.getChatId().isBlank()) { + return message.getChatId(); + } + if (message.getSenderId() != null && !message.getSenderId().isBlank()) { + return message.getSenderId(); + } + return null; + } + + private boolean isCardStreamingEnabled() { + // Default true — streaming cards are the better UX when CardKit is + // available. Operators can flip card_streaming_enabled=false in + // configJson to fall back to the text path (useful for debugging + // or when targeting an old Feishu tenant that hasn't rolled out + // CardKit v1 universally). + return getConfigBoolean("card_streaming_enabled", true); + } + @Override public void sendMessage(String targetId, String content) { if (httpClient == null) { diff --git a/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuStreamingCardManager.java b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuStreamingCardManager.java new file mode 100644 index 00000000..1b883057 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/channel/feishu/FeishuStreamingCardManager.java @@ -0,0 +1,440 @@ +package vip.mate.channel.feishu; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.lark.oapi.Client; +import com.lark.oapi.service.cardkit.v1.model.ContentCardElementReq; +import com.lark.oapi.service.cardkit.v1.model.ContentCardElementReqBody; +import com.lark.oapi.service.cardkit.v1.model.ContentCardElementResp; +import com.lark.oapi.service.cardkit.v1.model.CreateCardReq; +import com.lark.oapi.service.cardkit.v1.model.CreateCardReqBody; +import com.lark.oapi.service.cardkit.v1.model.CreateCardResp; +import com.lark.oapi.service.cardkit.v1.model.SettingsCardReq; +import com.lark.oapi.service.cardkit.v1.model.SettingsCardReqBody; +import com.lark.oapi.service.cardkit.v1.model.SettingsCardResp; +import com.lark.oapi.service.im.v1.model.CreateMessageReq; +import com.lark.oapi.service.im.v1.model.CreateMessageReqBody; +import com.lark.oapi.service.im.v1.model.CreateMessageResp; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Streaming-card lifecycle for the Feishu CardKit v1 API. + * + *

Flow per stream: + *

    + *
  1. {@link #createAndDeliver} — build a {@code streaming_mode=true} + * schema-2.0 card via {@code cardkit/v1/card.create}, then send + * it to the user via {@code im/v1/message.create} as an + * {@code interactive} message referencing the new {@code card_id}. + * Returns a session key used by subsequent calls.
    2. + *
  2. {@link #appendContent} — accumulate delta text and, when the + * throttle window permits or the caller forces a flush, push the + * current accumulator to the card's markdown element via + * {@code cardkit/v1/cardElement.content} with a monotonic + * sequence number.
    3. + *
  3. {@link #finishCard} — push the final full content one last + * time, then turn off {@code streaming_mode} via + * {@code cardkit/v1/card.settings} so the receiver UI stops + * showing the typing animation.
    4. + *
  4. {@link #failCard} — append an error marker to whatever was + * accumulated, then close streaming the same way.
    5. + *
+ * + *

Designed mirror-image to {@code DingTalkAICardManager}: same + * create/append/finish/fail shape, same per-session throttling, + * same activeSessions map for hand-off between threads. The + * implementation is end-to-end {@code oapi-sdk} — no hand-rolled HTTP. + * + *

The four SDK call sites are {@code protected} so unit tests can + * subclass and verify session/throttle behavior without booting a real + * Feishu credential or hitting the network. + */ +@Slf4j +@Component +public class FeishuStreamingCardManager { + + /** Throttle window for {@link #appendContent}, ms — matches DingTalk AICard. */ + static final long THROTTLE_INTERVAL_MS = 500; + + /** + * Markdown element id baked into the initial streaming card. + * Content-update calls reference this id. Public so tests can assert. + */ + public static final String STREAM_ELEMENT_ID = "stream_md"; + + /** Default text shown when the card is first created, before any delta arrives. */ + public static final String DEFAULT_INITIAL_TEXT = "🤔 思考中..."; + + private final FeishuClientFactory clientFactory; + private final ObjectMapper objectMapper; + + /** sessionKey → CardSession. sessionKey is an opaque UUID handed back to the caller. */ + private final ConcurrentHashMap activeSessions = new ConcurrentHashMap<>(); + + public FeishuStreamingCardManager(FeishuClientFactory clientFactory, ObjectMapper objectMapper) { + this.clientFactory = clientFactory; + this.objectMapper = objectMapper; + } + + // ------------------------------------------------------------------ + // Session state + // ------------------------------------------------------------------ + + /** Terminal-state CAS guard — at most one of {finishCard, failCard} wins per session. */ + enum Status { STREAMING, FINISHED, FAILED } + + /** + * One in-flight streaming card. State is mutated by a single Reactor + * thread per session (the one consuming the {@code Flux}), so all + * mutable fields are either {@code volatile} (visibility across the + * eventual terminal call) or guarded by the session monitor. + */ + static final class CardSession { + final String sessionKey; + final Long channelId; + final String cardId; + final String messageId; + final StringBuilder accumulated = new StringBuilder(); + final AtomicInteger sequence = new AtomicInteger(0); + final AtomicReference status = new AtomicReference<>(Status.STREAMING); + /** + * Time of the most recent flush. Initialised to a value well in + * the past so the very first {@link #appendContent} always + * flushes — the receiver sees the first token instantly instead + * of waiting up to {@link #THROTTLE_INTERVAL_MS} for the second. + */ + volatile long lastFlushMs = Long.MIN_VALUE / 2; + + CardSession(String sessionKey, Long channelId, String cardId, String messageId) { + this.sessionKey = sessionKey; + this.channelId = channelId; + this.cardId = cardId; + this.messageId = messageId; + } + + boolean isStreaming() { + return status.get() == Status.STREAMING; + } + } + + // ------------------------------------------------------------------ + // Public API + // ------------------------------------------------------------------ + + /** + * Build the streaming card, push it as an interactive message, and + * register an in-memory session. + * + * @param channelId mate_channel row id — picks SDK client + * @param receiveIdType one of {@code open_id} / {@code chat_id} / + * {@code email} / {@code union_id} / + * {@code user_id} + * @param receiveId the chat or user id to receive the card + * @param initialText bubble text shown before the first delta; + * null → {@link #DEFAULT_INITIAL_TEXT} + * @return sessionKey for subsequent calls, or null on failure + */ + public String createAndDeliver(Long channelId, String receiveIdType, + String receiveId, String initialText) { + if (channelId == null || receiveIdType == null || receiveId == null) { + log.warn("[feishu-stream] createAndDeliver missing required arg(s)"); + return null; + } + String firstText = (initialText == null || initialText.isBlank()) + ? DEFAULT_INITIAL_TEXT + : initialText; + try { + Client client = clientFactory.client(channelId); + + String cardId = sdkCreateCard(client, firstText); + if (cardId == null) { + return null; + } + String messageId = sdkSendInteractiveMessage(client, receiveIdType, receiveId, cardId); + if (messageId == null) { + // Card built but couldn't deliver — best effort close so the + // server-side card isn't orphaned in streaming mode forever. + tryCloseStreamingSilently(client, cardId); + return null; + } + String sessionKey = UUID.randomUUID().toString(); + CardSession session = new CardSession(sessionKey, channelId, cardId, messageId); + activeSessions.put(sessionKey, session); + log.info("[feishu-stream] Card created: sessionKey={}, cardId={}, messageId={}", + sessionKey, abbrev(cardId), abbrev(messageId)); + return sessionKey; + } catch (Exception e) { + log.error("[feishu-stream] createAndDeliver failed: {}", e.getMessage(), e); + return null; + } + } + + /** + * Append delta text to the running session. May flush immediately + * (force) or wait for the next throttle window. + * + *

No-op when {@code sessionKey} is unknown or the session has + * already reached a terminal status — keeps the caller's + * {@code doOnNext} loop simple ("just push every chunk"). + */ + public void appendContent(String sessionKey, String contentDelta, boolean forceFlush) { + CardSession session = activeSessions.get(sessionKey); + if (session == null || !session.isStreaming()) { + return; + } + if (contentDelta != null && !contentDelta.isEmpty()) { + synchronized (session) { + session.accumulated.append(contentDelta); + } + } + long now = currentTimeMs(); + if (!forceFlush && now - session.lastFlushMs < THROTTLE_INTERVAL_MS) { + return; + } + flush(session, now); + } + + /** + * Push final content and turn off streaming mode. Idempotent — + * a second call is a no-op. After return, the sessionKey is no + * longer known to the manager. + */ + public void finishCard(String sessionKey, String finalContent) { + CardSession session = activeSessions.get(sessionKey); + if (session == null) return; + if (!session.status.compareAndSet(Status.STREAMING, Status.FINISHED)) { + return; + } + try { + replaceAccumulated(session, finalContent != null ? finalContent : ""); + flush(session, currentTimeMs()); + closeStreaming(session); + } finally { + activeSessions.remove(sessionKey); + log.info("[feishu-stream] Card finished: sessionKey={}, contentLen={}", + sessionKey, finalContent == null ? 0 : finalContent.length()); + } + } + + /** + * Mark the session failed. The current accumulator gets an error + * suffix; the card is closed so the typing animation stops. + * Idempotent. + */ + public void failCard(String sessionKey, String errorMessage) { + CardSession session = activeSessions.get(sessionKey); + if (session == null) return; + if (!session.status.compareAndSet(Status.STREAMING, Status.FAILED)) { + return; + } + try { + String tail; + synchronized (session) { + if (session.accumulated.length() == 0) { + tail = "⚠️ 处理失败：" + safe(errorMessage); + } else { + tail = session.accumulated + "\n\n⚠️ " + safe(errorMessage); + } + session.accumulated.setLength(0); + session.accumulated.append(tail); + } + flush(session, currentTimeMs()); + closeStreaming(session); + } finally { + activeSessions.remove(sessionKey); + log.warn("[feishu-stream] Card failed: sessionKey={}, error={}", sessionKey, errorMessage); + } + } + + // ------------------------------------------------------------------ + // Inspection helpers (tests / metrics) + // ------------------------------------------------------------------ + + /** Visible for tests / metrics — number of in-flight sessions. */ + public int activeSessionCount() { + return activeSessions.size(); + } + + /** Visible for tests — direct session lookup. */ + CardSession sessionFor(String sessionKey) { + return activeSessions.get(sessionKey); + } + + // ------------------------------------------------------------------ + // Internal — flush + SDK seams + // ------------------------------------------------------------------ + + private void flush(CardSession session, long now) { + String snapshot; + synchronized (session) { + snapshot = session.accumulated.toString(); + } + int seq = session.sequence.incrementAndGet(); + try { + Client client = clientFactory.client(session.channelId); + sdkPushElementContent(client, session.cardId, STREAM_ELEMENT_ID, snapshot, seq); + session.lastFlushMs = now; + } catch (Exception e) { + log.warn("[feishu-stream] flush failed: sessionKey={}, seq={}, err={}", + session.sessionKey, seq, e.getMessage()); + } + } + + private void closeStreaming(CardSession session) { + int seq = session.sequence.incrementAndGet(); + try { + Client client = clientFactory.client(session.channelId); + sdkCloseStreamingMode(client, session.cardId, seq); + } catch (Exception e) { + log.warn("[feishu-stream] closeStreaming failed: sessionKey={}, err={}", + session.sessionKey, e.getMessage()); + } + } + + private void tryCloseStreamingSilently(Client client, String cardId) { + try { + sdkCloseStreamingMode(client, cardId, 1); + } catch (Exception ignore) { + // best-effort — already in an error path + } + } + + private void replaceAccumulated(CardSession session, String content) { + synchronized (session) { + session.accumulated.setLength(0); + session.accumulated.append(content); + } + } + + // ------------------------------------------------------------------ + // SDK seams (overridable in tests) + // ------------------------------------------------------------------ + + /** Build a streaming-mode schema-2.0 card. Returns the new card_id or null. */ + protected String sdkCreateCard(Client client, String initialText) throws Exception { + String cardJson = objectMapper.writeValueAsString(buildInitialCardJson(initialText)); + CreateCardReq req = CreateCardReq.newBuilder() + .createCardReqBody(CreateCardReqBody.newBuilder() + .type("card_json") + .data(cardJson) + .build()) + .build(); + CreateCardResp resp = client.cardkit().v1().card().create(req); + if (!resp.success() || resp.getData() == null) { + log.warn("[feishu-stream] card.create failed: code={}, msg={}", resp.getCode(), resp.getMsg()); + return null; + } + return resp.getData().getCardId(); + } + + /** Send the freshly-built card as an interactive message. Returns message_id or null. */ + protected String sdkSendInteractiveMessage(Client client, String receiveIdType, + String receiveId, String cardId) throws Exception { + Map content = Map.of( + "type", "card", + "data", Map.of("card_id", cardId) + ); + CreateMessageReq req = CreateMessageReq.newBuilder() + .receiveIdType(receiveIdType) + .createMessageReqBody(CreateMessageReqBody.newBuilder() + .receiveId(receiveId) + .msgType("interactive") + .content(objectMapper.writeValueAsString(content)) + .build()) + .build(); + CreateMessageResp resp = client.im().v1().message().create(req); + if (!resp.success() || resp.getData() == null) { + log.warn("[feishu-stream] interactive message send failed: code={}, msg={}", + resp.getCode(), resp.getMsg()); + return null; + } + return resp.getData().getMessageId(); + } + + /** Push the latest accumulator snapshot to the streaming element. */ + protected void sdkPushElementContent(Client client, String cardId, String elementId, + String content, int sequence) throws Exception { + ContentCardElementReq req = ContentCardElementReq.newBuilder() + .cardId(cardId) + .elementId(elementId) + .contentCardElementReqBody(ContentCardElementReqBody.newBuilder() + .content(content) + .uuid(UUID.randomUUID().toString()) + .sequence(sequence) + .build()) + .build(); + ContentCardElementResp resp = client.cardkit().v1().cardElement().content(req); + if (!resp.success()) { + log.warn("[feishu-stream] cardElement.content failed: cardId={}, seq={}, code={}, msg={}", + abbrev(cardId), sequence, resp.getCode(), resp.getMsg()); + } + } + + /** Flip streaming_mode=false so the receiving UI stops the typing animation. */ + protected void sdkCloseStreamingMode(Client client, String cardId, int sequence) throws Exception { + Map settings = Map.of( + "config", Map.of("streaming_mode", false) + ); + SettingsCardReq req = SettingsCardReq.newBuilder() + .cardId(cardId) + .settingsCardReqBody(SettingsCardReqBody.newBuilder() + .settings(objectMapper.writeValueAsString(settings)) + .uuid(UUID.randomUUID().toString()) + .sequence(sequence) + .build()) + .build(); + SettingsCardResp resp = client.cardkit().v1().card().settings(req); + if (!resp.success()) { + log.warn("[feishu-stream] card.settings (close) failed: cardId={}, code={}, msg={}", + abbrev(cardId), resp.getCode(), resp.getMsg()); + } + } + + // ------------------------------------------------------------------ + // Test seams + tiny helpers + // ------------------------------------------------------------------ + + /** Overridable so tests can pin time without involving Clock + reflection. */ + protected long currentTimeMs() { + return System.currentTimeMillis(); + } + + /** Visible for tests. The "schema 2.0 streaming card" baseline. */ + Map buildInitialCardJson(String initialText) { + // LinkedHashMap → deterministic JSON order, easier to log-grep + Map config = new LinkedHashMap<>(); + config.put("streaming_mode", true); + config.put("update_multi", true); + + Map element = new LinkedHashMap<>(); + element.put("tag", "markdown"); + element.put("element_id", STREAM_ELEMENT_ID); + element.put("content", initialText); + + Map body = new LinkedHashMap<>(); + body.put("elements", List.of(element)); + + Map card = new LinkedHashMap<>(); + card.put("schema", "2.0"); + card.put("config", config); + card.put("body", body); + return card; + } + + private static String abbrev(String s) { + if (s == null || s.length() <= 12) return s == null ? "" : s; + return s.substring(0, 12) + "…"; + } + + private static String safe(String s) { + return s == null ? "" : s; + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java index 32b0102a..60f17469 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/controller/SecurityController.java @@ -17,6 +17,7 @@ import vip.mate.tool.guard.service.ToolGuardConfigService; import vip.mate.tool.guard.service.ToolGuardRuleService; import java.util.HashMap; +import java.util.List; import java.util.Map; import vip.mate.workspace.core.annotation.RequireWorkspaceRole; @@ -166,6 +167,36 @@ public class SecurityController { } } + @Operation(summary = "导出全部规则为 JSON") + @GetMapping("/guard/rules/export") + @RequireWorkspaceRole("admin") + public R> exportRules() { + return R.ok(ruleService.exportRules()); + } + + @Operation(summary = "从 JSON 批量导入规则（upsert 语义）") + @PostMapping("/guard/rules/import") + @RequireWorkspaceRole("admin") + public R> importRules(@RequestBody Map body) { + try { + Object rulesNode = body == null ? null : body.get("rules"); + if (!(rulesNode instanceof List raw)) { + return R.fail("Body must contain a 'rules' array"); + } + com.fasterxml.jackson.databind.ObjectMapper om = new com.fasterxml.jackson.databind.ObjectMapper(); + List incoming = new java.util.ArrayList<>(); + for (Object item : raw) { + ToolGuardRuleEntity rule = om.convertValue(item, ToolGuardRuleEntity.class); + incoming.add(rule); + } + return R.ok(ruleService.importRules(incoming)); + } catch (IllegalArgumentException e) { + return R.fail(e.getMessage()); + } catch (Exception e) { + return R.fail("Import failed: " + e.getMessage()); + } + } + // ==================== Audit ==================== @Operation(summary = "审计日志") diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolGuardRuleRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolGuardRuleRegistry.java index 00de1410..9ca3a412 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolGuardRuleRegistry.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolGuardRuleRegistry.java @@ -66,6 +66,19 @@ public class ToolGuardRuleRegistry implements ApplicationRunner { .collect(Collectors.toList()); } + /** + * 按 category 取所有已启用规则（不限工具）。 + * 用于 alwaysRun 类的横切 Guardian（凭据扫描、PII 扫描等）。 + */ + public List getRulesByCategory(String category) { + if (category == null || category.isEmpty()) { + return List.of(); + } + return allRules.stream() + .filter(r -> category.equals(r.getCategory())) + .collect(Collectors.toList()); + } + /** * 获取所有已启用规则 */ diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolPolicyResolver.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolPolicyResolver.java index 8b3b36af..7c60f349 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolPolicyResolver.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/engine/ToolPolicyResolver.java @@ -5,7 +5,6 @@ import org.springframework.stereotype.Component; import vip.mate.tool.guard.model.*; import java.util.List; -import java.util.Set; /** * 策略解析器 @@ -15,44 +14,66 @@ import java.util.Set; *

  • Guardian 只负责发现风险事实
    • *
  • PolicyResolver 负责把事实映射为执行策略
    • * - * 采用 findings-driven approval 策略，不按工具类型默认审批。 + * 聚合规则（取最严格者）： + *
      + *
    1. 遍历每条 finding，得到该 finding 的目标 decision：
      2. + *
        + *
      • 若 finding 显式带 decision（来自 DB 规则），用之；
        • + *
      • 否则按 severity 回退：CRITICAL→BLOCK，HIGH/MEDIUM→NEEDS_APPROVAL，LOW/INFO→ALLOW。
        • + *
      + *
    2. 取所有 finding 中最严格的：BLOCK > NEEDS_APPROVAL > ALLOW。
      3. + *
    */ @Slf4j @Component public class ToolPolicyResolver { - /** - * 根据 findings 和上下文产出最终裁决 - *

    - * 策略（findings-driven approval）： - *

    - */ public GuardDecision resolve(List findings, ToolInvocationContext context) { - // 无 findings → 直接允许（不再按工具类型默认审批） if (findings == null || findings.isEmpty()) { return GuardDecision.ALLOW; } - GuardSeverity maxSeverity = findings.stream() - .map(GuardFinding::severity) - .reduce(GuardSeverity.INFO, GuardSeverity::max); + GuardDecision aggregate = GuardDecision.ALLOW; + for (GuardFinding f : findings) { + GuardDecision perFinding = resolveSingle(f); + aggregate = stricter(aggregate, perFinding); + if (aggregate == GuardDecision.BLOCK) { + return aggregate; + } + } + return aggregate; + } - // CRITICAL → 直接 BLOCK - if (maxSeverity.isAtLeast(GuardSeverity.CRITICAL)) { + /** + * 单条 finding 的目标 decision：显式优先，否则按 severity 默认映射 + */ + private GuardDecision resolveSingle(GuardFinding finding) { + if (finding.decision() != null) { + return finding.decision(); + } + GuardSeverity sev = finding.severity(); + if (sev == null) { + return GuardDecision.ALLOW; + } + if (sev.isAtLeast(GuardSeverity.CRITICAL)) { return GuardDecision.BLOCK; } - - // HIGH / MEDIUM → 需要审批 - if (maxSeverity.isAtLeast(GuardSeverity.MEDIUM)) { + if (sev.isAtLeast(GuardSeverity.MEDIUM)) { return GuardDecision.NEEDS_APPROVAL; } + return GuardDecision.ALLOW; + } - // LOW / INFO → 允许 + /** + * 取两个 decision 的严格上界：BLOCK > NEEDS_APPROVAL > ALLOW + */ + private GuardDecision stricter(GuardDecision a, GuardDecision b) { + if (a == GuardDecision.BLOCK || b == GuardDecision.BLOCK) { + return GuardDecision.BLOCK; + } + if (a == GuardDecision.NEEDS_APPROVAL || b == GuardDecision.NEEDS_APPROVAL) { + return GuardDecision.NEEDS_APPROVAL; + } return GuardDecision.ALLOW; } @@ -61,14 +82,12 @@ public class ToolPolicyResolver { */ public String buildSummary(List findings, GuardDecision decision) { if (findings == null || findings.isEmpty()) { - // 无 findings 时不应该有 NEEDS_APPROVAL 或 BLOCK return null; } StringBuilder sb = new StringBuilder(); sb.append("检测到 ").append(findings.size()).append(" 项安全风险"); - // 列出最高风险的发现 findings.stream() .filter(f -> f.severity() != null && f.severity().isAtLeast(GuardSeverity.MEDIUM)) .limit(3) diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/CredentialExposureGuardian.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/CredentialExposureGuardian.java index a76cc389..3add1c34 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/CredentialExposureGuardian.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/CredentialExposureGuardian.java @@ -2,6 +2,7 @@ package vip.mate.tool.guard.guardian; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; +import vip.mate.tool.guard.engine.ToolGuardRuleRegistry; import vip.mate.tool.guard.model.*; import java.util.ArrayList; @@ -14,8 +15,13 @@ import java.util.regex.Pattern; /** * 凭据泄露守卫 *

    - * 检测工具参数中可能包含的敏感凭据信息。 - * alwaysRun=true，不受 guarded tools 范围限制。 + * 检测工具参数中可能包含的敏感凭据信息。alwaysRun=true，不受 guarded tools 范围限制。 + *

    + * 规则优先级： + *

      + *
    1. 优先读取 DB 中 category=CREDENTIAL_EXPOSURE 的已启用规则（受 UI 开关控制）；
      2. + *
    2. DB 无任何凭据规则时回退到内置硬编码列表（保证未初始化部署也能工作）。
      3. + *
    */ @Slf4j @Component @@ -23,31 +29,43 @@ public class CredentialExposureGuardian implements ToolGuardGuardian { private static final Map COMPILED = new ConcurrentHashMap<>(); - private record CredentialRule(String ruleId, String pattern, String title, String description) {} + private record CredentialRule(String ruleId, String pattern, String title, + String description, GuardDecision decision) {} - private static final List RULES = List.of( + private static final List BUILTIN_FALLBACK = List.of( new CredentialRule("CRED_PASSWORD_ASSIGN", "(password|secret|api[_-]?key|token)\\s*=\\s*['\"]?\\S{8,}", "凭据信息暴露", - "检测到可能的密码/密钥/Token 赋值"), + "检测到可能的密码/密钥/Token 赋值", + GuardDecision.NEEDS_APPROVAL), new CredentialRule("CRED_AWS_KEY", "AKIA[0-9A-Z]{16}", "AWS Access Key 泄露", - "检测到 AWS Access Key ID 模式"), + "检测到 AWS Access Key ID 模式", + GuardDecision.NEEDS_APPROVAL), new CredentialRule("CRED_PRIVATE_KEY", "-----BEGIN\\s+(RSA\\s+)?PRIVATE\\s+KEY-----", "私钥泄露", - "检测到 PEM 格式私钥"), + "检测到 PEM 格式私钥", + GuardDecision.BLOCK), new CredentialRule("CRED_JWT_TOKEN", "eyJ[A-Za-z0-9_-]{10,}\\.eyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]+", "JWT Token 泄露", - "检测到 JWT Token 格式的字符串"), + "检测到 JWT Token 格式的字符串", + GuardDecision.NEEDS_APPROVAL), new CredentialRule("CRED_GITHUB_TOKEN", "gh[pousr]_[A-Za-z0-9_]{36,}", "GitHub Token 泄露", - "检测到 GitHub Personal Access Token") + "检测到 GitHub Personal Access Token", + GuardDecision.NEEDS_APPROVAL) ); + private final ToolGuardRuleRegistry ruleRegistry; + + public CredentialExposureGuardian(ToolGuardRuleRegistry ruleRegistry) { + this.ruleRegistry = ruleRegistry; + } + @Override public boolean supports(ToolInvocationContext context) { return true; @@ -69,7 +87,45 @@ public class CredentialExposureGuardian implements ToolGuardGuardian { if (raw == null || raw.isEmpty()) return List.of(); List findings = new ArrayList<>(); - for (CredentialRule rule : RULES) { + + // 1) 优先使用 DB 规则（受 UI 启用/禁用开关控制） + List dbRules = ruleRegistry.getRulesByCategory( + GuardCategory.CREDENTIAL_EXPOSURE.name()); + if (!dbRules.isEmpty()) { + for (ToolGuardRuleEntity rule : dbRules) { + if (rule.getPattern() == null || rule.getPattern().isBlank()) continue; + Pattern p = ruleRegistry.getCompiledPattern(rule.getPattern()); + Matcher matcher = p.matcher(raw); + if (!matcher.find()) continue; + + // 排除模式（白名单） + if (rule.getExcludePattern() != null && !rule.getExcludePattern().isBlank()) { + Pattern exclude = ruleRegistry.getCompiledExcludePattern(rule.getExcludePattern()); + if (exclude.matcher(raw).find()) continue; + } + + String snippet = extractSnippet(raw, matcher.start(), 30); + GuardSeverity severity = parseSeverity(rule.getSeverity()); + GuardDecision decision = parseDecision(rule.getDecision()); + findings.add(new GuardFinding( + rule.getRuleId(), + severity, + GuardCategory.CREDENTIAL_EXPOSURE, + rule.getName(), + rule.getDescription(), + rule.getRemediation(), + context.toolName(), + rule.getParamName(), + rule.getPattern(), + maskCredential(snippet), + decision + )); + } + return findings; + } + + // 2) DB 未初始化 → 回退内置规则 + for (CredentialRule rule : BUILTIN_FALLBACK) { Pattern p = COMPILED.computeIfAbsent(rule.pattern, r -> Pattern.compile(r, Pattern.CASE_INSENSITIVE)); Matcher matcher = p.matcher(raw); @@ -85,13 +141,32 @@ public class CredentialExposureGuardian implements ToolGuardGuardian { context.toolName(), null, rule.pattern, - maskCredential(snippet) + maskCredential(snippet), + rule.decision )); } } return findings; } + private GuardSeverity parseSeverity(String raw) { + if (raw == null || raw.isBlank()) return GuardSeverity.HIGH; + try { + return GuardSeverity.valueOf(raw); + } catch (IllegalArgumentException e) { + return GuardSeverity.HIGH; + } + } + + private GuardDecision parseDecision(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return GuardDecision.valueOf(raw); + } catch (IllegalArgumentException e) { + return null; + } + } + private String extractSnippet(String input, int matchStart, int contextLen) { int start = Math.max(0, matchStart - contextLen / 2); int end = Math.min(input.length(), matchStart + contextLen / 2); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/ShellCommandGuardian.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/ShellCommandGuardian.java index 21501140..1276765c 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/ShellCommandGuardian.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/guardian/ShellCommandGuardian.java @@ -79,7 +79,8 @@ public class ShellCommandGuardian implements ToolGuardGuardian { context.toolName(), rule.getParamName() != null ? rule.getParamName() : "command", rule.getPattern(), - snippet + snippet, + parseDecision(rule.getDecision()) )); } } @@ -115,6 +116,15 @@ public class ShellCommandGuardian implements ToolGuardGuardian { return (context.toolName() != null ? context.toolName() + " " : "") + raw; } + private GuardDecision parseDecision(String raw) { + if (raw == null || raw.isBlank()) return null; + try { + return GuardDecision.valueOf(raw); + } catch (IllegalArgumentException e) { + return null; + } + } + private String extractSnippet(String input, int matchStart, int contextLen) { int start = Math.max(0, matchStart - contextLen / 2); int end = Math.min(input.length(), matchStart + contextLen / 2); diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/model/GuardFinding.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/model/GuardFinding.java index f4c9e4b3..17ea1368 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/model/GuardFinding.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/model/GuardFinding.java @@ -7,6 +7,10 @@ import java.util.Map; *

    * 由 Guardian 评估产出，携带完整的威胁上下文信息。 * 使用不可变 record，产出后不允许被修改。 + *

    + * {@code decision} 为可选的"该规则希望的最终裁决"。Guardian 若从 DB 加载规则， + * 应把 rule.decision 透传过来；PolicyResolver 在聚合阶段会取所有 findings 中 + * 最严格的一项作为最终 decision，未设置时回退到 severity → 默认动作。 */ public record GuardFinding( String ruleId, @@ -19,6 +23,7 @@ public record GuardFinding( String paramName, String matchedPattern, String snippet, + GuardDecision decision, Map metadata ) { @@ -26,7 +31,15 @@ public record GuardFinding( String title, String description, String remediation, String toolName, String paramName, String matchedPattern, String snippet) { this(ruleId, severity, category, title, description, remediation, - toolName, paramName, matchedPattern, snippet, Map.of()); + toolName, paramName, matchedPattern, snippet, null, Map.of()); + } + + public GuardFinding(String ruleId, GuardSeverity severity, GuardCategory category, + String title, String description, String remediation, + String toolName, String paramName, String matchedPattern, String snippet, + GuardDecision decision) { + this(ruleId, severity, category, title, description, remediation, + toolName, paramName, matchedPattern, snippet, decision, Map.of()); } /** @@ -43,7 +56,8 @@ public record GuardFinding( Map.entry("toolName", toolName != null ? toolName : ""), Map.entry("paramName", paramName != null ? paramName : ""), Map.entry("matchedPattern", matchedPattern != null ? matchedPattern : ""), - Map.entry("snippet", snippet != null ? snippet : "") + Map.entry("snippet", snippet != null ? snippet : ""), + Map.entry("decision", decision != null ? decision.name() : "") ); } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleSeedService.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleSeedService.java index 4a488710..0a366d3b 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleSeedService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleSeedService.java @@ -169,19 +169,18 @@ public class ToolGuardRuleSeedService implements ApplicationRunner { log.debug("[RuleSeed] Rule {} insert failed: {}", rule.getRuleId(), e.getMessage()); } } else if (needsUpdate(existing, rule)) { - // 已存在但内容有变化 → 更新 + // 已存在但内容字段有变化 → 同步代码侧拥有的字段（content fields）。 + // 严格保留用户侧拥有的策略字段（severity / decision / priority / enabled + // / excludePattern）—— 这些一旦用户在 UI 上调整，重启后不应被覆盖。 ruleMapper.update(null, new LambdaUpdateWrapper() .eq(ToolGuardRuleEntity::getRuleId, rule.getRuleId()) .set(ToolGuardRuleEntity::getName, rule.getName()) .set(ToolGuardRuleEntity::getDescription, rule.getDescription()) .set(ToolGuardRuleEntity::getPattern, rule.getPattern()) - .set(ToolGuardRuleEntity::getSeverity, rule.getSeverity()) .set(ToolGuardRuleEntity::getCategory, rule.getCategory()) - .set(ToolGuardRuleEntity::getDecision, rule.getDecision()) .set(ToolGuardRuleEntity::getToolName, rule.getToolName()) - .set(ToolGuardRuleEntity::getRemediation, rule.getRemediation()) - .set(ToolGuardRuleEntity::getPriority, rule.getPriority())); + .set(ToolGuardRuleEntity::getRemediation, rule.getRemediation())); updated++; } else { unchanged++; @@ -195,16 +194,21 @@ public class ToolGuardRuleSeedService implements ApplicationRunner { } /** - * 判断已有 builtin 规则是否需要更新（任一核心字段有变化即需要） + * 判断已有 builtin 规则是否需要更新。 + *

    + * 只比较"内容字段"（代码侧拥有，应当随版本升级同步）： + * name / description / pattern / category / toolName / remediation。 + *

    + * 故意不比较"策略字段"（用户侧拥有，UI 可调）：severity / decision / priority / enabled / excludePattern。 + * 这样用户把某条 builtin 规则的 decision 从 NEEDS_APPROVAL 改成 BLOCK、或者关闭某条规则， + * 重启不会把改动覆盖回种子初值。 */ private boolean needsUpdate(ToolGuardRuleEntity existing, ToolGuardRuleEntity expected) { - return !Objects.equals(existing.getPattern(), expected.getPattern()) - || !Objects.equals(existing.getSeverity(), expected.getSeverity()) + return !Objects.equals(existing.getName(), expected.getName()) + || !Objects.equals(existing.getDescription(), expected.getDescription()) + || !Objects.equals(existing.getPattern(), expected.getPattern()) || !Objects.equals(existing.getCategory(), expected.getCategory()) - || !Objects.equals(existing.getDecision(), expected.getDecision()) || !Objects.equals(existing.getToolName(), expected.getToolName()) - || !Objects.equals(existing.getPriority(), expected.getPriority()) - || !Objects.equals(existing.getName(), expected.getName()) || !Objects.equals(existing.getRemediation(), expected.getRemediation()); } @@ -330,6 +334,16 @@ public class ToolGuardRuleSeedService implements ApplicationRunner { GuardSeverity.HIGH, GuardCategory.CREDENTIAL_EXPOSURE, "BLOCK", null, gf("CRED_PRIVATE_KEY"), 140)); + rules.add(rule("CRED_JWT_TOKEN", gn("CRED_JWT_TOKEN"), + "eyJ[A-Za-z0-9_-]{10,}\\.eyJ[A-Za-z0-9_-]{10,}\\.[A-Za-z0-9_-]+", + GuardSeverity.HIGH, GuardCategory.CREDENTIAL_EXPOSURE, "NEEDS_APPROVAL", + null, gf("CRED_JWT_TOKEN"), 140)); + + rules.add(rule("CRED_GITHUB_TOKEN", gn("CRED_GITHUB_TOKEN"), + "gh[pousr]_[A-Za-z0-9_]{36,}", + GuardSeverity.HIGH, GuardCategory.CREDENTIAL_EXPOSURE, "NEEDS_APPROVAL", + null, gf("CRED_GITHUB_TOKEN"), 140)); + return rules; } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java index 66ab98b4..aa024d24 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/guard/service/ToolGuardRuleService.java @@ -10,6 +10,11 @@ import vip.mate.tool.guard.engine.ToolGuardRuleRegistry; import vip.mate.tool.guard.model.ToolGuardRuleEntity; import vip.mate.tool.guard.repository.ToolGuardRuleMapper; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + /** * 工具安全规则 CRUD 服务 */ @@ -94,6 +99,11 @@ public class ToolGuardRuleService { if (existing == null) { throw new IllegalArgumentException("Rule not found: " + ruleId); } + // 内置规则只允许调整策略字段；内容字段（pattern / category / name 等）由 + // 代码侧种子管理，写进 DB 也会在下次启动被覆盖回去，提前在这里拦掉以免误用。 + if (Boolean.TRUE.equals(existing.getBuiltin())) { + return updateBuiltinPolicy(ruleId, update); + } if (update.getName() != null) { requireNonBlank(update.getName(), "Rule name"); @@ -153,6 +163,130 @@ public class ToolGuardRuleService { ruleRegistry.reload(); } + /** + * 更新内置规则时，限制只允许调整策略字段（severity / decision / priority / enabled + * / excludePattern）。内容字段（pattern / category / name 等）由代码侧种子管理， + * UI 改动也会在下次重启被覆盖回去，提前在 API 层拦截避免误用。 + */ + public ToolGuardRuleEntity updateBuiltinPolicy(String ruleId, ToolGuardRuleEntity patch) { + ToolGuardRuleEntity existing = getByRuleId(ruleId); + if (existing == null) { + throw new IllegalArgumentException("Rule not found: " + ruleId); + } + if (!Boolean.TRUE.equals(existing.getBuiltin())) { + throw new IllegalArgumentException("Rule is not builtin: " + ruleId); + } + if (patch.getSeverity() != null) existing.setSeverity(patch.getSeverity()); + if (patch.getDecision() != null) existing.setDecision(patch.getDecision()); + if (patch.getPriority() != null) existing.setPriority(patch.getPriority()); + if (patch.getEnabled() != null) existing.setEnabled(patch.getEnabled()); + if (patch.getExcludePattern() != null) existing.setExcludePattern(patch.getExcludePattern()); + ruleMapper.updateById(existing); + ruleRegistry.reload(); + return existing; + } + + /** + * 导出全部规则（含 builtin），格式可被 importRules 直接吃回去。 + * 导出时保留 ruleId 作为主键标识，省略 id / createTime / updateTime / deleted 这些 + * 部署敏感的字段；builtin 标志保留，import 时用来判断走 builtin policy 通道还是 + * 创建/覆盖 custom 规则。 + */ + public Map exportRules() { + List all = ruleMapper.selectList( + new LambdaQueryWrapper() + .orderByDesc(ToolGuardRuleEntity::getPriority)); + List> rows = new ArrayList<>(); + for (ToolGuardRuleEntity r : all) { + Map row = new LinkedHashMap<>(); + row.put("ruleId", r.getRuleId()); + row.put("name", r.getName()); + row.put("description", r.getDescription()); + row.put("toolName", r.getToolName()); + row.put("paramName", r.getParamName()); + row.put("category", r.getCategory()); + row.put("severity", r.getSeverity()); + row.put("decision", r.getDecision()); + row.put("pattern", r.getPattern()); + row.put("excludePattern", r.getExcludePattern()); + row.put("remediation", r.getRemediation()); + row.put("priority", r.getPriority()); + row.put("enabled", r.getEnabled()); + row.put("builtin", r.getBuiltin()); + rows.add(row); + } + Map envelope = new LinkedHashMap<>(); + envelope.put("schema", "mateclaw.tool-guard.rules.v1"); + envelope.put("exportedAt", java.time.OffsetDateTime.now().toString()); + envelope.put("count", rows.size()); + envelope.put("rules", rows); + return envelope; + } + + /** + * 导入规则。upsert 语义： + *

      + *
    • ruleId 已存在 + builtin → 仅同步策略字段（severity / decision / priority / enabled / excludePattern）；
      • + *
    • ruleId 已存在 + custom → 全字段覆盖；
      • + *
    • ruleId 不存在 → 作为 custom 规则插入（强制 builtin=false，避免被 import 篡改内置标记）。
      • + *
    + */ + public Map importRules(List incoming) { + if (incoming == null || incoming.isEmpty()) { + throw new IllegalArgumentException("No rules to import"); + } + int inserted = 0; + int updatedBuiltin = 0; + int updatedCustom = 0; + int skipped = 0; + List errors = new ArrayList<>(); + + for (ToolGuardRuleEntity rule : incoming) { + try { + if (rule.getRuleId() == null || rule.getRuleId().isBlank()) { + skipped++; + errors.add("missing ruleId"); + continue; + } + if (rule.getPattern() == null || rule.getPattern().isBlank()) { + skipped++; + errors.add(rule.getRuleId() + ": missing pattern"); + continue; + } + String rid = rule.getRuleId().trim(); + ToolGuardRuleEntity existing = getByRuleId(rid); + if (existing == null) { + rule.setRuleId(rid); + rule.setBuiltin(false); + if (rule.getEnabled() == null) rule.setEnabled(true); + if (rule.getPriority() == null) rule.setPriority(100); + if (rule.getSeverity() == null) rule.setSeverity("HIGH"); + if (rule.getDecision() == null) rule.setDecision("NEEDS_APPROVAL"); + ruleMapper.insert(rule); + inserted++; + } else if (Boolean.TRUE.equals(existing.getBuiltin())) { + updateBuiltinPolicy(rid, rule); + updatedBuiltin++; + } else { + updateRule(rid, rule); + updatedCustom++; + } + } catch (Exception e) { + skipped++; + errors.add((rule.getRuleId() == null ? "" : rule.getRuleId()) + + ": " + e.getMessage()); + } + } + ruleRegistry.reload(); + Map summary = new LinkedHashMap<>(); + summary.put("inserted", inserted); + summary.put("updatedBuiltin", updatedBuiltin); + summary.put("updatedCustom", updatedCustom); + summary.put("skipped", skipped); + summary.put("errors", errors); + return summary; + } + /** * 按主键 ID 删除自定义规则。兜底通道：当 rule_id 因历史脏数据为空或无法走 * /guard/rules/{ruleId} 路径变量时，UI 仍可通过主键删除。 diff --git a/mateclaw-server/src/main/resources/messages.properties b/mateclaw-server/src/main/resources/messages.properties index 6896a568..a02c46f7 100644 --- a/mateclaw-server/src/main/resources/messages.properties +++ b/mateclaw-server/src/main/resources/messages.properties @@ -154,6 +154,10 @@ guard.CRED_AWS_KEY.name=AWS Access Key \u6cc4\u9732 guard.CRED_AWS_KEY.fix=\u4f7f\u7528 IAM Role \u6216 AWS Secrets Manager guard.CRED_PRIVATE_KEY.name=\u79c1\u94a5\u6cc4\u9732 guard.CRED_PRIVATE_KEY.fix=\u8bf7\u52ff\u5728\u53c2\u6570\u4e2d\u4f20\u9012\u79c1\u94a5 +guard.CRED_JWT_TOKEN.name=JWT Token \u6cc4\u9732 +guard.CRED_JWT_TOKEN.fix=\u8bf7\u4f7f\u7528\u73af\u5883\u53d8\u91cf\u6216\u5bc6\u94a5\u7ba1\u7406\u670d\u52a1\uff0c\u907f\u514d\u660e\u6587\u4f20\u9012 JWT +guard.CRED_GITHUB_TOKEN.name=GitHub Token \u6cc4\u9732 +guard.CRED_GITHUB_TOKEN.fix=\u8bf7\u7acb\u5373\u5728 GitHub \u540e\u53f0\u64a4\u9500\u8be5 Token\uff0c\u6539\u7528\u73af\u5883\u53d8\u91cf # --- Exception Messages (structured keys) --- err.auth.invalid_credentials=\u7528\u6237\u540d\u6216\u5bc6\u7801\u9519\u8bef diff --git a/mateclaw-server/src/main/resources/messages_en.properties b/mateclaw-server/src/main/resources/messages_en.properties index a792c4a0..8aca1cf0 100644 --- a/mateclaw-server/src/main/resources/messages_en.properties +++ b/mateclaw-server/src/main/resources/messages_en.properties @@ -154,6 +154,10 @@ guard.CRED_AWS_KEY.name=AWS Access Key leak guard.CRED_AWS_KEY.fix=Use IAM Role or AWS Secrets Manager guard.CRED_PRIVATE_KEY.name=Private key leak guard.CRED_PRIVATE_KEY.fix=Do not pass private keys in parameters +guard.CRED_JWT_TOKEN.name=JWT token leak +guard.CRED_JWT_TOKEN.fix=Use environment variables or secret manager; avoid passing JWTs in plaintext +guard.CRED_GITHUB_TOKEN.name=GitHub token leak +guard.CRED_GITHUB_TOKEN.fix=Revoke the token in GitHub immediately and switch to environment variables # --- WorkspacePathGuard --- guard.path.not_allowed=Path is outside workspace boundary: {0}, allowed root: {1} diff --git a/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java b/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java index bc2b4e49..244369c4 100644 --- a/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java +++ b/mateclaw-server/src/test/java/vip/mate/channel/ChannelManagerReconcileTest.java @@ -58,6 +58,7 @@ class ChannelManagerReconcileTest { mock(vip.mate.channel.wecom.WeComKeepaliveScheduler.class), mock(vip.mate.channel.feishu.FeishuMediaUploader.class), mock(vip.mate.channel.media.GeneratedFileScrubber.class), + mock(vip.mate.channel.feishu.FeishuStreamingCardManager.class), election); adapter = new TrackingAdapter(); } diff --git a/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuMediaWiringIT.java b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuMediaWiringIT.java index 706db7d5..8aa9e8ca 100644 --- a/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuMediaWiringIT.java +++ b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuMediaWiringIT.java @@ -33,6 +33,7 @@ class FeishuMediaWiringIT { @Autowired private FeishuMediaUploader mediaUploader; @Autowired private FeishuSizePolicy sizePolicy; @Autowired private GeneratedFileScrubber scrubber; + @Autowired private FeishuStreamingCardManager streamingCardManager; @Autowired private List uploaderBeans; @Autowired private List policyBeans; @@ -43,6 +44,7 @@ class FeishuMediaWiringIT { assertNotNull(mediaUploader); assertNotNull(sizePolicy); assertNotNull(scrubber); + assertNotNull(streamingCardManager); } @Test diff --git a/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuStreamingCardManagerTest.java b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuStreamingCardManagerTest.java new file mode 100644 index 00000000..ee8d206e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/channel/feishu/FeishuStreamingCardManagerTest.java @@ -0,0 +1,256 @@ +package vip.mate.channel.feishu; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.lark.oapi.Client; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Pin the FeishuStreamingCardManager state machine + throttle. + * + *

    SDK calls are stubbed via {@code protected} seams so this test + * stays Spring-free and network-free — we verify ordering and + * lifecycle, not Feishu API shape (the IT covers wiring). + * + *

    Behaviour pinned: + *

      + *
    • throttle window suppresses sub-window flushes; forceFlush + * bypasses it; finishCard always flushes
      • + *
    • session is removed from {@code activeSessions} on terminal + * transition; subsequent appends are no-ops
      • + *
    • finish vs fail is a CAS-guarded one-shot — second terminal + * call is silent and does not double-close streaming
      • + *
    • sequence numbers are monotonic across content + close
      • + *
    • initial card JSON carries schema 2.0 + streaming_mode + the + * agreed element id
      • + *
    + */ +class FeishuStreamingCardManagerTest { + + /** Recording SDK seam — captures each call so the test can replay them. */ + private static final class RecordingManager extends FeishuStreamingCardManager { + record ContentCall(String cardId, String elementId, String content, int sequence) {} + record CloseCall(String cardId, int sequence) {} + + final List contentCalls = new java.util.concurrent.CopyOnWriteArrayList<>(); + final List closeCalls = new java.util.concurrent.CopyOnWriteArrayList<>(); + final AtomicLong fakeNowMs = new AtomicLong(0); + final AtomicReference nextCardId = new AtomicReference<>("card_abc"); + final AtomicReference nextMessageId = new AtomicReference<>("msg_abc"); + + RecordingManager(FeishuClientFactory factory, ObjectMapper objectMapper) { + super(factory, objectMapper); + } + + @Override protected long currentTimeMs() { return fakeNowMs.get(); } + + @Override protected String sdkCreateCard(Client client, String initialText) { return nextCardId.get(); } + + @Override protected String sdkSendInteractiveMessage(Client client, String receiveIdType, + String receiveId, String cardId) { + return nextMessageId.get(); + } + + @Override protected void sdkPushElementContent(Client client, String cardId, String elementId, + String content, int sequence) { + contentCalls.add(new ContentCall(cardId, elementId, content, sequence)); + } + + @Override protected void sdkCloseStreamingMode(Client client, String cardId, int sequence) { + closeCalls.add(new CloseCall(cardId, sequence)); + } + } + + private RecordingManager manager; + + @BeforeEach + void setUp() { + FeishuClientFactory factory = mock(FeishuClientFactory.class); + when(factory.client(anyLong())).thenReturn(mock(Client.class)); + // Mockito.any() also returns mock Client for boxed Long lookups + when(factory.client(any())).thenReturn(mock(Client.class)); + manager = new RecordingManager(factory, new ObjectMapper()); + } + + @Test + @DisplayName("createAndDeliver registers a session keyed by UUID") + void createAndDeliverRegistersSession() { + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + assertNotNull(key); + assertEquals(1, manager.activeSessionCount()); + assertNotNull(manager.sessionFor(key)); + } + + @Test + @DisplayName("createAndDeliver returns null when card creation fails") + void createReturnsNullOnFailure() { + manager.nextCardId.set(null); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + assertNull(key); + assertEquals(0, manager.activeSessionCount()); + } + + @Test + @DisplayName("the very first append flushes immediately (no prior flush gates)") + void firstAppendFlushesImmediately() { + manager.fakeNowMs.set(1000L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + + manager.fakeNowMs.set(1010L); // only 10ms later — still flushes because no prior flush + manager.appendContent(key, "Hel", false); + + assertEquals(1, manager.contentCalls.size(), + "First-ever append should flush immediately so the user sees an instant first token"); + assertEquals("Hel", manager.contentCalls.get(0).content()); + } + + @Test + @DisplayName("subsequent appends inside the throttle window are suppressed and accumulated") + void throttleSuppressesPostFirstFlushAppends() { + manager.fakeNowMs.set(1000L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + + // First — flushes immediately (seq 1) and sets lastFlushMs=1010 + manager.fakeNowMs.set(1010L); + manager.appendContent(key, "first ", false); + assertEquals(1, manager.contentCalls.size()); + + // Inside 500ms window → suppressed, just accumulated + manager.fakeNowMs.set(1100L); + manager.appendContent(key, "mid ", false); + manager.fakeNowMs.set(1400L); + manager.appendContent(key, "more ", false); + assertEquals(1, manager.contentCalls.size(), + "Mid-window appends should NOT trigger SDK calls"); + + // Past 500ms window → flushes the full accumulator (seq 2) + manager.fakeNowMs.set(1600L); + manager.appendContent(key, "end", false); + assertEquals(2, manager.contentCalls.size()); + assertEquals("first mid more end", manager.contentCalls.get(1).content()); + assertEquals(2, manager.contentCalls.get(1).sequence()); + } + + @Test + @DisplayName("forceFlush bypasses the throttle even inside the window") + void forceFlushBypassesThrottle() { + manager.fakeNowMs.set(0L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + + // First append flushes regardless (lastFlushMs=0) + manager.appendContent(key, "a", false); + assertEquals(1, manager.contentCalls.size()); + + // 50ms later (inside 500ms window) — without force this would be suppressed + manager.fakeNowMs.set(50L); + manager.appendContent(key, "b", true); + assertEquals(2, manager.contentCalls.size(), "forceFlush must bypass throttle"); + assertEquals("ab", manager.contentCalls.get(1).content()); + } + + @Test + @DisplayName("finishCard emits final content + close, in monotonic sequence order, then removes session") + void finishCardClosesAndUnregisters() { + manager.fakeNowMs.set(0L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + + manager.appendContent(key, "Hello, ", true); // seq 1 + manager.fakeNowMs.set(600L); + manager.appendContent(key, "world", false); // seq 2 + manager.finishCard(key, "Hello, world!"); // seq 3 (content) + seq 4 (close) + + assertEquals(3, manager.contentCalls.size()); + assertEquals("Hello, world!", manager.contentCalls.get(2).content()); + // Sequence is monotonic across all content + close calls + assertEquals(1, manager.contentCalls.get(0).sequence()); + assertEquals(2, manager.contentCalls.get(1).sequence()); + assertEquals(3, manager.contentCalls.get(2).sequence()); + assertEquals(1, manager.closeCalls.size()); + assertEquals(4, manager.closeCalls.get(0).sequence()); + assertEquals(0, manager.activeSessionCount()); + } + + @Test + @DisplayName("appendContent after finish is a no-op (no SDK call, no resurrection)") + void appendAfterFinishIsNoop() { + manager.fakeNowMs.set(0L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + manager.finishCard(key, "done"); + + int before = manager.contentCalls.size(); + manager.fakeNowMs.set(10_000L); + manager.appendContent(key, "ghost delta", true); + assertEquals(before, manager.contentCalls.size(), + "Append after terminal status must not produce another SDK call"); + } + + @Test + @DisplayName("failCard appends error suffix, closes, and ignores second terminal call") + void failCardAppendsTailAndIsOneShot() { + manager.fakeNowMs.set(0L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + manager.appendContent(key, "Partial reply", true); + + manager.failCard(key, "rate limited"); + + // last content push carries the failure suffix + assertTrue(manager.contentCalls.get(manager.contentCalls.size() - 1).content().contains("rate limited")); + assertEquals(1, manager.closeCalls.size()); + + // A subsequent finishCard must NOT trigger another close + manager.finishCard(key, "ignored"); + assertEquals(1, manager.closeCalls.size(), "Second terminal call must be ignored"); + } + + @Test + @DisplayName("failCard with no accumulator emits a stand-alone error message") + void failCardWithEmptyAccumulator() { + manager.fakeNowMs.set(0L); + String key = manager.createAndDeliver(7L, "open_id", "ou_abc", null); + + manager.failCard(key, "network down"); + assertEquals(1, manager.contentCalls.size()); + assertTrue(manager.contentCalls.get(0).content().contains("network down")); + } + + @Test + @DisplayName("initial card JSON declares schema 2.0 + streaming_mode + the stream element id") + @SuppressWarnings("unchecked") + void initialCardJsonShape() { + Map card = manager.buildInitialCardJson("test"); + assertEquals("2.0", card.get("schema")); + Map config = (Map) card.get("config"); + assertEquals(Boolean.TRUE, config.get("streaming_mode")); + Map body = (Map) card.get("body"); + List> elements = (List>) body.get("elements"); + assertEquals(FeishuStreamingCardManager.STREAM_ELEMENT_ID, elements.get(0).get("element_id")); + assertEquals("markdown", elements.get(0).get("tag")); + } + + @Test + @DisplayName("unknown session key is a silent no-op on every public method") + void unknownSessionIsNoop() { + manager.appendContent("never-existed", "x", true); + manager.finishCard("never-existed", "x"); + manager.failCard("never-existed", "x"); + assertEquals(0, manager.contentCalls.size()); + assertEquals(0, manager.closeCalls.size()); + assertFalse(false); // assertion just to make junit happy with no real check + } +} diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 6317968d..3f999d97 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -671,6 +671,8 @@ export const securityApi = { toggleRule: (ruleId: string, enabled: boolean) => http.put(`/security/guard/rules/${ruleId}/toggle?enabled=${enabled}`), deleteRule: (ruleId: string) => http.delete(`/security/guard/rules/${ruleId}`), + exportRules: () => http.get('/security/guard/rules/export'), + importRules: (data: { rules: any[] }) => http.post('/security/guard/rules/import', data), listAuditLogs: (params?: any) => http.get('/security/audit/logs', { params }), getAuditStats: () => http.get('/security/audit/stats'), listApprovals: (params?: any) => http.get('/security/approvals', { params }), diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 27f086e8..e7c33964 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -1457,6 +1457,13 @@ export default { ruleIdDuplicate: 'Rule ID already exists', saveFailed: 'Failed to save rule', }, + exportBtn: 'Export JSON', + importBtn: 'Import JSON', + exportFileName: 'mateclaw-guard-rules-{date}.json', + importPromptHint: 'Upload a JSON exported by this UI. Built-in rules only sync their policy fields.', + importSummary: 'Import done: {inserted} added, {updatedBuiltin} built-in updated, {updatedCustom} custom updated, {skipped} skipped', + importFailed: 'Import failed: {msg}', + builtinLockedHint: 'Built-in rule name, pattern and category are system-managed. You can only adjust severity, decision, priority, exclude pattern and enabled state.', }, fileGuard: { title: 'File Guard', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 70012927..ac0a40aa 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -1349,6 +1349,13 @@ export default { ruleIdDuplicate: '规则 ID 已存在', saveFailed: '保存规则失败', }, + exportBtn: '导出 JSON', + importBtn: '导入 JSON', + exportFileName: 'mateclaw-guard-rules-{date}.json', + importPromptHint: '上传通过"导出"产生的 JSON 文件；内置规则只会同步策略字段。', + importSummary: '导入完成：新增 {inserted}，更新内置 {updatedBuiltin}，更新自定义 {updatedCustom}，跳过 {skipped}', + importFailed: '导入失败：{msg}', + builtinLockedHint: '内置规则的名称、模式与分类由系统管理，此处仅可调整严重度、决策、优先级、白名单与启用状态。', }, fileGuard: { title: '文件防护', diff --git a/mateclaw-ui/src/views/Security/ToolGuard/index.vue b/mateclaw-ui/src/views/Security/ToolGuard/index.vue index c2af53d9..a7b9006a 100644 --- a/mateclaw-ui/src/views/Security/ToolGuard/index.vue +++ b/mateclaw-ui/src/views/Security/ToolGuard/index.vue @@ -5,9 +5,24 @@

    {{ t('security.toolGuard.title') }}

    {{ t('security.toolGuard.desc') }}

    - +
    + + + + +
    @@ -144,6 +159,9 @@
    +
    + {{ t('security.toolGuard.builtinLockedHint') }} +
    @@ -156,11 +174,11 @@
    - +
    - +
    @@ -174,7 +192,7 @@
    - @@ -195,16 +213,20 @@
    - +
    - +
    +
    + + +
    @@ -218,7 +240,7 @@