diff --git a/docker-compose.yml b/docker-compose.yml index f304fe6e..47735131 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,23 @@ services: timeout: 5s retries: 5 + # SearXNG 搜索引擎(keyless 搜索 provider,零配置可用) + searxng: + image: searxng/searxng:latest + container_name: mateclaw-searxng + restart: unless-stopped + environment: + - SEARXNG_BASE_URL=http://searxng:8080 + volumes: + - searxng_data:/etc/searxng + ports: + - "8088:8080" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"] + interval: 30s + timeout: 5s + retries: 3 + # MateClaw 后端服务 mateclaw-server: build: @@ -35,6 +52,8 @@ services: depends_on: mysql: condition: service_healthy + searxng: + condition: service_healthy environment: SPRING_PROFILES_ACTIVE: mysql DB_HOST: mysql @@ -52,3 +71,4 @@ services: volumes: mysql_data: server_data: + searxng_data: diff --git a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java index f302a9b8..e4f8a594 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java +++ b/mateclaw-server/src/main/java/vip/mate/system/model/SystemSettingsDTO.java @@ -24,6 +24,12 @@ public class SystemSettingsDTO { private String tavilyApiKey; private String tavilyBaseUrl; + // ===== Keyless 搜索 provider 配置 ===== + /** DuckDuckGo 是否启用(默认 true,作为零配置兜底) */ + private Boolean duckduckgoEnabled; + /** SearXNG 实例地址(如 http://searxng:8080),为空则不使用 */ + private String searxngBaseUrl; + // 用于前端回显脱敏后的 API Key private String serperApiKeyMasked; private String tavilyApiKeyMasked; diff --git a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java index f3a1b199..d2ae1602 100644 --- a/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java +++ b/mateclaw-server/src/main/java/vip/mate/system/service/SystemSettingService.java @@ -24,6 +24,8 @@ public class SystemSettingService { private static final String SERPER_BASE_URL_KEY = "serperBaseUrl"; private static final String TAVILY_API_KEY_KEY = "tavilyApiKey"; private static final String TAVILY_BASE_URL_KEY = "tavilyBaseUrl"; + private static final String DUCKDUCKGO_ENABLED_KEY = "duckduckgoEnabled"; + private static final String SEARXNG_BASE_URL_KEY = "searxngBaseUrl"; private final SystemSettingMapper systemSettingMapper; @@ -40,6 +42,9 @@ public class SystemSettingService { dto.setSearchFallbackEnabled(Boolean.parseBoolean(getValue(SEARCH_FALLBACK_ENABLED_KEY, "false"))); dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search")); dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search")); + // Keyless provider 配置 + dto.setDuckduckgoEnabled(Boolean.parseBoolean(getValue(DUCKDUCKGO_ENABLED_KEY, "true"))); + dto.setSearxngBaseUrl(getValue(SEARXNG_BASE_URL_KEY, "")); // API Key 脱敏回显 dto.setSerperApiKeyMasked(maskApiKey(getValue(SERPER_API_KEY_KEY, ""))); dto.setTavilyApiKeyMasked(maskApiKey(getValue(TAVILY_API_KEY_KEY, ""))); @@ -58,6 +63,8 @@ public class SystemSettingService { dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search")); dto.setTavilyApiKey(getValue(TAVILY_API_KEY_KEY, "")); dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search")); + dto.setDuckduckgoEnabled(Boolean.parseBoolean(getValue(DUCKDUCKGO_ENABLED_KEY, "true"))); + dto.setSearxngBaseUrl(getValue(SEARXNG_BASE_URL_KEY, "")); return dto; } @@ -90,6 +97,13 @@ public class SystemSettingService { if (dto.getTavilyBaseUrl() != null) { saveValue(TAVILY_BASE_URL_KEY, dto.getTavilyBaseUrl(), "Tavily 接口地址"); } + // Keyless provider 配置 + if (dto.getDuckduckgoEnabled() != null) { + saveValue(DUCKDUCKGO_ENABLED_KEY, String.valueOf(dto.getDuckduckgoEnabled()), "DuckDuckGo 免 Key 搜索(零配置兜底)"); + } + if (dto.getSearxngBaseUrl() != null) { + saveValue(SEARXNG_BASE_URL_KEY, dto.getSearxngBaseUrl(), "SearXNG 实例地址"); + } return getSettings(); } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/builtin/WebSearchService.java b/mateclaw-server/src/main/java/vip/mate/tool/builtin/WebSearchService.java index ca29a240..be839cbf 100644 --- a/mateclaw-server/src/main/java/vip/mate/tool/builtin/WebSearchService.java +++ b/mateclaw-server/src/main/java/vip/mate/tool/builtin/WebSearchService.java @@ -1,16 +1,27 @@ package vip.mate.tool.builtin; -import cn.hutool.http.HttpUtil; -import cn.hutool.json.JSONObject; -import cn.hutool.json.JSONUtil; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import vip.mate.system.model.SystemSettingsDTO; import vip.mate.system.service.SystemSettingService; +import vip.mate.tool.search.SearchProvider; +import vip.mate.tool.search.SearchProviderRegistry; +import vip.mate.tool.search.SearchProviderRegistry.ResolvedProvider; +import vip.mate.tool.search.SearchResult; + +import java.util.List; +import java.util.stream.Collectors; /** - * 搜索服务:封装 Serper / Tavily 双 provider + fallback 逻辑 + * 搜索服务:通过 {@link SearchProviderRegistry} 实现 provider chain 路由与 keyless fallback + * + *

调度策略: + *

    + *
  1. 用户配置的 primary provider(有 key)
  2. + *
  3. 自动探测其他有 key 的 provider
  4. + *
  5. Keyless fallback(DuckDuckGo / SearXNG)
  6. + *
* * @author MateClaw Team */ @@ -20,9 +31,10 @@ import vip.mate.system.service.SystemSettingService; public class WebSearchService { private final SystemSettingService systemSettingService; + private final SearchProviderRegistry providerRegistry; /** - * 执行搜索,根据系统设置动态选择 provider + * 执行搜索,根据系统设置动态选择 provider(含 keyless fallback) */ public String search(String query) { SystemSettingsDTO config = systemSettingService.getSearchSettings(); @@ -31,94 +43,60 @@ public class WebSearchService { return "搜索功能已关闭,请在系统设置中启用。"; } - String primaryProvider = config.getSearchProvider(); - if (primaryProvider == null || primaryProvider.isBlank()) { - primaryProvider = "serper"; - } + // 通过 provider registry 解析最佳 provider + ResolvedProvider resolved = providerRegistry.resolve(config); + log.info("搜索 provider 解析: {}", resolved != null + ? resolved.provider().id() + " (source=" + resolved.source() + ")" + : "无可用 provider"); - // 尝试主 provider - String result = doSearch(query, primaryProvider, config); - if (result != null) { - return result; - } - - // 主 provider 失败,尝试 fallback - if (Boolean.TRUE.equals(config.getSearchFallbackEnabled())) { - String fallbackProvider = "serper".equals(primaryProvider) ? "tavily" : "serper"; - log.info("主搜索提供商 {} 调用失败,回退到 {}", primaryProvider, fallbackProvider); - result = doSearch(query, fallbackProvider, config); + if (resolved != null) { + String result = tryProvider(resolved.provider(), query, config); if (result != null) { + log.info("搜索成功 (provider={}, source={})", resolved.provider().id(), resolved.source()); return result; } } - return "搜索失败:所有搜索提供商均不可用,请在系统设置中检查 API Key 配置。"; + // 首选 provider 失败或不可用,遍历 fallback chain + if (Boolean.TRUE.equals(config.getSearchFallbackEnabled()) || resolved == null) { + for (SearchProvider p : providerRegistry.allSorted()) { + if (resolved != null && p.id().equals(resolved.provider().id())) continue; + if (!p.isAvailable(config)) continue; + + String result = tryProvider(p, query, config); + if (result != null) { + log.info("搜索 fallback 成功 (provider={})", p.id()); + return result; + } + } + } + + return "搜索暂时不可用。建议在系统设置中配置 Serper 或 Tavily API Key 以获得更好的搜索体验。"; + } + + private String tryProvider(SearchProvider provider, String query, SystemSettingsDTO config) { + try { + List results = provider.search(query, config); + if (results == null || results.isEmpty()) { + log.debug("Provider {} 返回空结果", provider.id()); + return null; + } + return formatResults(results, provider.id()); + } catch (Exception e) { + log.warn("Provider {} 搜索失败: {}", provider.id(), e.getMessage()); + return null; + } } /** - * 调用指定 provider 执行搜索,失败返回 null + * 将结构化搜索结果格式化为 Markdown(供 LLM 消费) */ - private String doSearch(String query, String provider, SystemSettingsDTO config) { - try { - return switch (provider) { - case "serper" -> searchWithSerper(query, config); - case "tavily" -> searchWithTavily(query, config); - default -> { - log.warn("未知的搜索提供商: {}", provider); - yield null; - } - }; - } catch (Exception e) { - log.error("搜索提供商 {} 调用异常: {}", provider, e.getMessage(), e); - return null; + private String formatResults(List results, String providerId) { + StringBuilder sb = new StringBuilder(); + sb.append("Search results (via ").append(providerId).append("):\n\n"); + for (int i = 0; i < results.size(); i++) { + sb.append(i + 1).append(". ").append(results.get(i).toMarkdown()).append("\n"); } - } - - private String searchWithSerper(String query, SystemSettingsDTO config) { - String apiKey = config.getSerperApiKey(); - if (apiKey == null || apiKey.isBlank()) { - log.warn("Serper API Key 未配置"); - return null; - } - String baseUrl = config.getSerperBaseUrl(); - if (baseUrl == null || baseUrl.isBlank()) { - baseUrl = "https://google.serper.dev/search"; - } - - String body = JSONUtil.toJsonStr(new JSONObject().set("q", query).set("num", 5)); - String result = HttpUtil.createPost(baseUrl) - .header("X-API-KEY", apiKey) - .header("Content-Type", "application/json") - .body(body) - .timeout(15000) - .execute() - .body(); - log.debug("Serper search result for '{}': {}", query, result); - return result; - } - - private String searchWithTavily(String query, SystemSettingsDTO config) { - String apiKey = config.getTavilyApiKey(); - if (apiKey == null || apiKey.isBlank()) { - log.warn("Tavily API Key 未配置"); - return null; - } - String baseUrl = config.getTavilyBaseUrl(); - if (baseUrl == null || baseUrl.isBlank()) { - baseUrl = "https://api.tavily.com/search"; - } - - String body = JSONUtil.toJsonStr(new JSONObject() - .set("query", query) - .set("max_results", 5) - .set("api_key", apiKey)); - String result = HttpUtil.createPost(baseUrl) - .header("Content-Type", "application/json") - .body(body) - .timeout(15000) - .execute() - .body(); - log.debug("Tavily search result for '{}': {}", query, result); - return result; + return sb.toString(); } } diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/DuckDuckGoSearchProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/search/DuckDuckGoSearchProvider.java new file mode 100644 index 00000000..311b188b --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/DuckDuckGoSearchProvider.java @@ -0,0 +1,176 @@ +package vip.mate.tool.search; + +import cn.hutool.http.HttpUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * DuckDuckGo 搜索提供商 — 无需 API Key(keyless fallback) + * + *

通过 DuckDuckGo HTML 端点实现免 key 搜索。 + * 作为零配置兜底方案,结果质量不如 Serper/Tavily 但"总比没有强"。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class DuckDuckGoSearchProvider implements SearchProvider { + + private static final String SEARCH_URL = "https://html.duckduckgo.com/html/"; + + // 匹配 DuckDuckGo HTML 搜索结果 + private static final Pattern RESULT_PATTERN = Pattern.compile( + "]+class=\"result__a\"[^>]+href=\"([^\"]+)\"[^>]*>(.+?)", + Pattern.DOTALL); + private static final Pattern SNIPPET_PATTERN = Pattern.compile( + "]+class=\"result__snippet\"[^>]*>(.+?)", + Pattern.DOTALL); + private static final Pattern RESULT_BLOCK_PATTERN = Pattern.compile( + "]+class=\"result results_links[^\"]*\"[^>]*>(.*?)\\s*(?=]+class=\"result |$)", + Pattern.DOTALL); + + @Override + public String id() { + return "duckduckgo"; + } + + @Override + public String label() { + return "DuckDuckGo"; + } + + @Override + public boolean requiresCredential() { + return false; + } + + @Override + public int autoDetectOrder() { + return 100; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + // 通过 duckduckgoEnabled 控制,默认启用 + return !Boolean.FALSE.equals(config.getDuckduckgoEnabled()); + } + + @Override + public List search(String query, SystemSettingsDTO config) { + String encoded = URLEncoder.encode(query, StandardCharsets.UTF_8); + + // DuckDuckGo 在部分网络环境下可能出现 SSLHandshakeException,加一次重试 + String response = null; + int maxAttempts = 2; + for (int attempt = 1; attempt <= maxAttempts; attempt++) { + try { + response = HttpUtil.createPost(SEARCH_URL) + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") + .header("Content-Type", "application/x-www-form-urlencoded") + .header("Accept", "text/html") + .header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8") + .body("q=" + encoded) + .timeout(15000) + .execute() + .body(); + break; // 成功则跳出 + } catch (Exception e) { + log.warn("DuckDuckGo 请求失败 (attempt {}/{}): {}", attempt, maxAttempts, e.getMessage()); + if (attempt == maxAttempts) { + throw e; // 最后一次仍失败则抛出,由 WebSearchService 捕获 + } + try { + Thread.sleep(1000); // 短暂等待后重试 + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new RuntimeException("DuckDuckGo 搜索被中断", ie); + } + } + } + + log.debug("DuckDuckGo search completed for '{}', response length: {}", query, + response != null ? response.length() : 0); + return parseHtmlResults(response); + } + + private List parseHtmlResults(String html) { + List results = new ArrayList<>(); + if (html == null || html.isBlank()) return results; + + // 提取每个结果块 + Matcher titleMatcher = RESULT_PATTERN.matcher(html); + Matcher snippetMatcher = SNIPPET_PATTERN.matcher(html); + + int count = 0; + while (titleMatcher.find() && count < 5) { + String rawUrl = titleMatcher.group(1); + String rawTitle = titleMatcher.group(2); + + String url = decodeUrl(rawUrl); + String title = stripHtml(rawTitle); + + String snippet = ""; + if (snippetMatcher.find()) { + snippet = stripHtml(snippetMatcher.group(1)); + } + + if (title.isBlank() && snippet.isBlank()) continue; + + results.add(SearchResult.builder() + .title(title) + .url(url) + .snippet(snippet) + .source(extractDomain(url)) + .providerId(id()) + .build()); + count++; + } + + log.debug("DuckDuckGo 解析到 {} 条结果", results.size()); + return results; + } + + /** + * DuckDuckGo 返回的 URL 经过跳转编码 (//duckduckgo.com/l/?uddg=...),需要解码 + */ + private String decodeUrl(String rawUrl) { + if (rawUrl == null) return ""; + // DuckDuckGo 使用 redirect URL + if (rawUrl.contains("uddg=")) { + try { + String encoded = rawUrl.substring(rawUrl.indexOf("uddg=") + 5); + int ampIndex = encoded.indexOf('&'); + if (ampIndex > 0) encoded = encoded.substring(0, ampIndex); + return java.net.URLDecoder.decode(encoded, StandardCharsets.UTF_8); + } catch (Exception e) { + log.debug("URL 解码失败: {}", rawUrl); + } + } + return rawUrl; + } + + private String stripHtml(String html) { + if (html == null) return ""; + return html.replaceAll("<[^>]+>", "").replaceAll("&", "&") + .replaceAll("<", "<").replaceAll(">", ">") + .replaceAll(""", "\"").replaceAll("'", "'") + .trim(); + } + + private String extractDomain(String url) { + try { + return URI.create(url).getHost(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/SearXNGSearchProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/search/SearXNGSearchProvider.java new file mode 100644 index 00000000..1a2d47bf --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/SearXNGSearchProvider.java @@ -0,0 +1,111 @@ +package vip.mate.tool.search; + +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.net.URI; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +/** + * SearXNG 搜索提供商 — 无需 API Key,但需要 base URL(自部署实例) + * + *

SearXNG 是开源的元搜索引擎,可自部署。Docker 部署 MateClaw 时自动包含 SearXNG 服务, + * 默认 base URL 为 {@code http://searxng:8080}。 + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class SearXNGSearchProvider implements SearchProvider { + + @Override + public String id() { + return "searxng"; + } + + @Override + public String label() { + return "SearXNG"; + } + + @Override + public boolean requiresCredential() { + return false; + } + + @Override + public int autoDetectOrder() { + // SearXNG 聚合多引擎,结果质量高于 DuckDuckGo(order=100),应优先 + return 50; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + String baseUrl = config.getSearxngBaseUrl(); + return baseUrl != null && !baseUrl.isBlank(); + } + + @Override + public List search(String query, SystemSettingsDTO config) { + String baseUrl = config.getSearxngBaseUrl(); + if (baseUrl.endsWith("/")) { + baseUrl = baseUrl.substring(0, baseUrl.length() - 1); + } + + String encoded = URLEncoder.encode(query, StandardCharsets.UTF_8); + String url = baseUrl + "/search?q=" + encoded + "&format=json&categories=general&language=auto"; + + String response = HttpUtil.createGet(url) + .header("Accept", "application/json") + .timeout(15000) + .execute() + .body(); + + log.debug("SearXNG result for '{}': length={}", query, response != null ? response.length() : 0); + return parseResponse(response); + } + + private List parseResponse(String response) { + List results = new ArrayList<>(); + if (response == null || response.isBlank()) return results; + + try { + JSONObject json = JSONUtil.parseObj(response); + JSONArray items = json.getJSONArray("results"); + if (items == null) return results; + + int limit = Math.min(items.size(), 5); + for (int i = 0; i < limit; i++) { + JSONObject item = items.getJSONObject(i); + String itemUrl = item.getStr("url"); + results.add(SearchResult.builder() + .title(item.getStr("title")) + .url(itemUrl) + .snippet(item.getStr("content")) + .source(extractDomain(itemUrl)) + .date(item.getStr("publishedDate")) + .providerId(id()) + .build()); + } + } catch (Exception e) { + log.warn("SearXNG 结果解析失败: {}", e.getMessage()); + } + return results; + } + + private String extractDomain(String url) { + try { + return URI.create(url).getHost(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProvider.java new file mode 100644 index 00000000..cf336fd3 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProvider.java @@ -0,0 +1,45 @@ +package vip.mate.tool.search; + +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.List; + +/** + * 搜索提供商接口 — 所有搜索 provider(含 keyless)统一实现此接口 + * + * @author MateClaw Team + */ +public interface SearchProvider { + + /** 提供商唯一 ID,如 "serper"、"tavily"、"duckduckgo"、"searxng" */ + String id(); + + /** 显示名称 */ + String label(); + + /** 是否需要 API Key / Credential */ + boolean requiresCredential(); + + /** + * 自动探测排序优先级(升序)。 + *

有 key 的 provider 用较大值(300+),keyless 用较小值(100+), + * 但实际调度时"有 credential 的 provider"优先于 keyless, + * 此字段仅在同类 provider 之间排序。 + */ + int autoDetectOrder(); + + /** + * 判断该 provider 在当前配置下是否可用 + * (key 已配置 / keyless provider 已启用等) + */ + boolean isAvailable(SystemSettingsDTO config); + + /** + * 执行搜索,返回结构化结果列表 + * + * @param query 搜索关键词 + * @param config 系统配置(包含 key、baseUrl 等) + * @return 搜索结果列表;不应返回 null,失败时抛异常 + */ + List search(String query, SystemSettingsDTO config); +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProviderRegistry.java b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProviderRegistry.java new file mode 100644 index 00000000..3ef10131 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchProviderRegistry.java @@ -0,0 +1,102 @@ +package vip.mate.tool.search; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * 搜索提供商注册表 — 收集所有 {@link SearchProvider} 实现,提供优先级排序与自动探测 + * + *

借鉴 openclaw 的 provider auto-detect 机制: + *

    + *
  1. 用户显式配置的 primary provider → 直接使用
  2. + *
  3. 按 autoDetectOrder 遍历,优先选有 credential 的 provider
  4. + *
  5. 如果没有有 credential 的 provider,回退到第一个可用的 keyless provider
  6. + *
+ * + * @author MateClaw Team + */ +@Slf4j +@Component +public class SearchProviderRegistry { + + private final List sortedProviders; + private final Map providerMap; + + public SearchProviderRegistry(List providers) { + this.sortedProviders = providers.stream() + .sorted(Comparator.comparingInt(SearchProvider::autoDetectOrder)) + .toList(); + this.providerMap = providers.stream() + .collect(Collectors.toMap(SearchProvider::id, Function.identity())); + log.info("注册搜索提供商 {} 个: {}", sortedProviders.size(), + sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList()); + } + + /** 按 ID 获取指定 provider */ + public SearchProvider getById(String id) { + return providerMap.get(id); + } + + /** 获取按 autoDetectOrder 排序的全部 provider 列表 */ + public List allSorted() { + return sortedProviders; + } + + /** + * 根据当前配置,解析应使用的 provider。 + * + *

解析策略(借鉴 openclaw resolveWebSearchProviderId): + *

    + *
  1. 如果用户配置了 primary provider 且该 provider 可用 → 选中
  2. + *
  3. 否则按 autoDetectOrder 遍历,跳过 keyless,先找有 credential 的
  4. + *
  5. 如果没找到 → 回退到第一个可用的 keyless provider
  6. + *
+ * + * @return 选中的 provider,或 null(完全无可用 provider) + */ + public ResolvedProvider resolve(SystemSettingsDTO config) { + // 1. 用户显式配置的 primary provider + String configuredId = config.getSearchProvider(); + if (configuredId != null && !configuredId.isBlank()) { + SearchProvider configured = providerMap.get(configuredId); + if (configured != null && configured.isAvailable(config)) { + return new ResolvedProvider(configured, "configured"); + } + } + + // 2. 按优先级遍历,先找有 credential 的 + SearchProvider keylessFallback = null; + for (SearchProvider p : sortedProviders) { + if (!p.requiresCredential()) { + // 记住第一个可用的 keyless provider + if (keylessFallback == null && p.isAvailable(config)) { + keylessFallback = p; + } + continue; + } + if (p.isAvailable(config)) { + return new ResolvedProvider(p, "auto-detect"); + } + } + + // 3. 回退到 keyless + if (keylessFallback != null) { + return new ResolvedProvider(keylessFallback, "keyless-fallback"); + } + + return null; + } + + /** + * 解析结果 + */ + public record ResolvedProvider(SearchProvider provider, String source) { + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/SearchResult.java b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchResult.java new file mode 100644 index 00000000..4214a7a5 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/SearchResult.java @@ -0,0 +1,54 @@ +package vip.mate.tool.search; + +import lombok.Builder; +import lombok.Data; + +/** + * 统一搜索结果结构 — 所有 provider 返回此格式 + * + * @author MateClaw Team + */ +@Data +@Builder +public class SearchResult { + + /** 结果标题 */ + private String title; + + /** 结果链接 */ + private String url; + + /** 摘要/片段 */ + private String snippet; + + /** 来源域名 (如 "reuters.com") */ + private String source; + + /** 发布时间(原始字符串,尽可能保留) */ + private String date; + + /** 提供该结果的 provider id */ + private String providerId; + + /** + * 格式化为 Markdown 行,用于返回给 LLM + */ + public String toMarkdown() { + StringBuilder sb = new StringBuilder(); + sb.append("**").append(title != null ? title : "Untitled").append("**"); + if (source != null || date != null) { + sb.append(" — "); + if (source != null) sb.append(source); + if (source != null && date != null) sb.append(" | "); + if (date != null) sb.append(date); + } + sb.append("\n"); + if (snippet != null && !snippet.isBlank()) { + sb.append(snippet).append("\n"); + } + if (url != null && !url.isBlank()) { + sb.append("🔗 ").append(url).append("\n"); + } + return sb.toString(); + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/SerperSearchProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/search/SerperSearchProvider.java new file mode 100644 index 00000000..a6bc6ef1 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/SerperSearchProvider.java @@ -0,0 +1,105 @@ +package vip.mate.tool.search; + +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +/** + * Serper (Google Search) 搜索提供商 — 需要 API Key + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class SerperSearchProvider implements SearchProvider { + + private static final String DEFAULT_BASE_URL = "https://google.serper.dev/search"; + + @Override + public String id() { + return "serper"; + } + + @Override + public String label() { + return "Serper (Google)"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 300; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + String key = config.getSerperApiKey(); + return key != null && !key.isBlank(); + } + + @Override + public List search(String query, SystemSettingsDTO config) { + String apiKey = config.getSerperApiKey(); + String baseUrl = config.getSerperBaseUrl(); + if (baseUrl == null || baseUrl.isBlank()) { + baseUrl = DEFAULT_BASE_URL; + } + + String body = JSONUtil.toJsonStr(new JSONObject().set("q", query).set("num", 5)); + String response = HttpUtil.createPost(baseUrl) + .header("X-API-KEY", apiKey) + .header("Content-Type", "application/json") + .body(body) + .timeout(15000) + .execute() + .body(); + + log.debug("Serper result for '{}': {}", query, response); + return parseResponse(response); + } + + private List parseResponse(String response) { + List results = new ArrayList<>(); + try { + JSONObject json = JSONUtil.parseObj(response); + JSONArray organic = json.getJSONArray("organic"); + if (organic == null) return results; + + for (int i = 0; i < organic.size(); i++) { + JSONObject item = organic.getJSONObject(i); + String url = item.getStr("link"); + results.add(SearchResult.builder() + .title(item.getStr("title")) + .url(url) + .snippet(item.getStr("snippet")) + .source(extractDomain(url)) + .date(item.getStr("date")) + .providerId(id()) + .build()); + } + } catch (Exception e) { + log.warn("Serper 结果解析失败: {}", e.getMessage()); + } + return results; + } + + private String extractDomain(String url) { + try { + return URI.create(url).getHost(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/java/vip/mate/tool/search/TavilySearchProvider.java b/mateclaw-server/src/main/java/vip/mate/tool/search/TavilySearchProvider.java new file mode 100644 index 00000000..26951582 --- /dev/null +++ b/mateclaw-server/src/main/java/vip/mate/tool/search/TavilySearchProvider.java @@ -0,0 +1,107 @@ +package vip.mate.tool.search; + +import cn.hutool.http.HttpUtil; +import cn.hutool.json.JSONArray; +import cn.hutool.json.JSONObject; +import cn.hutool.json.JSONUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import vip.mate.system.model.SystemSettingsDTO; + +import java.net.URI; +import java.util.ArrayList; +import java.util.List; + +/** + * Tavily 搜索提供商 — 需要 API Key + * + * @author MateClaw Team + */ +@Slf4j +@Component +public class TavilySearchProvider implements SearchProvider { + + private static final String DEFAULT_BASE_URL = "https://api.tavily.com/search"; + + @Override + public String id() { + return "tavily"; + } + + @Override + public String label() { + return "Tavily"; + } + + @Override + public boolean requiresCredential() { + return true; + } + + @Override + public int autoDetectOrder() { + return 400; + } + + @Override + public boolean isAvailable(SystemSettingsDTO config) { + String key = config.getTavilyApiKey(); + return key != null && !key.isBlank(); + } + + @Override + public List search(String query, SystemSettingsDTO config) { + String apiKey = config.getTavilyApiKey(); + String baseUrl = config.getTavilyBaseUrl(); + if (baseUrl == null || baseUrl.isBlank()) { + baseUrl = DEFAULT_BASE_URL; + } + + String body = JSONUtil.toJsonStr(new JSONObject() + .set("query", query) + .set("max_results", 5) + .set("api_key", apiKey)); + String response = HttpUtil.createPost(baseUrl) + .header("Content-Type", "application/json") + .body(body) + .timeout(15000) + .execute() + .body(); + + log.debug("Tavily result for '{}': {}", query, response); + return parseResponse(response); + } + + private List parseResponse(String response) { + List results = new ArrayList<>(); + try { + JSONObject json = JSONUtil.parseObj(response); + JSONArray items = json.getJSONArray("results"); + if (items == null) return results; + + for (int i = 0; i < items.size(); i++) { + JSONObject item = items.getJSONObject(i); + String url = item.getStr("url"); + results.add(SearchResult.builder() + .title(item.getStr("title")) + .url(url) + .snippet(item.getStr("content")) + .source(extractDomain(url)) + .date(item.getStr("published_date")) + .providerId(id()) + .build()); + } + } catch (Exception e) { + log.warn("Tavily 结果解析失败: {}", e.getMessage()); + } + return results; + } + + private String extractDomain(String url) { + try { + return URI.create(url).getHost(); + } catch (Exception e) { + return null; + } + } +} diff --git a/mateclaw-server/src/main/resources/db/data-en.sql b/mateclaw-server/src/main/resources/db/data-en.sql index fed8af41..5a3091ee 100644 --- a/mateclaw-server/src/main/resources/db/data-en.sql +++ b/mateclaw-server/src/main/resources/db/data-en.sql @@ -294,6 +294,14 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre KEY (id) VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW()); +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo keyless search fallback (zero-config)', NOW(), NOW()); + +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW()); + -- Built-in tool: Date & Time MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) KEY (id) diff --git a/mateclaw-server/src/main/resources/db/data-mysql-en.sql b/mateclaw-server/src/main/resources/db/data-mysql-en.sql index 4cbff52a..1c158238 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-en.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-en.sql @@ -296,6 +296,14 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW()) ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo keyless search fallback (zero-config)', NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); + +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); + -- Built-in tool: Date & Time INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) VALUES (1000000001, 'DateTimeTool', 'Date & Time', 'Get current date and time information', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0) diff --git a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql index 1b93ad3b..19a20434 100644 --- a/mateclaw-server/src/main/resources/db/data-mysql-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-mysql-zh.sql @@ -296,6 +296,14 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW()) ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo 免 Key 搜索兜底(零配置可用)', NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); + +INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址(Docker 部署时自动配置)', NOW(), NOW()) +ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time); + -- 内置工具:日期时间 INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) VALUES (1000000001, 'DateTimeTool', '日期时间', '获取当前日期和时间信息', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0) diff --git a/mateclaw-server/src/main/resources/db/data-zh.sql b/mateclaw-server/src/main/resources/db/data-zh.sql index e5b6da6c..671a72fb 100644 --- a/mateclaw-server/src/main/resources/db/data-zh.sql +++ b/mateclaw-server/src/main/resources/db/data-zh.sql @@ -300,6 +300,14 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre KEY (id) VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW()); +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo 免 Key 搜索兜底(零配置可用)', NOW(), NOW()); + +MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time) +KEY (id) +VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址(Docker 部署时自动配置)', NOW(), NOW()); + -- 内置工具:日期时间 MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted) KEY (id) diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index f0a45749..fba3fedb 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -330,6 +330,8 @@ export default { serperBaseUrl: 'Serper Base URL', tavilyApiKey: 'Tavily API Key', tavilyBaseUrl: 'Tavily Base URL', + duckduckgoEnabled: 'DuckDuckGo (Keyless)', + searxngBaseUrl: 'SearXNG Base URL', }, hints: { provider: 'Current implementation applies DashScope model options at runtime.', @@ -343,6 +345,8 @@ export default { serperBaseUrl: 'Usually no need to change unless using a custom proxy.', tavilyApiKey: 'API key for Tavily search, get it from tavily.com.', tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.', + duckduckgoEnabled: 'Free keyless search fallback. No API key required. Enabled by default as a zero-config fallback.', + searxngBaseUrl: 'Self-hosted SearXNG instance URL. Auto-configured when deploying via Docker.', }, searchTitle: 'Search Service', searchDesc: 'Configure the built-in search tool provider and API credentials', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 94e75b09..9bdfba35 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -330,6 +330,8 @@ export default { serperBaseUrl: 'Serper 接口地址', tavilyApiKey: 'Tavily API Key', tavilyBaseUrl: 'Tavily 接口地址', + duckduckgoEnabled: 'DuckDuckGo(免 Key)', + searxngBaseUrl: 'SearXNG 地址', }, hints: { provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。', @@ -343,6 +345,8 @@ export default { serperBaseUrl: '通常无需修改,除非使用自定义代理地址。', tavilyApiKey: '用于 Tavily 搜索服务,从 tavily.com 获取。', tavilyBaseUrl: '通常无需修改,除非使用自定义代理地址。', + duckduckgoEnabled: '免费搜索兜底,无需 API Key。默认开启,作为零配置下的搜索降级方案。', + searxngBaseUrl: '自部署 SearXNG 实例地址。Docker 部署时自动配置。', }, searchTitle: '搜索服务', searchDesc: '配置内置搜索工具的提供商与 API 凭证', diff --git a/mateclaw-ui/src/types/index.ts b/mateclaw-ui/src/types/index.ts index 65001410..e7bd425a 100644 --- a/mateclaw-ui/src/types/index.ts +++ b/mateclaw-ui/src/types/index.ts @@ -499,6 +499,9 @@ export interface SystemSettings { tavilyBaseUrl: string serperApiKeyMasked?: string tavilyApiKeyMasked?: string + // Keyless 搜索 provider + duckduckgoEnabled: boolean + searxngBaseUrl: string } export interface ProviderModelInfo { diff --git a/mateclaw-ui/src/views/Settings/System/index.vue b/mateclaw-ui/src/views/Settings/System/index.vue index 70ed85f8..45e3eef9 100644 --- a/mateclaw-ui/src/views/Settings/System/index.vue +++ b/mateclaw-ui/src/views/Settings/System/index.vue @@ -159,6 +159,36 @@ /> + + +
+
+
{{ t('settings.fields.duckduckgoEnabled') }}
+
{{ t('settings.hints.duckduckgoEnabled') }}
+
+
+ +
+
+ +
+
+
{{ t('settings.fields.searxngBaseUrl') }}
+
{{ t('settings.hints.searxngBaseUrl') }}
+
+
+ +
+
@@ -193,6 +223,8 @@ const settings = reactive({ searchFallbackEnabled: false, serperBaseUrl: 'https://google.serper.dev/search', tavilyBaseUrl: 'https://api.tavily.com/search', + duckduckgoEnabled: true, + searxngBaseUrl: '', }) onMounted(async () => {