mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 20:08:18 +08:00
feat(search): add SearchProvider chain with keyless fallback (DuckDuckGo + SearXNG)
This commit is contained in:
parent
afe8466fa5
commit
b750306028
@ -25,6 +25,23 @@ services:
|
|||||||
timeout: 5s
|
timeout: 5s
|
||||||
retries: 5
|
retries: 5
|
||||||
|
|
||||||
|
# SearXNG 搜索引擎(keyless 搜索 provider,零配置可用)
|
||||||
|
searxng:
|
||||||
|
image: searxng/searxng:latest
|
||||||
|
container_name: mateclaw-searxng
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
- SEARXNG_BASE_URL=http://searxng:8080
|
||||||
|
volumes:
|
||||||
|
- searxng_data:/etc/searxng
|
||||||
|
ports:
|
||||||
|
- "8088:8080"
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "wget", "--spider", "-q", "http://localhost:8080/healthz"]
|
||||||
|
interval: 30s
|
||||||
|
timeout: 5s
|
||||||
|
retries: 3
|
||||||
|
|
||||||
# MateClaw 后端服务
|
# MateClaw 后端服务
|
||||||
mateclaw-server:
|
mateclaw-server:
|
||||||
build:
|
build:
|
||||||
@ -35,6 +52,8 @@ services:
|
|||||||
depends_on:
|
depends_on:
|
||||||
mysql:
|
mysql:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
searxng:
|
||||||
|
condition: service_healthy
|
||||||
environment:
|
environment:
|
||||||
SPRING_PROFILES_ACTIVE: mysql
|
SPRING_PROFILES_ACTIVE: mysql
|
||||||
DB_HOST: mysql
|
DB_HOST: mysql
|
||||||
@ -52,3 +71,4 @@ services:
|
|||||||
volumes:
|
volumes:
|
||||||
mysql_data:
|
mysql_data:
|
||||||
server_data:
|
server_data:
|
||||||
|
searxng_data:
|
||||||
|
|||||||
@ -24,6 +24,12 @@ public class SystemSettingsDTO {
|
|||||||
private String tavilyApiKey;
|
private String tavilyApiKey;
|
||||||
private String tavilyBaseUrl;
|
private String tavilyBaseUrl;
|
||||||
|
|
||||||
|
// ===== Keyless 搜索 provider 配置 =====
|
||||||
|
/** DuckDuckGo 是否启用(默认 true,作为零配置兜底) */
|
||||||
|
private Boolean duckduckgoEnabled;
|
||||||
|
/** SearXNG 实例地址(如 http://searxng:8080),为空则不使用 */
|
||||||
|
private String searxngBaseUrl;
|
||||||
|
|
||||||
// 用于前端回显脱敏后的 API Key
|
// 用于前端回显脱敏后的 API Key
|
||||||
private String serperApiKeyMasked;
|
private String serperApiKeyMasked;
|
||||||
private String tavilyApiKeyMasked;
|
private String tavilyApiKeyMasked;
|
||||||
|
|||||||
@ -24,6 +24,8 @@ public class SystemSettingService {
|
|||||||
private static final String SERPER_BASE_URL_KEY = "serperBaseUrl";
|
private static final String SERPER_BASE_URL_KEY = "serperBaseUrl";
|
||||||
private static final String TAVILY_API_KEY_KEY = "tavilyApiKey";
|
private static final String TAVILY_API_KEY_KEY = "tavilyApiKey";
|
||||||
private static final String TAVILY_BASE_URL_KEY = "tavilyBaseUrl";
|
private static final String TAVILY_BASE_URL_KEY = "tavilyBaseUrl";
|
||||||
|
private static final String DUCKDUCKGO_ENABLED_KEY = "duckduckgoEnabled";
|
||||||
|
private static final String SEARXNG_BASE_URL_KEY = "searxngBaseUrl";
|
||||||
|
|
||||||
private final SystemSettingMapper systemSettingMapper;
|
private final SystemSettingMapper systemSettingMapper;
|
||||||
|
|
||||||
@ -40,6 +42,9 @@ public class SystemSettingService {
|
|||||||
dto.setSearchFallbackEnabled(Boolean.parseBoolean(getValue(SEARCH_FALLBACK_ENABLED_KEY, "false")));
|
dto.setSearchFallbackEnabled(Boolean.parseBoolean(getValue(SEARCH_FALLBACK_ENABLED_KEY, "false")));
|
||||||
dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search"));
|
dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search"));
|
||||||
dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search"));
|
dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search"));
|
||||||
|
// Keyless provider 配置
|
||||||
|
dto.setDuckduckgoEnabled(Boolean.parseBoolean(getValue(DUCKDUCKGO_ENABLED_KEY, "true")));
|
||||||
|
dto.setSearxngBaseUrl(getValue(SEARXNG_BASE_URL_KEY, ""));
|
||||||
// API Key 脱敏回显
|
// API Key 脱敏回显
|
||||||
dto.setSerperApiKeyMasked(maskApiKey(getValue(SERPER_API_KEY_KEY, "")));
|
dto.setSerperApiKeyMasked(maskApiKey(getValue(SERPER_API_KEY_KEY, "")));
|
||||||
dto.setTavilyApiKeyMasked(maskApiKey(getValue(TAVILY_API_KEY_KEY, "")));
|
dto.setTavilyApiKeyMasked(maskApiKey(getValue(TAVILY_API_KEY_KEY, "")));
|
||||||
@ -58,6 +63,8 @@ public class SystemSettingService {
|
|||||||
dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search"));
|
dto.setSerperBaseUrl(getValue(SERPER_BASE_URL_KEY, "https://google.serper.dev/search"));
|
||||||
dto.setTavilyApiKey(getValue(TAVILY_API_KEY_KEY, ""));
|
dto.setTavilyApiKey(getValue(TAVILY_API_KEY_KEY, ""));
|
||||||
dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search"));
|
dto.setTavilyBaseUrl(getValue(TAVILY_BASE_URL_KEY, "https://api.tavily.com/search"));
|
||||||
|
dto.setDuckduckgoEnabled(Boolean.parseBoolean(getValue(DUCKDUCKGO_ENABLED_KEY, "true")));
|
||||||
|
dto.setSearxngBaseUrl(getValue(SEARXNG_BASE_URL_KEY, ""));
|
||||||
return dto;
|
return dto;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -90,6 +97,13 @@ public class SystemSettingService {
|
|||||||
if (dto.getTavilyBaseUrl() != null) {
|
if (dto.getTavilyBaseUrl() != null) {
|
||||||
saveValue(TAVILY_BASE_URL_KEY, dto.getTavilyBaseUrl(), "Tavily 接口地址");
|
saveValue(TAVILY_BASE_URL_KEY, dto.getTavilyBaseUrl(), "Tavily 接口地址");
|
||||||
}
|
}
|
||||||
|
// Keyless provider 配置
|
||||||
|
if (dto.getDuckduckgoEnabled() != null) {
|
||||||
|
saveValue(DUCKDUCKGO_ENABLED_KEY, String.valueOf(dto.getDuckduckgoEnabled()), "DuckDuckGo 免 Key 搜索(零配置兜底)");
|
||||||
|
}
|
||||||
|
if (dto.getSearxngBaseUrl() != null) {
|
||||||
|
saveValue(SEARXNG_BASE_URL_KEY, dto.getSearxngBaseUrl(), "SearXNG 实例地址");
|
||||||
|
}
|
||||||
return getSettings();
|
return getSettings();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -1,16 +1,27 @@
|
|||||||
package vip.mate.tool.builtin;
|
package vip.mate.tool.builtin;
|
||||||
|
|
||||||
import cn.hutool.http.HttpUtil;
|
|
||||||
import cn.hutool.json.JSONObject;
|
|
||||||
import cn.hutool.json.JSONUtil;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import lombok.extern.slf4j.Slf4j;
|
import lombok.extern.slf4j.Slf4j;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import vip.mate.system.model.SystemSettingsDTO;
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
import vip.mate.system.service.SystemSettingService;
|
import vip.mate.system.service.SystemSettingService;
|
||||||
|
import vip.mate.tool.search.SearchProvider;
|
||||||
|
import vip.mate.tool.search.SearchProviderRegistry;
|
||||||
|
import vip.mate.tool.search.SearchProviderRegistry.ResolvedProvider;
|
||||||
|
import vip.mate.tool.search.SearchResult;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 搜索服务:封装 Serper / Tavily 双 provider + fallback 逻辑
|
* 搜索服务:通过 {@link SearchProviderRegistry} 实现 provider chain 路由与 keyless fallback
|
||||||
|
*
|
||||||
|
* <p>调度策略:
|
||||||
|
* <ol>
|
||||||
|
* <li>用户配置的 primary provider(有 key)</li>
|
||||||
|
* <li>自动探测其他有 key 的 provider</li>
|
||||||
|
* <li>Keyless fallback(DuckDuckGo / SearXNG)</li>
|
||||||
|
* </ol>
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -20,9 +31,10 @@ import vip.mate.system.service.SystemSettingService;
|
|||||||
public class WebSearchService {
|
public class WebSearchService {
|
||||||
|
|
||||||
private final SystemSettingService systemSettingService;
|
private final SystemSettingService systemSettingService;
|
||||||
|
private final SearchProviderRegistry providerRegistry;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 执行搜索,根据系统设置动态选择 provider
|
* 执行搜索,根据系统设置动态选择 provider(含 keyless fallback)
|
||||||
*/
|
*/
|
||||||
public String search(String query) {
|
public String search(String query) {
|
||||||
SystemSettingsDTO config = systemSettingService.getSearchSettings();
|
SystemSettingsDTO config = systemSettingService.getSearchSettings();
|
||||||
@ -31,94 +43,60 @@ public class WebSearchService {
|
|||||||
return "搜索功能已关闭,请在系统设置中启用。";
|
return "搜索功能已关闭,请在系统设置中启用。";
|
||||||
}
|
}
|
||||||
|
|
||||||
String primaryProvider = config.getSearchProvider();
|
// 通过 provider registry 解析最佳 provider
|
||||||
if (primaryProvider == null || primaryProvider.isBlank()) {
|
ResolvedProvider resolved = providerRegistry.resolve(config);
|
||||||
primaryProvider = "serper";
|
log.info("搜索 provider 解析: {}", resolved != null
|
||||||
}
|
? resolved.provider().id() + " (source=" + resolved.source() + ")"
|
||||||
|
: "无可用 provider");
|
||||||
|
|
||||||
// 尝试主 provider
|
if (resolved != null) {
|
||||||
String result = doSearch(query, primaryProvider, config);
|
String result = tryProvider(resolved.provider(), query, config);
|
||||||
if (result != null) {
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// 主 provider 失败,尝试 fallback
|
|
||||||
if (Boolean.TRUE.equals(config.getSearchFallbackEnabled())) {
|
|
||||||
String fallbackProvider = "serper".equals(primaryProvider) ? "tavily" : "serper";
|
|
||||||
log.info("主搜索提供商 {} 调用失败,回退到 {}", primaryProvider, fallbackProvider);
|
|
||||||
result = doSearch(query, fallbackProvider, config);
|
|
||||||
if (result != null) {
|
if (result != null) {
|
||||||
|
log.info("搜索成功 (provider={}, source={})", resolved.provider().id(), resolved.source());
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return "搜索失败:所有搜索提供商均不可用,请在系统设置中检查 API Key 配置。";
|
// 首选 provider 失败或不可用,遍历 fallback chain
|
||||||
|
if (Boolean.TRUE.equals(config.getSearchFallbackEnabled()) || resolved == null) {
|
||||||
|
for (SearchProvider p : providerRegistry.allSorted()) {
|
||||||
|
if (resolved != null && p.id().equals(resolved.provider().id())) continue;
|
||||||
|
if (!p.isAvailable(config)) continue;
|
||||||
|
|
||||||
|
String result = tryProvider(p, query, config);
|
||||||
|
if (result != null) {
|
||||||
|
log.info("搜索 fallback 成功 (provider={})", p.id());
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return "搜索暂时不可用。建议在系统设置中配置 Serper 或 Tavily API Key 以获得更好的搜索体验。";
|
||||||
|
}
|
||||||
|
|
||||||
|
private String tryProvider(SearchProvider provider, String query, SystemSettingsDTO config) {
|
||||||
|
try {
|
||||||
|
List<SearchResult> results = provider.search(query, config);
|
||||||
|
if (results == null || results.isEmpty()) {
|
||||||
|
log.debug("Provider {} 返回空结果", provider.id());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return formatResults(results, provider.id());
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Provider {} 搜索失败: {}", provider.id(), e.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 调用指定 provider 执行搜索,失败返回 null
|
* 将结构化搜索结果格式化为 Markdown(供 LLM 消费)
|
||||||
*/
|
*/
|
||||||
private String doSearch(String query, String provider, SystemSettingsDTO config) {
|
private String formatResults(List<SearchResult> results, String providerId) {
|
||||||
try {
|
StringBuilder sb = new StringBuilder();
|
||||||
return switch (provider) {
|
sb.append("Search results (via ").append(providerId).append("):\n\n");
|
||||||
case "serper" -> searchWithSerper(query, config);
|
for (int i = 0; i < results.size(); i++) {
|
||||||
case "tavily" -> searchWithTavily(query, config);
|
sb.append(i + 1).append(". ").append(results.get(i).toMarkdown()).append("\n");
|
||||||
default -> {
|
|
||||||
log.warn("未知的搜索提供商: {}", provider);
|
|
||||||
yield null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
} catch (Exception e) {
|
|
||||||
log.error("搜索提供商 {} 调用异常: {}", provider, e.getMessage(), e);
|
|
||||||
return null;
|
|
||||||
}
|
}
|
||||||
}
|
return sb.toString();
|
||||||
|
|
||||||
private String searchWithSerper(String query, SystemSettingsDTO config) {
|
|
||||||
String apiKey = config.getSerperApiKey();
|
|
||||||
if (apiKey == null || apiKey.isBlank()) {
|
|
||||||
log.warn("Serper API Key 未配置");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String baseUrl = config.getSerperBaseUrl();
|
|
||||||
if (baseUrl == null || baseUrl.isBlank()) {
|
|
||||||
baseUrl = "https://google.serper.dev/search";
|
|
||||||
}
|
|
||||||
|
|
||||||
String body = JSONUtil.toJsonStr(new JSONObject().set("q", query).set("num", 5));
|
|
||||||
String result = HttpUtil.createPost(baseUrl)
|
|
||||||
.header("X-API-KEY", apiKey)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.body(body)
|
|
||||||
.timeout(15000)
|
|
||||||
.execute()
|
|
||||||
.body();
|
|
||||||
log.debug("Serper search result for '{}': {}", query, result);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
private String searchWithTavily(String query, SystemSettingsDTO config) {
|
|
||||||
String apiKey = config.getTavilyApiKey();
|
|
||||||
if (apiKey == null || apiKey.isBlank()) {
|
|
||||||
log.warn("Tavily API Key 未配置");
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
String baseUrl = config.getTavilyBaseUrl();
|
|
||||||
if (baseUrl == null || baseUrl.isBlank()) {
|
|
||||||
baseUrl = "https://api.tavily.com/search";
|
|
||||||
}
|
|
||||||
|
|
||||||
String body = JSONUtil.toJsonStr(new JSONObject()
|
|
||||||
.set("query", query)
|
|
||||||
.set("max_results", 5)
|
|
||||||
.set("api_key", apiKey));
|
|
||||||
String result = HttpUtil.createPost(baseUrl)
|
|
||||||
.header("Content-Type", "application/json")
|
|
||||||
.body(body)
|
|
||||||
.timeout(15000)
|
|
||||||
.execute()
|
|
||||||
.body();
|
|
||||||
log.debug("Tavily search result for '{}': {}", query, result);
|
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,176 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.regex.Matcher;
|
||||||
|
import java.util.regex.Pattern;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DuckDuckGo 搜索提供商 — 无需 API Key(keyless fallback)
|
||||||
|
*
|
||||||
|
* <p>通过 DuckDuckGo HTML 端点实现免 key 搜索。
|
||||||
|
* 作为零配置兜底方案,结果质量不如 Serper/Tavily 但"总比没有强"。
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class DuckDuckGoSearchProvider implements SearchProvider {
|
||||||
|
|
||||||
|
private static final String SEARCH_URL = "https://html.duckduckgo.com/html/";
|
||||||
|
|
||||||
|
// 匹配 DuckDuckGo HTML 搜索结果
|
||||||
|
private static final Pattern RESULT_PATTERN = Pattern.compile(
|
||||||
|
"<a[^>]+class=\"result__a\"[^>]+href=\"([^\"]+)\"[^>]*>(.+?)</a>",
|
||||||
|
Pattern.DOTALL);
|
||||||
|
private static final Pattern SNIPPET_PATTERN = Pattern.compile(
|
||||||
|
"<a[^>]+class=\"result__snippet\"[^>]*>(.+?)</a>",
|
||||||
|
Pattern.DOTALL);
|
||||||
|
private static final Pattern RESULT_BLOCK_PATTERN = Pattern.compile(
|
||||||
|
"<div[^>]+class=\"result results_links[^\"]*\"[^>]*>(.*?)</div>\\s*(?=<div[^>]+class=\"result |$)",
|
||||||
|
Pattern.DOTALL);
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String id() {
|
||||||
|
return "duckduckgo";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String label() {
|
||||||
|
return "DuckDuckGo";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean requiresCredential() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int autoDetectOrder() {
|
||||||
|
return 100;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable(SystemSettingsDTO config) {
|
||||||
|
// 通过 duckduckgoEnabled 控制,默认启用
|
||||||
|
return !Boolean.FALSE.equals(config.getDuckduckgoEnabled());
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<SearchResult> search(String query, SystemSettingsDTO config) {
|
||||||
|
String encoded = URLEncoder.encode(query, StandardCharsets.UTF_8);
|
||||||
|
|
||||||
|
// DuckDuckGo 在部分网络环境下可能出现 SSLHandshakeException,加一次重试
|
||||||
|
String response = null;
|
||||||
|
int maxAttempts = 2;
|
||||||
|
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||||
|
try {
|
||||||
|
response = HttpUtil.createPost(SEARCH_URL)
|
||||||
|
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36")
|
||||||
|
.header("Content-Type", "application/x-www-form-urlencoded")
|
||||||
|
.header("Accept", "text/html")
|
||||||
|
.header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8")
|
||||||
|
.body("q=" + encoded)
|
||||||
|
.timeout(15000)
|
||||||
|
.execute()
|
||||||
|
.body();
|
||||||
|
break; // 成功则跳出
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("DuckDuckGo 请求失败 (attempt {}/{}): {}", attempt, maxAttempts, e.getMessage());
|
||||||
|
if (attempt == maxAttempts) {
|
||||||
|
throw e; // 最后一次仍失败则抛出,由 WebSearchService 捕获
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
Thread.sleep(1000); // 短暂等待后重试
|
||||||
|
} catch (InterruptedException ie) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
throw new RuntimeException("DuckDuckGo 搜索被中断", ie);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug("DuckDuckGo search completed for '{}', response length: {}", query,
|
||||||
|
response != null ? response.length() : 0);
|
||||||
|
return parseHtmlResults(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SearchResult> parseHtmlResults(String html) {
|
||||||
|
List<SearchResult> results = new ArrayList<>();
|
||||||
|
if (html == null || html.isBlank()) return results;
|
||||||
|
|
||||||
|
// 提取每个结果块
|
||||||
|
Matcher titleMatcher = RESULT_PATTERN.matcher(html);
|
||||||
|
Matcher snippetMatcher = SNIPPET_PATTERN.matcher(html);
|
||||||
|
|
||||||
|
int count = 0;
|
||||||
|
while (titleMatcher.find() && count < 5) {
|
||||||
|
String rawUrl = titleMatcher.group(1);
|
||||||
|
String rawTitle = titleMatcher.group(2);
|
||||||
|
|
||||||
|
String url = decodeUrl(rawUrl);
|
||||||
|
String title = stripHtml(rawTitle);
|
||||||
|
|
||||||
|
String snippet = "";
|
||||||
|
if (snippetMatcher.find()) {
|
||||||
|
snippet = stripHtml(snippetMatcher.group(1));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (title.isBlank() && snippet.isBlank()) continue;
|
||||||
|
|
||||||
|
results.add(SearchResult.builder()
|
||||||
|
.title(title)
|
||||||
|
.url(url)
|
||||||
|
.snippet(snippet)
|
||||||
|
.source(extractDomain(url))
|
||||||
|
.providerId(id())
|
||||||
|
.build());
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.debug("DuckDuckGo 解析到 {} 条结果", results.size());
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* DuckDuckGo 返回的 URL 经过跳转编码 (//duckduckgo.com/l/?uddg=...),需要解码
|
||||||
|
*/
|
||||||
|
private String decodeUrl(String rawUrl) {
|
||||||
|
if (rawUrl == null) return "";
|
||||||
|
// DuckDuckGo 使用 redirect URL
|
||||||
|
if (rawUrl.contains("uddg=")) {
|
||||||
|
try {
|
||||||
|
String encoded = rawUrl.substring(rawUrl.indexOf("uddg=") + 5);
|
||||||
|
int ampIndex = encoded.indexOf('&');
|
||||||
|
if (ampIndex > 0) encoded = encoded.substring(0, ampIndex);
|
||||||
|
return java.net.URLDecoder.decode(encoded, StandardCharsets.UTF_8);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.debug("URL 解码失败: {}", rawUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rawUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String stripHtml(String html) {
|
||||||
|
if (html == null) return "";
|
||||||
|
return html.replaceAll("<[^>]+>", "").replaceAll("&", "&")
|
||||||
|
.replaceAll("<", "<").replaceAll(">", ">")
|
||||||
|
.replaceAll(""", "\"").replaceAll("'", "'")
|
||||||
|
.trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDomain(String url) {
|
||||||
|
try {
|
||||||
|
return URI.create(url).getHost();
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,111 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import cn.hutool.json.JSONArray;
|
||||||
|
import cn.hutool.json.JSONObject;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.net.URLEncoder;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* SearXNG 搜索提供商 — 无需 API Key,但需要 base URL(自部署实例)
|
||||||
|
*
|
||||||
|
* <p>SearXNG 是开源的元搜索引擎,可自部署。Docker 部署 MateClaw 时自动包含 SearXNG 服务,
|
||||||
|
* 默认 base URL 为 {@code http://searxng:8080}。
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class SearXNGSearchProvider implements SearchProvider {
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String id() {
|
||||||
|
return "searxng";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String label() {
|
||||||
|
return "SearXNG";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean requiresCredential() {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int autoDetectOrder() {
|
||||||
|
// SearXNG 聚合多引擎,结果质量高于 DuckDuckGo(order=100),应优先
|
||||||
|
return 50;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable(SystemSettingsDTO config) {
|
||||||
|
String baseUrl = config.getSearxngBaseUrl();
|
||||||
|
return baseUrl != null && !baseUrl.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<SearchResult> search(String query, SystemSettingsDTO config) {
|
||||||
|
String baseUrl = config.getSearxngBaseUrl();
|
||||||
|
if (baseUrl.endsWith("/")) {
|
||||||
|
baseUrl = baseUrl.substring(0, baseUrl.length() - 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
String encoded = URLEncoder.encode(query, StandardCharsets.UTF_8);
|
||||||
|
String url = baseUrl + "/search?q=" + encoded + "&format=json&categories=general&language=auto";
|
||||||
|
|
||||||
|
String response = HttpUtil.createGet(url)
|
||||||
|
.header("Accept", "application/json")
|
||||||
|
.timeout(15000)
|
||||||
|
.execute()
|
||||||
|
.body();
|
||||||
|
|
||||||
|
log.debug("SearXNG result for '{}': length={}", query, response != null ? response.length() : 0);
|
||||||
|
return parseResponse(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SearchResult> parseResponse(String response) {
|
||||||
|
List<SearchResult> results = new ArrayList<>();
|
||||||
|
if (response == null || response.isBlank()) return results;
|
||||||
|
|
||||||
|
try {
|
||||||
|
JSONObject json = JSONUtil.parseObj(response);
|
||||||
|
JSONArray items = json.getJSONArray("results");
|
||||||
|
if (items == null) return results;
|
||||||
|
|
||||||
|
int limit = Math.min(items.size(), 5);
|
||||||
|
for (int i = 0; i < limit; i++) {
|
||||||
|
JSONObject item = items.getJSONObject(i);
|
||||||
|
String itemUrl = item.getStr("url");
|
||||||
|
results.add(SearchResult.builder()
|
||||||
|
.title(item.getStr("title"))
|
||||||
|
.url(itemUrl)
|
||||||
|
.snippet(item.getStr("content"))
|
||||||
|
.source(extractDomain(itemUrl))
|
||||||
|
.date(item.getStr("publishedDate"))
|
||||||
|
.providerId(id())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("SearXNG 结果解析失败: {}", e.getMessage());
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDomain(String url) {
|
||||||
|
try {
|
||||||
|
return URI.create(url).getHost();
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,45 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索提供商接口 — 所有搜索 provider(含 keyless)统一实现此接口
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
public interface SearchProvider {
|
||||||
|
|
||||||
|
/** 提供商唯一 ID,如 "serper"、"tavily"、"duckduckgo"、"searxng" */
|
||||||
|
String id();
|
||||||
|
|
||||||
|
/** 显示名称 */
|
||||||
|
String label();
|
||||||
|
|
||||||
|
/** 是否需要 API Key / Credential */
|
||||||
|
boolean requiresCredential();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 自动探测排序优先级(升序)。
|
||||||
|
* <p>有 key 的 provider 用较大值(300+),keyless 用较小值(100+),
|
||||||
|
* 但实际调度时"有 credential 的 provider"优先于 keyless,
|
||||||
|
* 此字段仅在同类 provider 之间排序。
|
||||||
|
*/
|
||||||
|
int autoDetectOrder();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 判断该 provider 在当前配置下是否可用
|
||||||
|
* (key 已配置 / keyless provider 已启用等)
|
||||||
|
*/
|
||||||
|
boolean isAvailable(SystemSettingsDTO config);
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 执行搜索,返回结构化结果列表
|
||||||
|
*
|
||||||
|
* @param query 搜索关键词
|
||||||
|
* @param config 系统配置(包含 key、baseUrl 等)
|
||||||
|
* @return 搜索结果列表;不应返回 null,失败时抛异常
|
||||||
|
*/
|
||||||
|
List<SearchResult> search(String query, SystemSettingsDTO config);
|
||||||
|
}
|
||||||
@ -0,0 +1,102 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.util.Comparator;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 搜索提供商注册表 — 收集所有 {@link SearchProvider} 实现,提供优先级排序与自动探测
|
||||||
|
*
|
||||||
|
* <p>借鉴 openclaw 的 provider auto-detect 机制:
|
||||||
|
* <ol>
|
||||||
|
* <li>用户显式配置的 primary provider → 直接使用</li>
|
||||||
|
* <li>按 autoDetectOrder 遍历,优先选有 credential 的 provider</li>
|
||||||
|
* <li>如果没有有 credential 的 provider,回退到第一个可用的 keyless provider</li>
|
||||||
|
* </ol>
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class SearchProviderRegistry {
|
||||||
|
|
||||||
|
private final List<SearchProvider> sortedProviders;
|
||||||
|
private final Map<String, SearchProvider> providerMap;
|
||||||
|
|
||||||
|
public SearchProviderRegistry(List<SearchProvider> providers) {
|
||||||
|
this.sortedProviders = providers.stream()
|
||||||
|
.sorted(Comparator.comparingInt(SearchProvider::autoDetectOrder))
|
||||||
|
.toList();
|
||||||
|
this.providerMap = providers.stream()
|
||||||
|
.collect(Collectors.toMap(SearchProvider::id, Function.identity()));
|
||||||
|
log.info("注册搜索提供商 {} 个: {}", sortedProviders.size(),
|
||||||
|
sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 按 ID 获取指定 provider */
|
||||||
|
public SearchProvider getById(String id) {
|
||||||
|
return providerMap.get(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 获取按 autoDetectOrder 排序的全部 provider 列表 */
|
||||||
|
public List<SearchProvider> allSorted() {
|
||||||
|
return sortedProviders;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 根据当前配置,解析应使用的 provider。
|
||||||
|
*
|
||||||
|
* <p>解析策略(借鉴 openclaw resolveWebSearchProviderId):
|
||||||
|
* <ol>
|
||||||
|
* <li>如果用户配置了 primary provider 且该 provider 可用 → 选中</li>
|
||||||
|
* <li>否则按 autoDetectOrder 遍历,跳过 keyless,先找有 credential 的</li>
|
||||||
|
* <li>如果没找到 → 回退到第一个可用的 keyless provider</li>
|
||||||
|
* </ol>
|
||||||
|
*
|
||||||
|
* @return 选中的 provider,或 null(完全无可用 provider)
|
||||||
|
*/
|
||||||
|
public ResolvedProvider resolve(SystemSettingsDTO config) {
|
||||||
|
// 1. 用户显式配置的 primary provider
|
||||||
|
String configuredId = config.getSearchProvider();
|
||||||
|
if (configuredId != null && !configuredId.isBlank()) {
|
||||||
|
SearchProvider configured = providerMap.get(configuredId);
|
||||||
|
if (configured != null && configured.isAvailable(config)) {
|
||||||
|
return new ResolvedProvider(configured, "configured");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. 按优先级遍历,先找有 credential 的
|
||||||
|
SearchProvider keylessFallback = null;
|
||||||
|
for (SearchProvider p : sortedProviders) {
|
||||||
|
if (!p.requiresCredential()) {
|
||||||
|
// 记住第一个可用的 keyless provider
|
||||||
|
if (keylessFallback == null && p.isAvailable(config)) {
|
||||||
|
keylessFallback = p;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (p.isAvailable(config)) {
|
||||||
|
return new ResolvedProvider(p, "auto-detect");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. 回退到 keyless
|
||||||
|
if (keylessFallback != null) {
|
||||||
|
return new ResolvedProvider(keylessFallback, "keyless-fallback");
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解析结果
|
||||||
|
*/
|
||||||
|
public record ResolvedProvider(SearchProvider provider, String source) {
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,54 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import lombok.Builder;
|
||||||
|
import lombok.Data;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 统一搜索结果结构 — 所有 provider 返回此格式
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Data
|
||||||
|
@Builder
|
||||||
|
public class SearchResult {
|
||||||
|
|
||||||
|
/** 结果标题 */
|
||||||
|
private String title;
|
||||||
|
|
||||||
|
/** 结果链接 */
|
||||||
|
private String url;
|
||||||
|
|
||||||
|
/** 摘要/片段 */
|
||||||
|
private String snippet;
|
||||||
|
|
||||||
|
/** 来源域名 (如 "reuters.com") */
|
||||||
|
private String source;
|
||||||
|
|
||||||
|
/** 发布时间(原始字符串,尽可能保留) */
|
||||||
|
private String date;
|
||||||
|
|
||||||
|
/** 提供该结果的 provider id */
|
||||||
|
private String providerId;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 格式化为 Markdown 行,用于返回给 LLM
|
||||||
|
*/
|
||||||
|
public String toMarkdown() {
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("**").append(title != null ? title : "Untitled").append("**");
|
||||||
|
if (source != null || date != null) {
|
||||||
|
sb.append(" — ");
|
||||||
|
if (source != null) sb.append(source);
|
||||||
|
if (source != null && date != null) sb.append(" | ");
|
||||||
|
if (date != null) sb.append(date);
|
||||||
|
}
|
||||||
|
sb.append("\n");
|
||||||
|
if (snippet != null && !snippet.isBlank()) {
|
||||||
|
sb.append(snippet).append("\n");
|
||||||
|
}
|
||||||
|
if (url != null && !url.isBlank()) {
|
||||||
|
sb.append("🔗 ").append(url).append("\n");
|
||||||
|
}
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,105 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import cn.hutool.json.JSONArray;
|
||||||
|
import cn.hutool.json.JSONObject;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serper (Google Search) 搜索提供商 — 需要 API Key
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class SerperSearchProvider implements SearchProvider {
|
||||||
|
|
||||||
|
private static final String DEFAULT_BASE_URL = "https://google.serper.dev/search";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String id() {
|
||||||
|
return "serper";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String label() {
|
||||||
|
return "Serper (Google)";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean requiresCredential() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int autoDetectOrder() {
|
||||||
|
return 300;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable(SystemSettingsDTO config) {
|
||||||
|
String key = config.getSerperApiKey();
|
||||||
|
return key != null && !key.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<SearchResult> search(String query, SystemSettingsDTO config) {
|
||||||
|
String apiKey = config.getSerperApiKey();
|
||||||
|
String baseUrl = config.getSerperBaseUrl();
|
||||||
|
if (baseUrl == null || baseUrl.isBlank()) {
|
||||||
|
baseUrl = DEFAULT_BASE_URL;
|
||||||
|
}
|
||||||
|
|
||||||
|
String body = JSONUtil.toJsonStr(new JSONObject().set("q", query).set("num", 5));
|
||||||
|
String response = HttpUtil.createPost(baseUrl)
|
||||||
|
.header("X-API-KEY", apiKey)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(body)
|
||||||
|
.timeout(15000)
|
||||||
|
.execute()
|
||||||
|
.body();
|
||||||
|
|
||||||
|
log.debug("Serper result for '{}': {}", query, response);
|
||||||
|
return parseResponse(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SearchResult> parseResponse(String response) {
|
||||||
|
List<SearchResult> results = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
JSONObject json = JSONUtil.parseObj(response);
|
||||||
|
JSONArray organic = json.getJSONArray("organic");
|
||||||
|
if (organic == null) return results;
|
||||||
|
|
||||||
|
for (int i = 0; i < organic.size(); i++) {
|
||||||
|
JSONObject item = organic.getJSONObject(i);
|
||||||
|
String url = item.getStr("link");
|
||||||
|
results.add(SearchResult.builder()
|
||||||
|
.title(item.getStr("title"))
|
||||||
|
.url(url)
|
||||||
|
.snippet(item.getStr("snippet"))
|
||||||
|
.source(extractDomain(url))
|
||||||
|
.date(item.getStr("date"))
|
||||||
|
.providerId(id())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Serper 结果解析失败: {}", e.getMessage());
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDomain(String url) {
|
||||||
|
try {
|
||||||
|
return URI.create(url).getHost();
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,107 @@
|
|||||||
|
package vip.mate.tool.search;
|
||||||
|
|
||||||
|
import cn.hutool.http.HttpUtil;
|
||||||
|
import cn.hutool.json.JSONArray;
|
||||||
|
import cn.hutool.json.JSONObject;
|
||||||
|
import cn.hutool.json.JSONUtil;
|
||||||
|
import lombok.extern.slf4j.Slf4j;
|
||||||
|
import org.springframework.stereotype.Component;
|
||||||
|
import vip.mate.system.model.SystemSettingsDTO;
|
||||||
|
|
||||||
|
import java.net.URI;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tavily 搜索提供商 — 需要 API Key
|
||||||
|
*
|
||||||
|
* @author MateClaw Team
|
||||||
|
*/
|
||||||
|
@Slf4j
|
||||||
|
@Component
|
||||||
|
public class TavilySearchProvider implements SearchProvider {
|
||||||
|
|
||||||
|
private static final String DEFAULT_BASE_URL = "https://api.tavily.com/search";
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String id() {
|
||||||
|
return "tavily";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String label() {
|
||||||
|
return "Tavily";
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean requiresCredential() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public int autoDetectOrder() {
|
||||||
|
return 400;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public boolean isAvailable(SystemSettingsDTO config) {
|
||||||
|
String key = config.getTavilyApiKey();
|
||||||
|
return key != null && !key.isBlank();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public List<SearchResult> search(String query, SystemSettingsDTO config) {
|
||||||
|
String apiKey = config.getTavilyApiKey();
|
||||||
|
String baseUrl = config.getTavilyBaseUrl();
|
||||||
|
if (baseUrl == null || baseUrl.isBlank()) {
|
||||||
|
baseUrl = DEFAULT_BASE_URL;
|
||||||
|
}
|
||||||
|
|
||||||
|
String body = JSONUtil.toJsonStr(new JSONObject()
|
||||||
|
.set("query", query)
|
||||||
|
.set("max_results", 5)
|
||||||
|
.set("api_key", apiKey));
|
||||||
|
String response = HttpUtil.createPost(baseUrl)
|
||||||
|
.header("Content-Type", "application/json")
|
||||||
|
.body(body)
|
||||||
|
.timeout(15000)
|
||||||
|
.execute()
|
||||||
|
.body();
|
||||||
|
|
||||||
|
log.debug("Tavily result for '{}': {}", query, response);
|
||||||
|
return parseResponse(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<SearchResult> parseResponse(String response) {
|
||||||
|
List<SearchResult> results = new ArrayList<>();
|
||||||
|
try {
|
||||||
|
JSONObject json = JSONUtil.parseObj(response);
|
||||||
|
JSONArray items = json.getJSONArray("results");
|
||||||
|
if (items == null) return results;
|
||||||
|
|
||||||
|
for (int i = 0; i < items.size(); i++) {
|
||||||
|
JSONObject item = items.getJSONObject(i);
|
||||||
|
String url = item.getStr("url");
|
||||||
|
results.add(SearchResult.builder()
|
||||||
|
.title(item.getStr("title"))
|
||||||
|
.url(url)
|
||||||
|
.snippet(item.getStr("content"))
|
||||||
|
.source(extractDomain(url))
|
||||||
|
.date(item.getStr("published_date"))
|
||||||
|
.providerId(id())
|
||||||
|
.build());
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Tavily 结果解析失败: {}", e.getMessage());
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String extractDomain(String url) {
|
||||||
|
try {
|
||||||
|
return URI.create(url).getHost();
|
||||||
|
} catch (Exception e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -294,6 +294,14 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre
|
|||||||
KEY (id)
|
KEY (id)
|
||||||
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW());
|
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW());
|
||||||
|
|
||||||
|
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
KEY (id)
|
||||||
|
VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo keyless search fallback (zero-config)', NOW(), NOW());
|
||||||
|
|
||||||
|
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
KEY (id)
|
||||||
|
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW());
|
||||||
|
|
||||||
-- Built-in tool: Date & Time
|
-- Built-in tool: Date & Time
|
||||||
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||||
KEY (id)
|
KEY (id)
|
||||||
|
|||||||
@ -296,6 +296,14 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr
|
|||||||
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW())
|
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily base URL', NOW(), NOW())
|
||||||
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
|
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo keyless search fallback (zero-config)', NOW(), NOW())
|
||||||
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
|
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW())
|
||||||
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
-- Built-in tool: Date & Time
|
-- Built-in tool: Date & Time
|
||||||
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||||
VALUES (1000000001, 'DateTimeTool', 'Date & Time', 'Get current date and time information', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
VALUES (1000000001, 'DateTimeTool', 'Date & Time', 'Get current date and time information', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
||||||
|
|||||||
@ -296,6 +296,14 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr
|
|||||||
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW())
|
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW())
|
||||||
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
|
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo 免 Key 搜索兜底(零配置可用)', NOW(), NOW())
|
||||||
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
|
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址(Docker 部署时自动配置)', NOW(), NOW())
|
||||||
|
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||||
|
|
||||||
-- 内置工具:日期时间
|
-- 内置工具:日期时间
|
||||||
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||||
VALUES (1000000001, 'DateTimeTool', '日期时间', '获取当前日期和时间信息', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
VALUES (1000000001, 'DateTimeTool', '日期时间', '获取当前日期和时间信息', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
||||||
|
|||||||
@ -300,6 +300,14 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre
|
|||||||
KEY (id)
|
KEY (id)
|
||||||
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW());
|
VALUES (1000000011, 'tavilyBaseUrl', 'https://api.tavily.com/search', 'Tavily 接口地址', NOW(), NOW());
|
||||||
|
|
||||||
|
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
KEY (id)
|
||||||
|
VALUES (1000000012, 'duckduckgoEnabled', 'true', 'DuckDuckGo 免 Key 搜索兜底(零配置可用)', NOW(), NOW());
|
||||||
|
|
||||||
|
MERGE INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||||
|
KEY (id)
|
||||||
|
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址(Docker 部署时自动配置)', NOW(), NOW());
|
||||||
|
|
||||||
-- 内置工具:日期时间
|
-- 内置工具:日期时间
|
||||||
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||||
KEY (id)
|
KEY (id)
|
||||||
|
|||||||
@ -330,6 +330,8 @@ export default {
|
|||||||
serperBaseUrl: 'Serper Base URL',
|
serperBaseUrl: 'Serper Base URL',
|
||||||
tavilyApiKey: 'Tavily API Key',
|
tavilyApiKey: 'Tavily API Key',
|
||||||
tavilyBaseUrl: 'Tavily Base URL',
|
tavilyBaseUrl: 'Tavily Base URL',
|
||||||
|
duckduckgoEnabled: 'DuckDuckGo (Keyless)',
|
||||||
|
searxngBaseUrl: 'SearXNG Base URL',
|
||||||
},
|
},
|
||||||
hints: {
|
hints: {
|
||||||
provider: 'Current implementation applies DashScope model options at runtime.',
|
provider: 'Current implementation applies DashScope model options at runtime.',
|
||||||
@ -343,6 +345,8 @@ export default {
|
|||||||
serperBaseUrl: 'Usually no need to change unless using a custom proxy.',
|
serperBaseUrl: 'Usually no need to change unless using a custom proxy.',
|
||||||
tavilyApiKey: 'API key for Tavily search, get it from tavily.com.',
|
tavilyApiKey: 'API key for Tavily search, get it from tavily.com.',
|
||||||
tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.',
|
tavilyBaseUrl: 'Usually no need to change unless using a custom proxy.',
|
||||||
|
duckduckgoEnabled: 'Free keyless search fallback. No API key required. Enabled by default as a zero-config fallback.',
|
||||||
|
searxngBaseUrl: 'Self-hosted SearXNG instance URL. Auto-configured when deploying via Docker.',
|
||||||
},
|
},
|
||||||
searchTitle: 'Search Service',
|
searchTitle: 'Search Service',
|
||||||
searchDesc: 'Configure the built-in search tool provider and API credentials',
|
searchDesc: 'Configure the built-in search tool provider and API credentials',
|
||||||
|
|||||||
@ -330,6 +330,8 @@ export default {
|
|||||||
serperBaseUrl: 'Serper 接口地址',
|
serperBaseUrl: 'Serper 接口地址',
|
||||||
tavilyApiKey: 'Tavily API Key',
|
tavilyApiKey: 'Tavily API Key',
|
||||||
tavilyBaseUrl: 'Tavily 接口地址',
|
tavilyBaseUrl: 'Tavily 接口地址',
|
||||||
|
duckduckgoEnabled: 'DuckDuckGo(免 Key)',
|
||||||
|
searxngBaseUrl: 'SearXNG 地址',
|
||||||
},
|
},
|
||||||
hints: {
|
hints: {
|
||||||
provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。',
|
provider: '当前版本会把 DashScope 模型参数真实应用到 Agent 调用链路。',
|
||||||
@ -343,6 +345,8 @@ export default {
|
|||||||
serperBaseUrl: '通常无需修改,除非使用自定义代理地址。',
|
serperBaseUrl: '通常无需修改,除非使用自定义代理地址。',
|
||||||
tavilyApiKey: '用于 Tavily 搜索服务,从 tavily.com 获取。',
|
tavilyApiKey: '用于 Tavily 搜索服务,从 tavily.com 获取。',
|
||||||
tavilyBaseUrl: '通常无需修改,除非使用自定义代理地址。',
|
tavilyBaseUrl: '通常无需修改,除非使用自定义代理地址。',
|
||||||
|
duckduckgoEnabled: '免费搜索兜底,无需 API Key。默认开启,作为零配置下的搜索降级方案。',
|
||||||
|
searxngBaseUrl: '自部署 SearXNG 实例地址。Docker 部署时自动配置。',
|
||||||
},
|
},
|
||||||
searchTitle: '搜索服务',
|
searchTitle: '搜索服务',
|
||||||
searchDesc: '配置内置搜索工具的提供商与 API 凭证',
|
searchDesc: '配置内置搜索工具的提供商与 API 凭证',
|
||||||
|
|||||||
@ -499,6 +499,9 @@ export interface SystemSettings {
|
|||||||
tavilyBaseUrl: string
|
tavilyBaseUrl: string
|
||||||
serperApiKeyMasked?: string
|
serperApiKeyMasked?: string
|
||||||
tavilyApiKeyMasked?: string
|
tavilyApiKeyMasked?: string
|
||||||
|
// Keyless 搜索 provider
|
||||||
|
duckduckgoEnabled: boolean
|
||||||
|
searxngBaseUrl: string
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ProviderModelInfo {
|
export interface ProviderModelInfo {
|
||||||
|
|||||||
@ -159,6 +159,36 @@
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<!-- Keyless Provider 配置 -->
|
||||||
|
<div class="setting-item">
|
||||||
|
<div class="setting-info">
|
||||||
|
<div class="setting-label">{{ t('settings.fields.duckduckgoEnabled') }}</div>
|
||||||
|
<div class="setting-hint">{{ t('settings.hints.duckduckgoEnabled') }}</div>
|
||||||
|
</div>
|
||||||
|
<div class="setting-control">
|
||||||
|
<label class="toggle-switch">
|
||||||
|
<input v-model="settings.duckduckgoEnabled" type="checkbox" :disabled="!settings.searchEnabled" />
|
||||||
|
<span class="toggle-slider"></span>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="setting-item setting-item-vertical">
|
||||||
|
<div class="setting-info">
|
||||||
|
<div class="setting-label">{{ t('settings.fields.searxngBaseUrl') }}</div>
|
||||||
|
<div class="setting-hint">{{ t('settings.hints.searxngBaseUrl') }}</div>
|
||||||
|
</div>
|
||||||
|
<div class="setting-control-full">
|
||||||
|
<input
|
||||||
|
v-model="settings.searxngBaseUrl"
|
||||||
|
type="text"
|
||||||
|
class="form-input"
|
||||||
|
placeholder="http://searxng:8080"
|
||||||
|
:disabled="!settings.searchEnabled"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="save-bar">
|
<div class="save-bar">
|
||||||
@ -193,6 +223,8 @@ const settings = reactive<SystemSettings>({
|
|||||||
searchFallbackEnabled: false,
|
searchFallbackEnabled: false,
|
||||||
serperBaseUrl: 'https://google.serper.dev/search',
|
serperBaseUrl: 'https://google.serper.dev/search',
|
||||||
tavilyBaseUrl: 'https://api.tavily.com/search',
|
tavilyBaseUrl: 'https://api.tavily.com/search',
|
||||||
|
duckduckgoEnabled: true,
|
||||||
|
searxngBaseUrl: '',
|
||||||
})
|
})
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user