package vip.mate.tool.search; import cn.hutool.http.HttpUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import vip.mate.system.model.SystemSettingsDTO; import java.net.URI; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; /** * DuckDuckGo 搜索提供商 — 无需 API Key(keyless fallback) * *

通过 DuckDuckGo HTML 端点实现免 key 搜索。 * 作为零配置兜底方案,结果质量不如 Serper/Tavily 但"总比没有强"。 * * @author MateClaw Team */ @Slf4j @Component public class DuckDuckGoSearchProvider implements SearchProvider { private static final String SEARCH_URL = "https://html.duckduckgo.com/html/"; // 匹配 DuckDuckGo HTML 搜索结果 private static final Pattern RESULT_PATTERN = Pattern.compile( "]+class=\"result__a\"[^>]+href=\"([^\"]+)\"[^>]*>(.+?)", Pattern.DOTALL); private static final Pattern SNIPPET_PATTERN = Pattern.compile( "]+class=\"result__snippet\"[^>]*>(.+?)", Pattern.DOTALL); private static final Pattern RESULT_BLOCK_PATTERN = Pattern.compile( "]+class=\"result results_links[^\"]*\"[^>]*>(.*?)\\s*(?=]+class=\"result |$)", Pattern.DOTALL); @Override public String id() { return "duckduckgo"; } @Override public String label() { return "DuckDuckGo"; } @Override public boolean requiresCredential() { return false; } @Override public int autoDetectOrder() { return 100; } @Override public boolean isAvailable(SystemSettingsDTO config) { // 通过 duckduckgoEnabled 控制,默认启用 return !Boolean.FALSE.equals(config.getDuckduckgoEnabled()); } @Override public List search(String query, SystemSettingsDTO config) { return search(SearchQuery.of(query), config); } @Override public List search(SearchQuery searchQuery, SystemSettingsDTO config) { String encoded = URLEncoder.encode(searchQuery.query(), StandardCharsets.UTF_8); // 构建请求 body:基础 query + 可选的 freshness 和 language 参数 StringBuilder bodyBuilder = new StringBuilder("q=").append(encoded); // DuckDuckGo freshness: df=d (day), df=w (week), df=m (month), df=y (year) if (searchQuery.hasFreshness()) { String df = mapFreshnessToDf(searchQuery.freshness()); if (df != null) bodyBuilder.append("&df=").append(df); } // DuckDuckGo region: kl 参数(如 cn-zh, us-en, jp-jp) if (searchQuery.hasLanguage()) { String kl = mapLanguageToKl(searchQuery.language()); if (kl != null) bodyBuilder.append("&kl=").append(kl); } // DuckDuckGo 在部分网络环境下可能出现 SSLHandshakeException,加一次重试 String response = null; int maxAttempts = 2; for (int attempt = 1; attempt <= maxAttempts; attempt++) { try { response = HttpUtil.createPost(SEARCH_URL) .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") .header("Content-Type", "application/x-www-form-urlencoded") .header("Accept", "text/html") .header("Accept-Language", "en-US,en;q=0.9,zh-CN;q=0.8") .body(bodyBuilder.toString()) .timeout(15000) .execute() .body(); break; } catch (Exception e) { log.warn("DuckDuckGo 请求失败 (attempt {}/{}): {}", attempt, maxAttempts, e.getMessage()); if (attempt == maxAttempts) { throw e; } try { Thread.sleep(1000); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); throw new RuntimeException("DuckDuckGo 搜索被中断", ie); } } } log.debug("DuckDuckGo search completed for '{}', response length: {}", searchQuery.query(), response != null ? response.length() : 0); return parseHtmlResults(response, searchQuery.resolvedCount()); } private String mapFreshnessToDf(String freshness) { return switch (freshness.toLowerCase()) { case "day" -> "d"; case "week" -> "w"; case "month" -> "m"; case "year" -> "y"; default -> null; }; } private String mapLanguageToKl(String language) { String lang = language.toLowerCase(); if (lang.startsWith("zh")) return "cn-zh"; if (lang.startsWith("en")) return "us-en"; if (lang.startsWith("ja")) return "jp-jp"; return null; } private List parseHtmlResults(String html, int limit) { List results = new ArrayList<>(); if (html == null || html.isBlank()) return results; // 提取每个结果块 Matcher titleMatcher = RESULT_PATTERN.matcher(html); Matcher snippetMatcher = SNIPPET_PATTERN.matcher(html); int count = 0; while (titleMatcher.find() && count < limit) { String rawUrl = titleMatcher.group(1); String rawTitle = titleMatcher.group(2); String url = decodeUrl(rawUrl); String title = stripHtml(rawTitle); String snippet = ""; if (snippetMatcher.find()) { snippet = stripHtml(snippetMatcher.group(1)); } if (title.isBlank() && snippet.isBlank()) continue; results.add(SearchResult.builder() .title(title) .url(url) .snippet(snippet) .source(extractDomain(url)) .providerId(id()) .build()); count++; } log.debug("DuckDuckGo 解析到 {} 条结果", results.size()); return results; } /** * DuckDuckGo 返回的 URL 经过跳转编码 (//duckduckgo.com/l/?uddg=...),需要解码 */ private String decodeUrl(String rawUrl) { if (rawUrl == null) return ""; // DuckDuckGo 使用 redirect URL if (rawUrl.contains("uddg=")) { try { String encoded = rawUrl.substring(rawUrl.indexOf("uddg=") + 5); int ampIndex = encoded.indexOf('&'); if (ampIndex > 0) encoded = encoded.substring(0, ampIndex); return java.net.URLDecoder.decode(encoded, StandardCharsets.UTF_8); } catch (Exception e) { log.debug("URL 解码失败: {}", rawUrl); } } return rawUrl; } private String stripHtml(String html) { if (html == null) return ""; return html.replaceAll("<[^>]+>", "").replaceAll("&", "&") .replaceAll("<", "<").replaceAll(">", ">") .replaceAll(""", "\"").replaceAll("'", "'") .trim(); } private String extractDomain(String url) { try { return URI.create(url).getHost(); } catch (Exception e) { return null; } } }