fix(skill): align ClawHub client with actual marketplace API (issue #42)

This commit is contained in:
matevip 2026-04-30 10:25:21 +08:00
parent b40cbfb0a1
commit efbc858868
5 changed files with 318 additions and 131 deletions

View File

@ -6,7 +6,9 @@ import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Service;
import vip.mate.skill.installer.model.HubSkillInfo;
import vip.mate.skill.installer.model.SkillBundle;
import vip.mate.skill.runtime.SkillFrontmatterParser;
import java.io.ByteArrayInputStream;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
@ -17,9 +19,17 @@ import java.util.List;
import java.util.Map;
/**
* ClawHub 市场 API 客户端
* ClawHub marketplace API client.
* <p>
* 提供 skill 搜索和 bundle 获取能力
* Talks to two endpoints:
* <ul>
* <li>{@code GET /api/v1/search?q=&limit=} returns {@code {results: [{slug, displayName, summary, version, ...}]}}</li>
* <li>{@code GET /api/v1/skills/{slug}} returns {@code {skill: {...}, latestVersion: {...}, owner: {...}}}</li>
* <li>{@code GET /api/v1/download?slug=&version=} returns the bundle as a ZIP (application/zip)</li>
* </ul>
* The bundle's SKILL.md is delivered via the ZIP, not embedded in the metadata
* response, which is why an earlier "expect a {@code content} field on the
* skill JSON" implementation always saw empty content and aborted installs.
*
* @author MateClaw Team
*/
@ -29,11 +39,15 @@ public class SkillHubClient {
private final SkillHubProperties properties;
private final ObjectMapper objectMapper;
private final SkillFrontmatterParser frontmatterParser;
private final HttpClient httpClient;
public SkillHubClient(SkillHubProperties properties, ObjectMapper objectMapper) {
public SkillHubClient(SkillHubProperties properties,
ObjectMapper objectMapper,
SkillFrontmatterParser frontmatterParser) {
this.properties = properties;
this.objectMapper = objectMapper;
this.frontmatterParser = frontmatterParser;
this.httpClient = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(properties.getHttpTimeout()))
.followRedirects(HttpClient.Redirect.NORMAL)
@ -41,7 +55,7 @@ public class SkillHubClient {
}
/**
* 搜索 ClawHub 市场
* Search the ClawHub marketplace.
*/
public List<HubSkillInfo> search(String query, int limit) {
String url = properties.getBaseUrl() + properties.getSearchPath()
@ -49,14 +63,7 @@ public class SkillHubClient {
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
.GET()
.header("Accept", "application/json")
.header("User-Agent", "MateClaw/1.0")
.build();
HttpRequest request = jsonGet(url);
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() == 200) {
@ -92,55 +99,111 @@ public class SkillHubClient {
}
/**
* 获取 skill bundle 详情
* Fetch a skill bundle from ClawHub.
* <p>
* {@link #search} 共享同一套重试策略408/429/5xx 状态码或 IO 异常时按
* 指数退避800/1600/3200ms重试最多 {@code httpRetries}
* 此外当 bundle 内容{@code content}为空时直接返回 null bundle
* 重装会清空用户的 SKILL.md是不可接受的"成功"
* Two-step fetch: (1) GET metadata to learn version + author/icon
* defaults; (2) GET the ZIP and decompress in memory. The two requests
* share the same retry policy as {@link #search}.
* <p>
* Returns {@code null} if the bundle ZIP can't be downloaded or doesn't
* contain a SKILL.md never returns a SkillBundle with empty content,
* since reinstalling that would wipe the user's local SKILL.md.
*/
public SkillBundle fetchBundle(String slug, String version) {
String path = version != null && !version.isBlank()
? "/api/v1/skills/" + slug + "/versions/" + encodeParam(version)
: "/api/v1/skills/" + slug;
String url = properties.getBaseUrl() + path;
if (slug == null || slug.isBlank()) {
log.warn("fetchBundle called with blank slug");
return null;
}
// Step 1: best-effort metadata lookup. Failure isn't fatal the ZIP
// alone is enough to install, but metadata gives us author / icon
// / latest version when SKILL.md frontmatter omits them.
HubSkillMetadata metadata = fetchMetadata(slug);
String resolvedVersion = (version != null && !version.isBlank())
? version
: (metadata != null ? metadata.version() : null);
// Step 2: download the ZIP.
byte[] zipBytes = downloadBundleZip(slug, resolvedVersion);
if (zipBytes == null) {
return null;
}
// Step 3: extract + assemble SkillBundle.
try {
ZipSkillFetcher.ExtractedSkill extracted = ZipSkillFetcher.extract(new ByteArrayInputStream(zipBytes));
var parsed = frontmatterParser.parse(extracted.skillMdContent());
Map<String, Object> fm = parsed.getFrontmatter();
String name = firstNonBlank(parsed.getName(),
metadata != null ? metadata.displayName() : null,
slug);
String description = firstNonBlank(parsed.getDescription(),
metadata != null ? metadata.summary() : null,
"");
String resolvedVer = firstNonBlank(
fm != null && fm.get("version") != null ? String.valueOf(fm.get("version")) : null,
resolvedVersion,
metadata != null ? metadata.version() : null,
"1.0.0");
String author = firstNonBlank(
fm != null && fm.get("author") != null ? String.valueOf(fm.get("author")) : null,
metadata != null ? metadata.owner() : null,
"");
String icon = firstNonBlank(
fm != null && fm.get("icon") != null ? String.valueOf(fm.get("icon")) : null,
"📦");
String sourceUrl = properties.getBaseUrl() + "/skills/" + slug
+ (resolvedVer != null && !resolvedVer.isBlank() ? "@" + resolvedVer : "");
return new SkillBundle(
name,
extracted.skillMdContent(),
extracted.references(),
extracted.scripts(),
"clawhub",
sourceUrl,
resolvedVer,
description,
author,
icon
);
} catch (Exception e) {
log.warn("Failed to parse hub bundle ZIP for '{}': {}", slug, e.getMessage());
return null;
}
}
// ==================== HTTP fetch helpers ====================
private HubSkillMetadata fetchMetadata(String slug) {
String url = properties.getBaseUrl() + properties.getSkillsPath() + "/" + encodeParam(slug);
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
.GET()
.header("Accept", "application/json")
.header("User-Agent", "MateClaw/1.0")
.build();
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
HttpResponse<String> response = httpClient.send(jsonGet(url), HttpResponse.BodyHandlers.ofString());
int status = response.statusCode();
if (status == 200) {
SkillBundle bundle = parseBundleResponse(response.body(), slug);
if (bundle == null || bundle.content() == null || bundle.content().isBlank()) {
log.warn("Hub fetchBundle returned empty content for '{}'; treat as failure to avoid wiping local SKILL.md", slug);
return null;
}
return bundle;
return parseMetadataResponse(response.body());
}
if (status == 404) {
log.info("Hub metadata not found for '{}'", slug);
return null;
}
if (isRetryable(status) && attempt < properties.getHttpRetries()) {
log.warn("Hub fetchBundle attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
log.warn("Hub metadata attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
Thread.sleep(backoffMs(attempt));
continue;
}
log.warn("Hub fetchBundle failed for '{}': status {}", slug, status);
log.warn("Hub metadata fetch failed for '{}': status {}", slug, status);
return null;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
} catch (Exception e) {
if (attempt < properties.getHttpRetries()) {
log.warn("Hub fetchBundle attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
log.warn("Hub metadata attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
try {
Thread.sleep(backoffMs(attempt));
} catch (InterruptedException ie) {
@ -148,22 +211,90 @@ public class SkillHubClient {
return null;
}
} else {
log.error("Hub fetchBundle failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
log.warn("Hub metadata fetch failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
}
}
}
return null;
}
// ==================== 内部方法 ====================
private byte[] downloadBundleZip(String slug, String version) {
StringBuilder url = new StringBuilder()
.append(properties.getBaseUrl())
.append(properties.getDownloadPath())
.append("?slug=").append(encodeParam(slug));
if (version != null && !version.isBlank()) {
url.append("&version=").append(encodeParam(version));
}
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
try {
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url.toString()))
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
.GET()
.header("Accept", "application/zip, application/octet-stream")
.header("User-Agent", "MateClaw/1.0")
.build();
HttpResponse<byte[]> response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
int status = response.statusCode();
if (status == 200) {
byte[] body = response.body();
if (body == null || body.length == 0) {
log.warn("Hub download for '{}' returned empty body; treating as failure to avoid wiping local SKILL.md", slug);
return null;
}
return body;
}
if (isRetryable(status) && attempt < properties.getHttpRetries()) {
log.warn("Hub download attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
Thread.sleep(backoffMs(attempt));
continue;
}
log.warn("Hub download failed for '{}': status {}", slug, status);
return null;
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return null;
} catch (Exception e) {
if (attempt < properties.getHttpRetries()) {
log.warn("Hub download attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
try {
Thread.sleep(backoffMs(attempt));
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
return null;
}
} else {
log.error("Hub download failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
}
}
}
return null;
}
private HttpRequest jsonGet(String url) {
return HttpRequest.newBuilder()
.uri(URI.create(url))
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
.GET()
.header("Accept", "application/json")
.header("User-Agent", "MateClaw/1.0")
.build();
}
// ==================== JSON parsing ====================
@SuppressWarnings("unchecked")
private List<HubSkillInfo> parseSearchResponse(String body) {
try {
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
Object data = json.get("data");
Object data = json.get("results");
if (data == null) {
data = json.get("results");
data = json.get("data");
}
if (data == null) {
data = json.get("skills");
@ -179,36 +310,60 @@ public class SkillHubClient {
}
}
/**
* Parse the metadata payload returned by {@code /api/v1/skills/{slug}}.
* Tolerates both the current nested shape ({@code {skill, latestVersion, owner}})
* and a flat shape (some self-hosted hubs).
*/
@SuppressWarnings("unchecked")
private SkillBundle parseBundleResponse(String body, String slug) {
private HubSkillMetadata parseMetadataResponse(String body) {
try {
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
String name = getStr(json, "name", slug);
String content = getStr(json, "content", "");
String description = getStr(json, "description", "");
String author = getStr(json, "author", "");
String version = getStr(json, "version", "1.0.0");
String icon = getStr(json, "icon", "");
Map<String, String> references = json.containsKey("references")
? objectMapper.convertValue(json.get("references"), new TypeReference<>() {})
: Map.of();
Map<String, String> scripts = json.containsKey("scripts")
? objectMapper.convertValue(json.get("scripts"), new TypeReference<>() {})
: Map.of();
Map<String, Object> skill = json.get("skill") instanceof Map<?, ?> m
? (Map<String, Object>) m
: json;
return new SkillBundle(name, content, references, scripts,
"clawhub", properties.getBaseUrl() + "/skills/" + slug,
version, description, author, icon);
String displayName = stringOf(skill.get("displayName"));
if (displayName == null) displayName = stringOf(skill.get("name"));
String summary = stringOf(skill.get("summary"));
if (summary == null) summary = stringOf(skill.get("description"));
String version = null;
if (json.get("latestVersion") instanceof Map<?, ?> lv) {
version = stringOf(((Map<String, Object>) lv).get("version"));
}
if (version == null) version = stringOf(skill.get("version"));
String owner = null;
if (json.get("owner") instanceof Map<?, ?> ownerMap) {
Map<String, Object> o = (Map<String, Object>) ownerMap;
owner = firstNonBlank(stringOf(o.get("displayName")), stringOf(o.get("handle")));
}
if (owner == null) owner = stringOf(skill.get("author"));
return new HubSkillMetadata(displayName, summary, version, owner);
} catch (Exception e) {
log.warn("Failed to parse hub bundle response: {}", e.getMessage());
log.warn("Failed to parse hub metadata response: {}", e.getMessage());
return null;
}
}
private String getStr(Map<String, Object> map, String key, String defaultVal) {
Object v = map.get(key);
return v != null ? v.toString() : defaultVal;
private record HubSkillMetadata(String displayName, String summary, String version, String owner) {}
// ==================== misc helpers ====================
private static String stringOf(Object v) {
if (v == null) return null;
String s = v.toString();
return s.isBlank() ? null : s;
}
private static String firstNonBlank(String... values) {
for (String v : values) {
if (v != null && !v.isBlank()) return v;
}
return "";
}
private boolean isRetryable(int statusCode) {

View File

@ -12,15 +12,21 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "mateclaw.skill.hub")
public class SkillHubProperties {
/** Hub 基础 URL */
/** Hub base URL. */
private String baseUrl = "https://clawhub.ai";
/** 搜索 API 路径 */
/** Search API path. */
private String searchPath = "/api/v1/search";
/** HTTP 请求超时(秒) */
/** Skill metadata API path prefix; full path is {@code <skillsPath>/<slug>}. */
private String skillsPath = "/api/v1/skills";
/** Bundle ZIP download API path; supports {@code ?slug=&version=}. */
private String downloadPath = "/api/v1/download";
/** HTTP request timeout (seconds). */
private int httpTimeout = 15;
/** HTTP 重试次数 */
/** HTTP retry count. */
private int httpRetries = 3;
}

View File

@ -15,16 +15,14 @@ import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
/**
* ZIP 格式 Skill 包解析器
* Parses a ZIP-packaged Skill into a {@link SkillBundle}.
* <p>
* 从上传的 ZIP 文件中解析 SKILL.md + references/ + scripts/
* 构建统一的 {@link SkillBundle} 供安装流程使用
* <p>
* 安全防护
* Used by both the upload endpoint (MultipartFile) and the ClawHub install
* path (downloaded ZIP bytes). Hardened against:
* <ul>
* <li>Zip Slip 路径穿越检测</li>
* <li>单文件 1MB总解压 50MB</li>
* <li>仅接受 SKILL.md / references/ / scripts/ 下的文件</li>
* <li>Zip Slip path traversal</li>
* <li>Per-file 1MB, total 50MB</li>
* <li>Only SKILL.md / references/ / scripts/ entries are kept</li>
* </ul>
*
* @author MateClaw Team
@ -38,12 +36,17 @@ public class ZipSkillFetcher {
private static final String SKILL_MD_LOWER = "skill.md";
/**
* 解析 ZIP 文件为 SkillBundle
*
* @param zipFile 上传的 ZIP 文件
* @param parser frontmatter 解析器
* @return 解析后的 SkillBundle
* @throws IOException 解析失败
* Holds the in-memory result of decompressing a ZIP. Used by callers
* that want to enrich the SkillBundle with metadata (e.g. ClawHub author
* / icon) that isn't carried inside SKILL.md.
*/
public record ExtractedSkill(String skillMdContent,
Map<String, String> references,
Map<String, String> scripts) {}
/**
* Parse an uploaded ZIP file into a SkillBundle. Source type is "zip"
* and source URL is the original filename.
*/
public static SkillBundle parse(MultipartFile zipFile, SkillFrontmatterParser parser) throws IOException {
if (zipFile == null || zipFile.isEmpty()) {
@ -53,15 +56,52 @@ public class ZipSkillFetcher {
throw new IllegalArgumentException("ZIP file too large (max 50MB)");
}
ExtractedSkill extracted;
try (InputStream is = zipFile.getInputStream()) {
extracted = extract(is);
}
var parsed = parser.parse(extracted.skillMdContent());
String name = parsed.getName();
if (name == null || name.isBlank()) {
String zipName = zipFile.getOriginalFilename();
if (zipName != null) {
name = zipName.replaceAll("\\.zip$", "").replaceAll("[^a-zA-Z0-9_-]", "-");
} else {
name = "imported-skill";
}
}
log.info("[ZipSkillFetcher] Parsed: name={}, references={}, scripts={}",
name, extracted.references().size(), extracted.scripts().size());
Map<String, Object> fm = parsed.getFrontmatter();
return new SkillBundle(
name,
extracted.skillMdContent(),
extracted.references(),
extracted.scripts(),
"zip",
zipFile.getOriginalFilename(),
fm != null ? String.valueOf(fm.getOrDefault("version", "1.0.0")) : "1.0.0",
parsed.getDescription(),
fm != null ? String.valueOf(fm.getOrDefault("author", "")) : "",
fm != null ? String.valueOf(fm.getOrDefault("icon", "📦")) : "📦"
);
}
/**
* Decompress a ZIP stream into in-memory SKILL.md + references + scripts.
* Throws {@link IllegalArgumentException} if no SKILL.md is present.
*/
public static ExtractedSkill extract(InputStream zipStream) throws IOException {
String skillMdContent = null;
String skillMdPrefix = ""; // 如果 SKILL.md 在子目录中记录前缀
String skillMdPrefix = "";
Map<String, String> references = new HashMap<>();
Map<String, String> scripts = new HashMap<>();
long totalSize = 0;
try (InputStream is = zipFile.getInputStream();
ZipInputStream zis = new ZipInputStream(is, StandardCharsets.UTF_8)) {
try (ZipInputStream zis = new ZipInputStream(zipStream, StandardCharsets.UTF_8)) {
ZipEntry entry;
while ((entry = zis.getNextEntry()) != null) {
if (entry.isDirectory()) {
@ -71,7 +111,7 @@ public class ZipSkillFetcher {
String entryName = entry.getName();
// Zip Slip 防护normalize 后检查是否逃逸
// Zip Slip guard: normalize and reject absolute / traversal entries.
Path entryPath = Path.of(entryName).normalize();
if (entryPath.isAbsolute() || entryName.contains("..")) {
log.warn("[ZipSkillFetcher] Skipping suspicious entry: {}", entryName);
@ -79,16 +119,19 @@ public class ZipSkillFetcher {
continue;
}
// 文件大小检查
long size = entry.getSize();
if (size > MAX_FILE_SIZE) {
log.warn("[ZipSkillFetcher] Skipping oversized entry: {} ({}bytes)", entryName, size);
long declaredSize = entry.getSize();
if (declaredSize > MAX_FILE_SIZE) {
log.warn("[ZipSkillFetcher] Skipping oversized entry: {} ({}bytes)", entryName, declaredSize);
zis.closeEntry();
continue;
}
// 读取内容
byte[] bytes = zis.readAllBytes();
if (bytes.length > MAX_FILE_SIZE) {
log.warn("[ZipSkillFetcher] Skipping oversized entry post-read: {} ({}bytes)", entryName, bytes.length);
zis.closeEntry();
continue;
}
totalSize += bytes.length;
if (totalSize > MAX_TOTAL_SIZE) {
throw new IOException("Total extracted size exceeds 50MB limit");
@ -97,10 +140,8 @@ public class ZipSkillFetcher {
String content = new String(bytes, StandardCharsets.UTF_8);
String fileName = entryPath.getFileName().toString();
// 定位 SKILL.md根目录或一级子目录
if (skillMdContent == null && (SKILL_MD.equals(fileName) || SKILL_MD_LOWER.equals(fileName))) {
skillMdContent = content;
// 确定子目录前缀 "my-skill/SKILL.md" prefix = "my-skill/"
int slashIdx = entryName.lastIndexOf('/');
skillMdPrefix = slashIdx > 0 ? entryName.substring(0, slashIdx + 1) : "";
log.info("[ZipSkillFetcher] Found SKILL.md at: {}", entryName);
@ -108,19 +149,16 @@ public class ZipSkillFetcher {
zis.closeEntry();
// 先收集所有文件后面按前缀过滤
String normalizedName = entryPath.toString().replace('\\', '/');
// 收集 references/ scripts/ 文件
String relativeName = normalizedName;
if (!skillMdPrefix.isEmpty() && normalizedName.startsWith(skillMdPrefix)) {
relativeName = normalizedName.substring(skillMdPrefix.length());
}
if (relativeName.startsWith("references/")) {
references.put(relativeName, content);
references.put(relativeName.substring("references/".length()), content);
} else if (relativeName.startsWith("scripts/")) {
scripts.put(relativeName, content);
scripts.put(relativeName.substring("scripts/".length()), content);
}
}
}
@ -129,33 +167,6 @@ public class ZipSkillFetcher {
throw new IllegalArgumentException("ZIP does not contain SKILL.md");
}
// 解析 frontmatter
var parsed = parser.parse(skillMdContent);
String name = parsed.getName();
if (name == null || name.isBlank()) {
// ZIP 文件名推断
String zipName = zipFile.getOriginalFilename();
if (zipName != null) {
name = zipName.replaceAll("\\.zip$", "").replaceAll("[^a-zA-Z0-9_-]", "-");
} else {
name = "imported-skill";
}
}
log.info("[ZipSkillFetcher] Parsed: name={}, references={}, scripts={}, totalSize={}",
name, references.size(), scripts.size(), totalSize);
return new SkillBundle(
name,
skillMdContent,
references,
scripts,
"zip",
zipFile.getOriginalFilename(),
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("version", "1.0.0")) : "1.0.0",
parsed.getDescription(),
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("author", "")) : "",
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("icon", "📦")) : "📦"
);
return new ExtractedSkill(skillMdContent, references, scripts);
}
}

View File

@ -1,20 +1,33 @@
package vip.mate.skill.installer.model;
import com.fasterxml.jackson.annotation.JsonAlias;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
import java.util.List;
/**
* ClawHub 市场 skill 信息
* ClawHub marketplace skill summary.
* <p>
* The hub API uses {@code displayName} / {@code summary}; older / alternative
* deployments expose {@code name} / {@code description}. {@link JsonAlias}
* keeps both wire shapes deserializing into the same fields so the UI never
* shows blank rows when the upstream renames a key.
*
* @author MateClaw Team
*/
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class HubSkillInfo {
@JsonAlias({"displayName"})
private String name;
private String slug;
@JsonAlias({"summary"})
private String description;
private String author;
private String version;
private String icon;

View File

@ -109,6 +109,8 @@ mateclaw:
hub:
base-url: https://clawhub.ai
search-path: /api/v1/search
skills-path: /api/v1/skills
download-path: /api/v1/download
http-timeout: 15
http-retries: 3
plugin: