mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-15 03:55:09 +08:00
fix(skill): align ClawHub client with actual marketplace API (issue #42)
This commit is contained in:
parent
b40cbfb0a1
commit
efbc858868
@ -6,7 +6,9 @@ import lombok.extern.slf4j.Slf4j;
|
|||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
import vip.mate.skill.installer.model.HubSkillInfo;
|
import vip.mate.skill.installer.model.HubSkillInfo;
|
||||||
import vip.mate.skill.installer.model.SkillBundle;
|
import vip.mate.skill.installer.model.SkillBundle;
|
||||||
|
import vip.mate.skill.runtime.SkillFrontmatterParser;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.net.http.HttpClient;
|
import java.net.http.HttpClient;
|
||||||
import java.net.http.HttpRequest;
|
import java.net.http.HttpRequest;
|
||||||
@ -17,9 +19,17 @@ import java.util.List;
|
|||||||
import java.util.Map;
|
import java.util.Map;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ClawHub 市场 API 客户端
|
* ClawHub marketplace API client.
|
||||||
* <p>
|
* <p>
|
||||||
* 提供 skill 搜索和 bundle 获取能力。
|
* Talks to two endpoints:
|
||||||
|
* <ul>
|
||||||
|
* <li>{@code GET /api/v1/search?q=&limit=} — returns {@code {results: [{slug, displayName, summary, version, ...}]}}</li>
|
||||||
|
* <li>{@code GET /api/v1/skills/{slug}} — returns {@code {skill: {...}, latestVersion: {...}, owner: {...}}}</li>
|
||||||
|
* <li>{@code GET /api/v1/download?slug=&version=} — returns the bundle as a ZIP (application/zip)</li>
|
||||||
|
* </ul>
|
||||||
|
* The bundle's SKILL.md is delivered via the ZIP, not embedded in the metadata
|
||||||
|
* response, which is why an earlier "expect a {@code content} field on the
|
||||||
|
* skill JSON" implementation always saw empty content and aborted installs.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@ -29,11 +39,15 @@ public class SkillHubClient {
|
|||||||
|
|
||||||
private final SkillHubProperties properties;
|
private final SkillHubProperties properties;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
|
private final SkillFrontmatterParser frontmatterParser;
|
||||||
private final HttpClient httpClient;
|
private final HttpClient httpClient;
|
||||||
|
|
||||||
public SkillHubClient(SkillHubProperties properties, ObjectMapper objectMapper) {
|
public SkillHubClient(SkillHubProperties properties,
|
||||||
|
ObjectMapper objectMapper,
|
||||||
|
SkillFrontmatterParser frontmatterParser) {
|
||||||
this.properties = properties;
|
this.properties = properties;
|
||||||
this.objectMapper = objectMapper;
|
this.objectMapper = objectMapper;
|
||||||
|
this.frontmatterParser = frontmatterParser;
|
||||||
this.httpClient = HttpClient.newBuilder()
|
this.httpClient = HttpClient.newBuilder()
|
||||||
.connectTimeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
.connectTimeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
||||||
.followRedirects(HttpClient.Redirect.NORMAL)
|
.followRedirects(HttpClient.Redirect.NORMAL)
|
||||||
@ -41,7 +55,7 @@ public class SkillHubClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 搜索 ClawHub 市场
|
* Search the ClawHub marketplace.
|
||||||
*/
|
*/
|
||||||
public List<HubSkillInfo> search(String query, int limit) {
|
public List<HubSkillInfo> search(String query, int limit) {
|
||||||
String url = properties.getBaseUrl() + properties.getSearchPath()
|
String url = properties.getBaseUrl() + properties.getSearchPath()
|
||||||
@ -49,14 +63,7 @@ public class SkillHubClient {
|
|||||||
|
|
||||||
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
|
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
|
||||||
try {
|
try {
|
||||||
HttpRequest request = HttpRequest.newBuilder()
|
HttpRequest request = jsonGet(url);
|
||||||
.uri(URI.create(url))
|
|
||||||
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
|
||||||
.GET()
|
|
||||||
.header("Accept", "application/json")
|
|
||||||
.header("User-Agent", "MateClaw/1.0")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
||||||
|
|
||||||
if (response.statusCode() == 200) {
|
if (response.statusCode() == 200) {
|
||||||
@ -92,55 +99,111 @@ public class SkillHubClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取 skill bundle 详情
|
* Fetch a skill bundle from ClawHub.
|
||||||
* <p>
|
* <p>
|
||||||
* 与 {@link #search} 共享同一套重试策略:408/429/5xx 状态码或 IO 异常时按
|
* Two-step fetch: (1) GET metadata to learn version + author/icon
|
||||||
* 指数退避(800/1600/3200ms)重试,最多 {@code httpRetries} 次。
|
* defaults; (2) GET the ZIP and decompress in memory. The two requests
|
||||||
* 此外当 bundle 内容({@code content})为空时直接返回 null —— 空 bundle
|
* share the same retry policy as {@link #search}.
|
||||||
* 重装会清空用户的 SKILL.md,是不可接受的"成功"。
|
* <p>
|
||||||
|
* Returns {@code null} if the bundle ZIP can't be downloaded or doesn't
|
||||||
|
* contain a SKILL.md — never returns a SkillBundle with empty content,
|
||||||
|
* since reinstalling that would wipe the user's local SKILL.md.
|
||||||
*/
|
*/
|
||||||
public SkillBundle fetchBundle(String slug, String version) {
|
public SkillBundle fetchBundle(String slug, String version) {
|
||||||
String path = version != null && !version.isBlank()
|
if (slug == null || slug.isBlank()) {
|
||||||
? "/api/v1/skills/" + slug + "/versions/" + encodeParam(version)
|
log.warn("fetchBundle called with blank slug");
|
||||||
: "/api/v1/skills/" + slug;
|
return null;
|
||||||
String url = properties.getBaseUrl() + path;
|
}
|
||||||
|
|
||||||
|
// Step 1: best-effort metadata lookup. Failure isn't fatal — the ZIP
|
||||||
|
// alone is enough to install, but metadata gives us author / icon
|
||||||
|
// / latest version when SKILL.md frontmatter omits them.
|
||||||
|
HubSkillMetadata metadata = fetchMetadata(slug);
|
||||||
|
String resolvedVersion = (version != null && !version.isBlank())
|
||||||
|
? version
|
||||||
|
: (metadata != null ? metadata.version() : null);
|
||||||
|
|
||||||
|
// Step 2: download the ZIP.
|
||||||
|
byte[] zipBytes = downloadBundleZip(slug, resolvedVersion);
|
||||||
|
if (zipBytes == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 3: extract + assemble SkillBundle.
|
||||||
|
try {
|
||||||
|
ZipSkillFetcher.ExtractedSkill extracted = ZipSkillFetcher.extract(new ByteArrayInputStream(zipBytes));
|
||||||
|
|
||||||
|
var parsed = frontmatterParser.parse(extracted.skillMdContent());
|
||||||
|
Map<String, Object> fm = parsed.getFrontmatter();
|
||||||
|
|
||||||
|
String name = firstNonBlank(parsed.getName(),
|
||||||
|
metadata != null ? metadata.displayName() : null,
|
||||||
|
slug);
|
||||||
|
String description = firstNonBlank(parsed.getDescription(),
|
||||||
|
metadata != null ? metadata.summary() : null,
|
||||||
|
"");
|
||||||
|
String resolvedVer = firstNonBlank(
|
||||||
|
fm != null && fm.get("version") != null ? String.valueOf(fm.get("version")) : null,
|
||||||
|
resolvedVersion,
|
||||||
|
metadata != null ? metadata.version() : null,
|
||||||
|
"1.0.0");
|
||||||
|
String author = firstNonBlank(
|
||||||
|
fm != null && fm.get("author") != null ? String.valueOf(fm.get("author")) : null,
|
||||||
|
metadata != null ? metadata.owner() : null,
|
||||||
|
"");
|
||||||
|
String icon = firstNonBlank(
|
||||||
|
fm != null && fm.get("icon") != null ? String.valueOf(fm.get("icon")) : null,
|
||||||
|
"📦");
|
||||||
|
|
||||||
|
String sourceUrl = properties.getBaseUrl() + "/skills/" + slug
|
||||||
|
+ (resolvedVer != null && !resolvedVer.isBlank() ? "@" + resolvedVer : "");
|
||||||
|
|
||||||
|
return new SkillBundle(
|
||||||
|
name,
|
||||||
|
extracted.skillMdContent(),
|
||||||
|
extracted.references(),
|
||||||
|
extracted.scripts(),
|
||||||
|
"clawhub",
|
||||||
|
sourceUrl,
|
||||||
|
resolvedVer,
|
||||||
|
description,
|
||||||
|
author,
|
||||||
|
icon
|
||||||
|
);
|
||||||
|
} catch (Exception e) {
|
||||||
|
log.warn("Failed to parse hub bundle ZIP for '{}': {}", slug, e.getMessage());
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== HTTP fetch helpers ====================
|
||||||
|
|
||||||
|
private HubSkillMetadata fetchMetadata(String slug) {
|
||||||
|
String url = properties.getBaseUrl() + properties.getSkillsPath() + "/" + encodeParam(slug);
|
||||||
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
|
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
|
||||||
try {
|
try {
|
||||||
HttpRequest request = HttpRequest.newBuilder()
|
HttpResponse<String> response = httpClient.send(jsonGet(url), HttpResponse.BodyHandlers.ofString());
|
||||||
.uri(URI.create(url))
|
|
||||||
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
|
||||||
.GET()
|
|
||||||
.header("Accept", "application/json")
|
|
||||||
.header("User-Agent", "MateClaw/1.0")
|
|
||||||
.build();
|
|
||||||
|
|
||||||
HttpResponse<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
|
|
||||||
int status = response.statusCode();
|
int status = response.statusCode();
|
||||||
|
|
||||||
if (status == 200) {
|
if (status == 200) {
|
||||||
SkillBundle bundle = parseBundleResponse(response.body(), slug);
|
return parseMetadataResponse(response.body());
|
||||||
if (bundle == null || bundle.content() == null || bundle.content().isBlank()) {
|
}
|
||||||
log.warn("Hub fetchBundle returned empty content for '{}'; treat as failure to avoid wiping local SKILL.md", slug);
|
if (status == 404) {
|
||||||
return null;
|
log.info("Hub metadata not found for '{}'", slug);
|
||||||
}
|
return null;
|
||||||
return bundle;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isRetryable(status) && attempt < properties.getHttpRetries()) {
|
if (isRetryable(status) && attempt < properties.getHttpRetries()) {
|
||||||
log.warn("Hub fetchBundle attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
|
log.warn("Hub metadata attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
|
||||||
Thread.sleep(backoffMs(attempt));
|
Thread.sleep(backoffMs(attempt));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
log.warn("Hub metadata fetch failed for '{}': status {}", slug, status);
|
||||||
log.warn("Hub fetchBundle failed for '{}': status {}", slug, status);
|
|
||||||
return null;
|
return null;
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
Thread.currentThread().interrupt();
|
Thread.currentThread().interrupt();
|
||||||
return null;
|
return null;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
if (attempt < properties.getHttpRetries()) {
|
if (attempt < properties.getHttpRetries()) {
|
||||||
log.warn("Hub fetchBundle attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
|
log.warn("Hub metadata attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
|
||||||
try {
|
try {
|
||||||
Thread.sleep(backoffMs(attempt));
|
Thread.sleep(backoffMs(attempt));
|
||||||
} catch (InterruptedException ie) {
|
} catch (InterruptedException ie) {
|
||||||
@ -148,22 +211,90 @@ public class SkillHubClient {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.error("Hub fetchBundle failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
|
log.warn("Hub metadata fetch failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// ==================== 内部方法 ====================
|
private byte[] downloadBundleZip(String slug, String version) {
|
||||||
|
StringBuilder url = new StringBuilder()
|
||||||
|
.append(properties.getBaseUrl())
|
||||||
|
.append(properties.getDownloadPath())
|
||||||
|
.append("?slug=").append(encodeParam(slug));
|
||||||
|
if (version != null && !version.isBlank()) {
|
||||||
|
url.append("&version=").append(encodeParam(version));
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int attempt = 0; attempt <= properties.getHttpRetries(); attempt++) {
|
||||||
|
try {
|
||||||
|
HttpRequest request = HttpRequest.newBuilder()
|
||||||
|
.uri(URI.create(url.toString()))
|
||||||
|
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
||||||
|
.GET()
|
||||||
|
.header("Accept", "application/zip, application/octet-stream")
|
||||||
|
.header("User-Agent", "MateClaw/1.0")
|
||||||
|
.build();
|
||||||
|
|
||||||
|
HttpResponse<byte[]> response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray());
|
||||||
|
int status = response.statusCode();
|
||||||
|
|
||||||
|
if (status == 200) {
|
||||||
|
byte[] body = response.body();
|
||||||
|
if (body == null || body.length == 0) {
|
||||||
|
log.warn("Hub download for '{}' returned empty body; treating as failure to avoid wiping local SKILL.md", slug);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (isRetryable(status) && attempt < properties.getHttpRetries()) {
|
||||||
|
log.warn("Hub download attempt {} for '{}' failed with status {}, retrying...", attempt + 1, slug, status);
|
||||||
|
Thread.sleep(backoffMs(attempt));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.warn("Hub download failed for '{}': status {}", slug, status);
|
||||||
|
return null;
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
return null;
|
||||||
|
} catch (Exception e) {
|
||||||
|
if (attempt < properties.getHttpRetries()) {
|
||||||
|
log.warn("Hub download attempt {} for '{}' error: {}, retrying...", attempt + 1, slug, e.getMessage());
|
||||||
|
try {
|
||||||
|
Thread.sleep(backoffMs(attempt));
|
||||||
|
} catch (InterruptedException ie) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.error("Hub download failed after {} attempts for '{}': {}", properties.getHttpRetries() + 1, slug, e.getMessage());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
private HttpRequest jsonGet(String url) {
|
||||||
|
return HttpRequest.newBuilder()
|
||||||
|
.uri(URI.create(url))
|
||||||
|
.timeout(Duration.ofSeconds(properties.getHttpTimeout()))
|
||||||
|
.GET()
|
||||||
|
.header("Accept", "application/json")
|
||||||
|
.header("User-Agent", "MateClaw/1.0")
|
||||||
|
.build();
|
||||||
|
}
|
||||||
|
|
||||||
|
// ==================== JSON parsing ====================
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
|
||||||
private List<HubSkillInfo> parseSearchResponse(String body) {
|
private List<HubSkillInfo> parseSearchResponse(String body) {
|
||||||
try {
|
try {
|
||||||
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
|
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
|
||||||
Object data = json.get("data");
|
Object data = json.get("results");
|
||||||
if (data == null) {
|
if (data == null) {
|
||||||
data = json.get("results");
|
data = json.get("data");
|
||||||
}
|
}
|
||||||
if (data == null) {
|
if (data == null) {
|
||||||
data = json.get("skills");
|
data = json.get("skills");
|
||||||
@ -179,36 +310,60 @@ public class SkillHubClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse the metadata payload returned by {@code /api/v1/skills/{slug}}.
|
||||||
|
* Tolerates both the current nested shape ({@code {skill, latestVersion, owner}})
|
||||||
|
* and a flat shape (some self-hosted hubs).
|
||||||
|
*/
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private SkillBundle parseBundleResponse(String body, String slug) {
|
private HubSkillMetadata parseMetadataResponse(String body) {
|
||||||
try {
|
try {
|
||||||
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
|
Map<String, Object> json = objectMapper.readValue(body, new TypeReference<>() {});
|
||||||
String name = getStr(json, "name", slug);
|
|
||||||
String content = getStr(json, "content", "");
|
|
||||||
String description = getStr(json, "description", "");
|
|
||||||
String author = getStr(json, "author", "");
|
|
||||||
String version = getStr(json, "version", "1.0.0");
|
|
||||||
String icon = getStr(json, "icon", "");
|
|
||||||
|
|
||||||
Map<String, String> references = json.containsKey("references")
|
Map<String, Object> skill = json.get("skill") instanceof Map<?, ?> m
|
||||||
? objectMapper.convertValue(json.get("references"), new TypeReference<>() {})
|
? (Map<String, Object>) m
|
||||||
: Map.of();
|
: json;
|
||||||
Map<String, String> scripts = json.containsKey("scripts")
|
|
||||||
? objectMapper.convertValue(json.get("scripts"), new TypeReference<>() {})
|
|
||||||
: Map.of();
|
|
||||||
|
|
||||||
return new SkillBundle(name, content, references, scripts,
|
String displayName = stringOf(skill.get("displayName"));
|
||||||
"clawhub", properties.getBaseUrl() + "/skills/" + slug,
|
if (displayName == null) displayName = stringOf(skill.get("name"));
|
||||||
version, description, author, icon);
|
String summary = stringOf(skill.get("summary"));
|
||||||
|
if (summary == null) summary = stringOf(skill.get("description"));
|
||||||
|
|
||||||
|
String version = null;
|
||||||
|
if (json.get("latestVersion") instanceof Map<?, ?> lv) {
|
||||||
|
version = stringOf(((Map<String, Object>) lv).get("version"));
|
||||||
|
}
|
||||||
|
if (version == null) version = stringOf(skill.get("version"));
|
||||||
|
|
||||||
|
String owner = null;
|
||||||
|
if (json.get("owner") instanceof Map<?, ?> ownerMap) {
|
||||||
|
Map<String, Object> o = (Map<String, Object>) ownerMap;
|
||||||
|
owner = firstNonBlank(stringOf(o.get("displayName")), stringOf(o.get("handle")));
|
||||||
|
}
|
||||||
|
if (owner == null) owner = stringOf(skill.get("author"));
|
||||||
|
|
||||||
|
return new HubSkillMetadata(displayName, summary, version, owner);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
log.warn("Failed to parse hub bundle response: {}", e.getMessage());
|
log.warn("Failed to parse hub metadata response: {}", e.getMessage());
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getStr(Map<String, Object> map, String key, String defaultVal) {
|
private record HubSkillMetadata(String displayName, String summary, String version, String owner) {}
|
||||||
Object v = map.get(key);
|
|
||||||
return v != null ? v.toString() : defaultVal;
|
// ==================== misc helpers ====================
|
||||||
|
|
||||||
|
private static String stringOf(Object v) {
|
||||||
|
if (v == null) return null;
|
||||||
|
String s = v.toString();
|
||||||
|
return s.isBlank() ? null : s;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String firstNonBlank(String... values) {
|
||||||
|
for (String v : values) {
|
||||||
|
if (v != null && !v.isBlank()) return v;
|
||||||
|
}
|
||||||
|
return "";
|
||||||
}
|
}
|
||||||
|
|
||||||
private boolean isRetryable(int statusCode) {
|
private boolean isRetryable(int statusCode) {
|
||||||
|
|||||||
@ -12,15 +12,21 @@ import org.springframework.boot.context.properties.ConfigurationProperties;
|
|||||||
@ConfigurationProperties(prefix = "mateclaw.skill.hub")
|
@ConfigurationProperties(prefix = "mateclaw.skill.hub")
|
||||||
public class SkillHubProperties {
|
public class SkillHubProperties {
|
||||||
|
|
||||||
/** Hub 基础 URL */
|
/** Hub base URL. */
|
||||||
private String baseUrl = "https://clawhub.ai";
|
private String baseUrl = "https://clawhub.ai";
|
||||||
|
|
||||||
/** 搜索 API 路径 */
|
/** Search API path. */
|
||||||
private String searchPath = "/api/v1/search";
|
private String searchPath = "/api/v1/search";
|
||||||
|
|
||||||
/** HTTP 请求超时(秒) */
|
/** Skill metadata API path prefix; full path is {@code <skillsPath>/<slug>}. */
|
||||||
|
private String skillsPath = "/api/v1/skills";
|
||||||
|
|
||||||
|
/** Bundle ZIP download API path; supports {@code ?slug=&version=}. */
|
||||||
|
private String downloadPath = "/api/v1/download";
|
||||||
|
|
||||||
|
/** HTTP request timeout (seconds). */
|
||||||
private int httpTimeout = 15;
|
private int httpTimeout = 15;
|
||||||
|
|
||||||
/** HTTP 重试次数 */
|
/** HTTP retry count. */
|
||||||
private int httpRetries = 3;
|
private int httpRetries = 3;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -15,16 +15,14 @@ import java.util.zip.ZipEntry;
|
|||||||
import java.util.zip.ZipInputStream;
|
import java.util.zip.ZipInputStream;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ZIP 格式 Skill 包解析器
|
* Parses a ZIP-packaged Skill into a {@link SkillBundle}.
|
||||||
* <p>
|
* <p>
|
||||||
* 从上传的 ZIP 文件中解析 SKILL.md + references/ + scripts/,
|
* Used by both the upload endpoint (MultipartFile) and the ClawHub install
|
||||||
* 构建统一的 {@link SkillBundle} 供安装流程使用。
|
* path (downloaded ZIP bytes). Hardened against:
|
||||||
* <p>
|
|
||||||
* 安全防护:
|
|
||||||
* <ul>
|
* <ul>
|
||||||
* <li>Zip Slip 路径穿越检测</li>
|
* <li>Zip Slip path traversal</li>
|
||||||
* <li>单文件 ≤1MB,总解压 ≤50MB</li>
|
* <li>Per-file ≤1MB, total ≤50MB</li>
|
||||||
* <li>仅接受 SKILL.md / references/ / scripts/ 下的文件</li>
|
* <li>Only SKILL.md / references/ / scripts/ entries are kept</li>
|
||||||
* </ul>
|
* </ul>
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
@ -38,12 +36,17 @@ public class ZipSkillFetcher {
|
|||||||
private static final String SKILL_MD_LOWER = "skill.md";
|
private static final String SKILL_MD_LOWER = "skill.md";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 解析 ZIP 文件为 SkillBundle
|
* Holds the in-memory result of decompressing a ZIP. Used by callers
|
||||||
*
|
* that want to enrich the SkillBundle with metadata (e.g. ClawHub author
|
||||||
* @param zipFile 上传的 ZIP 文件
|
* / icon) that isn't carried inside SKILL.md.
|
||||||
* @param parser frontmatter 解析器
|
*/
|
||||||
* @return 解析后的 SkillBundle
|
public record ExtractedSkill(String skillMdContent,
|
||||||
* @throws IOException 解析失败
|
Map<String, String> references,
|
||||||
|
Map<String, String> scripts) {}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parse an uploaded ZIP file into a SkillBundle. Source type is "zip"
|
||||||
|
* and source URL is the original filename.
|
||||||
*/
|
*/
|
||||||
public static SkillBundle parse(MultipartFile zipFile, SkillFrontmatterParser parser) throws IOException {
|
public static SkillBundle parse(MultipartFile zipFile, SkillFrontmatterParser parser) throws IOException {
|
||||||
if (zipFile == null || zipFile.isEmpty()) {
|
if (zipFile == null || zipFile.isEmpty()) {
|
||||||
@ -53,15 +56,52 @@ public class ZipSkillFetcher {
|
|||||||
throw new IllegalArgumentException("ZIP file too large (max 50MB)");
|
throw new IllegalArgumentException("ZIP file too large (max 50MB)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
ExtractedSkill extracted;
|
||||||
|
try (InputStream is = zipFile.getInputStream()) {
|
||||||
|
extracted = extract(is);
|
||||||
|
}
|
||||||
|
|
||||||
|
var parsed = parser.parse(extracted.skillMdContent());
|
||||||
|
String name = parsed.getName();
|
||||||
|
if (name == null || name.isBlank()) {
|
||||||
|
String zipName = zipFile.getOriginalFilename();
|
||||||
|
if (zipName != null) {
|
||||||
|
name = zipName.replaceAll("\\.zip$", "").replaceAll("[^a-zA-Z0-9_-]", "-");
|
||||||
|
} else {
|
||||||
|
name = "imported-skill";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("[ZipSkillFetcher] Parsed: name={}, references={}, scripts={}",
|
||||||
|
name, extracted.references().size(), extracted.scripts().size());
|
||||||
|
|
||||||
|
Map<String, Object> fm = parsed.getFrontmatter();
|
||||||
|
return new SkillBundle(
|
||||||
|
name,
|
||||||
|
extracted.skillMdContent(),
|
||||||
|
extracted.references(),
|
||||||
|
extracted.scripts(),
|
||||||
|
"zip",
|
||||||
|
zipFile.getOriginalFilename(),
|
||||||
|
fm != null ? String.valueOf(fm.getOrDefault("version", "1.0.0")) : "1.0.0",
|
||||||
|
parsed.getDescription(),
|
||||||
|
fm != null ? String.valueOf(fm.getOrDefault("author", "")) : "",
|
||||||
|
fm != null ? String.valueOf(fm.getOrDefault("icon", "📦")) : "📦"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Decompress a ZIP stream into in-memory SKILL.md + references + scripts.
|
||||||
|
* Throws {@link IllegalArgumentException} if no SKILL.md is present.
|
||||||
|
*/
|
||||||
|
public static ExtractedSkill extract(InputStream zipStream) throws IOException {
|
||||||
String skillMdContent = null;
|
String skillMdContent = null;
|
||||||
String skillMdPrefix = ""; // 如果 SKILL.md 在子目录中,记录前缀
|
String skillMdPrefix = "";
|
||||||
Map<String, String> references = new HashMap<>();
|
Map<String, String> references = new HashMap<>();
|
||||||
Map<String, String> scripts = new HashMap<>();
|
Map<String, String> scripts = new HashMap<>();
|
||||||
long totalSize = 0;
|
long totalSize = 0;
|
||||||
|
|
||||||
try (InputStream is = zipFile.getInputStream();
|
try (ZipInputStream zis = new ZipInputStream(zipStream, StandardCharsets.UTF_8)) {
|
||||||
ZipInputStream zis = new ZipInputStream(is, StandardCharsets.UTF_8)) {
|
|
||||||
|
|
||||||
ZipEntry entry;
|
ZipEntry entry;
|
||||||
while ((entry = zis.getNextEntry()) != null) {
|
while ((entry = zis.getNextEntry()) != null) {
|
||||||
if (entry.isDirectory()) {
|
if (entry.isDirectory()) {
|
||||||
@ -71,7 +111,7 @@ public class ZipSkillFetcher {
|
|||||||
|
|
||||||
String entryName = entry.getName();
|
String entryName = entry.getName();
|
||||||
|
|
||||||
// Zip Slip 防护:normalize 后检查是否逃逸
|
// Zip Slip guard: normalize and reject absolute / traversal entries.
|
||||||
Path entryPath = Path.of(entryName).normalize();
|
Path entryPath = Path.of(entryName).normalize();
|
||||||
if (entryPath.isAbsolute() || entryName.contains("..")) {
|
if (entryPath.isAbsolute() || entryName.contains("..")) {
|
||||||
log.warn("[ZipSkillFetcher] Skipping suspicious entry: {}", entryName);
|
log.warn("[ZipSkillFetcher] Skipping suspicious entry: {}", entryName);
|
||||||
@ -79,16 +119,19 @@ public class ZipSkillFetcher {
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 文件大小检查
|
long declaredSize = entry.getSize();
|
||||||
long size = entry.getSize();
|
if (declaredSize > MAX_FILE_SIZE) {
|
||||||
if (size > MAX_FILE_SIZE) {
|
log.warn("[ZipSkillFetcher] Skipping oversized entry: {} ({}bytes)", entryName, declaredSize);
|
||||||
log.warn("[ZipSkillFetcher] Skipping oversized entry: {} ({}bytes)", entryName, size);
|
|
||||||
zis.closeEntry();
|
zis.closeEntry();
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 读取内容
|
|
||||||
byte[] bytes = zis.readAllBytes();
|
byte[] bytes = zis.readAllBytes();
|
||||||
|
if (bytes.length > MAX_FILE_SIZE) {
|
||||||
|
log.warn("[ZipSkillFetcher] Skipping oversized entry post-read: {} ({}bytes)", entryName, bytes.length);
|
||||||
|
zis.closeEntry();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
totalSize += bytes.length;
|
totalSize += bytes.length;
|
||||||
if (totalSize > MAX_TOTAL_SIZE) {
|
if (totalSize > MAX_TOTAL_SIZE) {
|
||||||
throw new IOException("Total extracted size exceeds 50MB limit");
|
throw new IOException("Total extracted size exceeds 50MB limit");
|
||||||
@ -97,10 +140,8 @@ public class ZipSkillFetcher {
|
|||||||
String content = new String(bytes, StandardCharsets.UTF_8);
|
String content = new String(bytes, StandardCharsets.UTF_8);
|
||||||
String fileName = entryPath.getFileName().toString();
|
String fileName = entryPath.getFileName().toString();
|
||||||
|
|
||||||
// 定位 SKILL.md(根目录或一级子目录)
|
|
||||||
if (skillMdContent == null && (SKILL_MD.equals(fileName) || SKILL_MD_LOWER.equals(fileName))) {
|
if (skillMdContent == null && (SKILL_MD.equals(fileName) || SKILL_MD_LOWER.equals(fileName))) {
|
||||||
skillMdContent = content;
|
skillMdContent = content;
|
||||||
// 确定子目录前缀(如 "my-skill/SKILL.md" → prefix = "my-skill/")
|
|
||||||
int slashIdx = entryName.lastIndexOf('/');
|
int slashIdx = entryName.lastIndexOf('/');
|
||||||
skillMdPrefix = slashIdx > 0 ? entryName.substring(0, slashIdx + 1) : "";
|
skillMdPrefix = slashIdx > 0 ? entryName.substring(0, slashIdx + 1) : "";
|
||||||
log.info("[ZipSkillFetcher] Found SKILL.md at: {}", entryName);
|
log.info("[ZipSkillFetcher] Found SKILL.md at: {}", entryName);
|
||||||
@ -108,19 +149,16 @@ public class ZipSkillFetcher {
|
|||||||
|
|
||||||
zis.closeEntry();
|
zis.closeEntry();
|
||||||
|
|
||||||
// 先收集所有文件,后面按前缀过滤
|
|
||||||
String normalizedName = entryPath.toString().replace('\\', '/');
|
String normalizedName = entryPath.toString().replace('\\', '/');
|
||||||
|
|
||||||
// 收集 references/ 和 scripts/ 文件
|
|
||||||
String relativeName = normalizedName;
|
String relativeName = normalizedName;
|
||||||
if (!skillMdPrefix.isEmpty() && normalizedName.startsWith(skillMdPrefix)) {
|
if (!skillMdPrefix.isEmpty() && normalizedName.startsWith(skillMdPrefix)) {
|
||||||
relativeName = normalizedName.substring(skillMdPrefix.length());
|
relativeName = normalizedName.substring(skillMdPrefix.length());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (relativeName.startsWith("references/")) {
|
if (relativeName.startsWith("references/")) {
|
||||||
references.put(relativeName, content);
|
references.put(relativeName.substring("references/".length()), content);
|
||||||
} else if (relativeName.startsWith("scripts/")) {
|
} else if (relativeName.startsWith("scripts/")) {
|
||||||
scripts.put(relativeName, content);
|
scripts.put(relativeName.substring("scripts/".length()), content);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -129,33 +167,6 @@ public class ZipSkillFetcher {
|
|||||||
throw new IllegalArgumentException("ZIP does not contain SKILL.md");
|
throw new IllegalArgumentException("ZIP does not contain SKILL.md");
|
||||||
}
|
}
|
||||||
|
|
||||||
// 解析 frontmatter
|
return new ExtractedSkill(skillMdContent, references, scripts);
|
||||||
var parsed = parser.parse(skillMdContent);
|
|
||||||
String name = parsed.getName();
|
|
||||||
if (name == null || name.isBlank()) {
|
|
||||||
// 从 ZIP 文件名推断
|
|
||||||
String zipName = zipFile.getOriginalFilename();
|
|
||||||
if (zipName != null) {
|
|
||||||
name = zipName.replaceAll("\\.zip$", "").replaceAll("[^a-zA-Z0-9_-]", "-");
|
|
||||||
} else {
|
|
||||||
name = "imported-skill";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
log.info("[ZipSkillFetcher] Parsed: name={}, references={}, scripts={}, totalSize={}",
|
|
||||||
name, references.size(), scripts.size(), totalSize);
|
|
||||||
|
|
||||||
return new SkillBundle(
|
|
||||||
name,
|
|
||||||
skillMdContent,
|
|
||||||
references,
|
|
||||||
scripts,
|
|
||||||
"zip",
|
|
||||||
zipFile.getOriginalFilename(),
|
|
||||||
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("version", "1.0.0")) : "1.0.0",
|
|
||||||
parsed.getDescription(),
|
|
||||||
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("author", "")) : "",
|
|
||||||
parsed.getFrontmatter() != null ? String.valueOf(parsed.getFrontmatter().getOrDefault("icon", "📦")) : "📦"
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -1,20 +1,33 @@
|
|||||||
package vip.mate.skill.installer.model;
|
package vip.mate.skill.installer.model;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.annotation.JsonAlias;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
|
||||||
import lombok.Data;
|
import lombok.Data;
|
||||||
|
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* ClawHub 市场 skill 信息
|
* ClawHub marketplace skill summary.
|
||||||
|
* <p>
|
||||||
|
* The hub API uses {@code displayName} / {@code summary}; older / alternative
|
||||||
|
* deployments expose {@code name} / {@code description}. {@link JsonAlias}
|
||||||
|
* keeps both wire shapes deserializing into the same fields so the UI never
|
||||||
|
* shows blank rows when the upstream renames a key.
|
||||||
*
|
*
|
||||||
* @author MateClaw Team
|
* @author MateClaw Team
|
||||||
*/
|
*/
|
||||||
@Data
|
@Data
|
||||||
|
@JsonIgnoreProperties(ignoreUnknown = true)
|
||||||
public class HubSkillInfo {
|
public class HubSkillInfo {
|
||||||
|
|
||||||
|
@JsonAlias({"displayName"})
|
||||||
private String name;
|
private String name;
|
||||||
|
|
||||||
private String slug;
|
private String slug;
|
||||||
|
|
||||||
|
@JsonAlias({"summary"})
|
||||||
private String description;
|
private String description;
|
||||||
|
|
||||||
private String author;
|
private String author;
|
||||||
private String version;
|
private String version;
|
||||||
private String icon;
|
private String icon;
|
||||||
|
|||||||
@ -109,6 +109,8 @@ mateclaw:
|
|||||||
hub:
|
hub:
|
||||||
base-url: https://clawhub.ai
|
base-url: https://clawhub.ai
|
||||||
search-path: /api/v1/search
|
search-path: /api/v1/search
|
||||||
|
skills-path: /api/v1/skills
|
||||||
|
download-path: /api/v1/download
|
||||||
http-timeout: 15
|
http-timeout: 15
|
||||||
http-retries: 3
|
http-retries: 3
|
||||||
plugin:
|
plugin:
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user