diff --git a/.env.example b/.env.example index de591168..80b70f27 100644 --- a/.env.example +++ b/.env.example @@ -87,6 +87,15 @@ MATECLAW_OAUTH_OPENAI_CALLBACK_BIND_HOST= # - /your/host/path:/data/wiki MATE_WIKI_ALLOWED_SOURCE_ROOTS= +# ── Skill 工作区目录 ───────────────────────────────────────────── +# 已安装的 skill、运行时积累的 LESSONS.md、skill 运行产物都落在这个目录。 +# 默认(容器内)已指向 /app/data/skills,由 docker-compose 的 server_data 卷 +# 持久化,容器重启不丢,无需额外挂卷。一般无需修改。 +# 内置 skill 由 JAR classpath 每次启动现场释放,挂空卷也不会丢内置文件。 +# 仅当你想把 skill 目录放到别处(如独立的 bind mount)时才覆盖此项, +# 并记得在 docker-compose.yml 的 volumes 里把对应宿主机目录挂进容器。 +MATECLAW_SKILL_WORKSPACE_ROOT= + # ── Maven 镜像(国内加速)───────────────────────────────────────── # 在中国大陆构建时取消注释,将 Aliyun 仓库优先级提前,大幅提速 mvn 拉包。 # 空值(默认)使用 US Maven Central → Google CDN → Aliyun 的顺序。 diff --git a/docker-compose.yml b/docker-compose.yml index 55df7c18..060cc464 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -100,6 +100,10 @@ services: # 示例:MATE_WIKI_ALLOWED_SOURCE_ROOTS=/data/wiki,/opt/docs # 记得同步在 volumes 里把宿主机路径挂进容器。 MATE_WIKI_ALLOWED_SOURCE_ROOTS: ${MATE_WIKI_ALLOWED_SOURCE_ROOTS:-} + # Skill 工作区根目录。放在 /app/data 下,让现有的 server_data 卷一并持久化 + # 已安装的 skill、运行时积累的 LESSONS.md 以及 skill 运行产物,容器重启不丢。 + # 内置 skill 仍由 JAR classpath 每次启动现场释放,空卷不会丢内置文件。 + MATECLAW_SKILL_WORKSPACE_ROOT: ${MATECLAW_SKILL_WORKSPACE_ROOT:-/app/data/skills} # Chromium needs a real /dev/shm. Docker defaults to 64MB which causes # SIGBUS / "Target page closed" errors under load. 2GB is the usual # recommendation for Playwright / headless chrome. @@ -108,10 +112,11 @@ services: - "18080:18088" # host:container — app listens on 18088 inside the container - "1455:1455" volumes: + # server_data covers /app/data — H2 DB, wiki-uploads, AND the skill + # workspace (MATECLAW_SKILL_WORKSPACE_ROOT=/app/data/skills above), so a + # single volume persists everything. No separate skills volume needed. - server_data:/app/data - - skills_data:/root/.mateclaw/skills volumes: mysql_data: server_data: - skills_data: diff --git a/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java b/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java index ef483df5..e1e8221f 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/installer/ZipSkillFetcher.java @@ -29,6 +29,8 @@ import java.util.zip.ZipInputStream; *
  • Zip Slip path traversal
  • *
  • Per-file ≤1MB, total ≤50MB
  • *
  • Only SKILL.md / references/ / scripts/ entries are kept
  • + *
  • Binary entries are skipped with a WARN — bundle storage is text-only, + * so decoding them as text would persist corrupted content
  • * * *

    Extraction is two-pass: the entire archive is buffered in memory first @@ -229,6 +231,23 @@ public class ZipSkillFetcher { throw new IOException("Total extracted size exceeds 50MB limit"); } + // Skill bundles persist file contents as text (mate_skill_file + // is a TEXT column; SkillBundle carries Map). + // Decoding a binary entry (.png/.woff/.zip/compiled helper, …) + // as text replaces every invalid byte with U+FFFD, so the file + // would be stored permanently corrupted and "restored" broken + // on every sync. Binary resources are not supported in a bundle + // today, so skip them with a clear WARN instead of silently + // mangling them — matches how unknown root-level files are + // already handled below. (Root-level binaries were already + // dropped; this also covers binaries nested in scripts/ and + // references/, which previously slipped through corrupted.) + if (isLikelyBinary(bytes)) { + log.warn("[ZipSkillFetcher] Skipping binary entry (not supported in skill bundles): {}", entryName); + zis.closeEntry(); + continue; + } + String content = new String(bytes, charset); String normalizedName = entryPath.toString().replace('\\', '/'); String fileName = entryPath.getFileName().toString(); @@ -289,6 +308,27 @@ public class ZipSkillFetcher { return new ExtractedSkill(skillMdContent, references, scripts); } + /** + * Heuristic binary detector: an entry is treated as binary if a NUL byte + * (0x00) appears within the inspected prefix. UTF-8 and GBK text never + * contain a NUL, while virtually every binary format (PNG/WOFF/ZIP/class/ + * native executable) carries one near the start — this is the same cheap, + * reliable test git uses to decide "is this a text file". Inspecting only a + * prefix keeps it O(1) for large entries. + */ + private static boolean isLikelyBinary(byte[] bytes) { + if (bytes == null || bytes.length == 0) { + return false; + } + int limit = Math.min(bytes.length, 8000); + for (int i = 0; i < limit; i++) { + if (bytes[i] == 0x00) { + return true; + } + } + return false; + } + /** * Classify a root-level file (sibling of SKILL.md, no directory prefix) * by extension. Returns {@code "scripts"} / {@code "references"} for diff --git a/mateclaw-server/src/main/resources/application.yml b/mateclaw-server/src/main/resources/application.yml index d064b00c..a547ece6 100644 --- a/mateclaw-server/src/main/resources/application.yml +++ b/mateclaw-server/src/main/resources/application.yml @@ -148,7 +148,11 @@ mateclaw: mode: ${MATECLAW_TOOLS_DISCLOSURE_MODE:progressive} skill: workspace: - root: ${user.home}/.mateclaw/skills + # Skill workspace root. Override with MATECLAW_SKILL_WORKSPACE_ROOT to + # relocate it onto a persistent volume — in Docker this is pointed at + # /app/data/skills so the existing server_data volume persists installed + # skills, accumulated LESSONS.md, and skill runtime files across restarts. + root: ${MATECLAW_SKILL_WORKSPACE_ROOT:${user.home}/.mateclaw/skills} auto-init: true delete-policy: archive disclosure: diff --git a/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java b/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java index 0037c325..6e5606d0 100644 --- a/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java +++ b/mateclaw-server/src/test/java/vip/mate/skill/installer/ZipSkillFetcherTest.java @@ -205,6 +205,44 @@ class ZipSkillFetcherTest { assertEquals("#!/bin/sh\n", ex.scripts().get("setup.sh")); } + private record RawEntry(String name, byte[] content) {} + + private static byte[] zipOfRaw(List entries) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zos = new ZipOutputStream(baos, StandardCharsets.UTF_8)) { + for (RawEntry e : entries) { + zos.putNextEntry(new ZipEntry(e.name())); + zos.write(e.content()); + zos.closeEntry(); + } + } + return baos.toByteArray(); + } + + @Test + @DisplayName("Binary entry under scripts/ is skipped, not stored corrupted (#273)") + void binaryEntryInScriptsIsSkipped() throws IOException { + // A PNG header carries a NUL byte; decoding it as UTF-8 would replace + // bytes with U+FFFD and persist a corrupted "text" file. The fetcher + // must drop it (with a WARN) while keeping the legitimate text script. + byte[] pngBytes = new byte[]{(byte) 0x89, 'P', 'N', 'G', 0x00, 0x1A, 0x0A, 'x'}; + byte[] zip = zipOfRaw(List.of( + new RawEntry("pkg/SKILL.md", SKILL_MD.getBytes(StandardCharsets.UTF_8)), + new RawEntry("pkg/scripts/run.py", "print('ok')\n".getBytes(StandardCharsets.UTF_8)), + new RawEntry("pkg/scripts/logo.png", pngBytes), + new RawEntry("pkg/references/font.woff", new byte[]{'w', 'O', 'F', 'F', 0x00, 0x01}) + )); + + ZipSkillFetcher.ExtractedSkill ex = ZipSkillFetcher.extract(new ByteArrayInputStream(zip)); + + // Text script survives; both binaries are dropped (no corrupted entry). + assertEquals(Map.of("run.py", "print('ok')\n"), ex.scripts(), + "Binary logo.png must not be stored; the text script stays"); + assertTrue(ex.references().isEmpty(), + "Binary font.woff must not be stored as corrupted text"); + assertFalse(ex.scripts().containsKey("logo.png")); + } + @Test @DisplayName("GBK-encoded entry names (Windows-authored zip) fall back from UTF-8 to GBK") void extractsGbkEncodedNames() throws IOException {