feat(skill): configurable pip index for skill Python scripts (mirror & private LAN sources)

Let skill Python scripts install packages from a configurable pip index instead of the default PyPI. docker-compose passes PIP_INDEX_URL / PIP_TRUSTED_HOST into the container; for the desktop app (host JVM, no Docker env) SkillScriptExecutionService falls back to mateclaw.pip.index-url / trusted-host Spring config and injects them into the subprocess, auto-deriving the trusted host for plain-HTTP LAN mirrors. The runtime image gains pip and a build toolchain (with the PEP 668 marker removed so on-the-fly installs work), and the script timeout ceiling is raised to accommodate large installs.

.env.example

@@ -145,6 +145,20 @@ MATECLAW_SKILL_WORKSPACE_ROOT=
 MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB=
 MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB=
 
+# ── Python pip 镜像源（可选）──────────────────────────────────────
+# skill 里 Python 脚本缺包时 pip install 走的源。默认用 PyPI 官方源。
+# pip 原生读 PIP_INDEX_URL / PIP_TRUSTED_HOST 环境变量，容器内自动继承。
+# HTTP 源会自动从 URL 推导 PIP_TRUSTED_HOST；自签 HTTPS 需手动填。
+# 互联网加速:   https://pypi.tuna.tsinghua.edu.cn/simple
+# 局域网私有源: http://192.168.1.100:8080/simple（trusted-host 自动推导）
+#PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
+#PIP_TRUSTED_HOST=
+# ── 桌面版补充（非 Docker，宿主机直接跑 Java）─────────────────────
+# 桌面版不继承上面的 Docker 变量。用 Spring 配置注入 Python 子进程；
+# 也可直接设系统环境变量 PIP_INDEX_URL / PIP_TRUSTED_HOST（覆盖更全）。
+#MATECLAW_PIP_INDEX_URL=
+#MATECLAW_PIP_TRUSTED_HOST=
+
 # ── Maven 镜像（国内加速）─────────────────────────────────────────
 # 在中国大陆构建时取消注释，将 Aliyun 仓库优先级提前，大幅提速 mvn 拉包。
 # 空值（默认）使用 US Maven Central → Google CDN → Aliyun 的顺序。

docker-compose.yml

@@ -162,6 +162,14 @@ services:
       # max-total 调多大，单次安装的峰值内存就可能吃多大。
       MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB:-1}
       MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB:-50}
+      # pip 镜像源配置（可选）。skill 里的 Python 脚本缺包时 pip install 会走这个源。
+      # 留空则用 PyPI 默认源（pypi.org）。pip 原生读 PIP_INDEX_URL / PIP_TRUSTED_HOST
+      # 环境变量，容器内所有进程（JVM、Python 子进程、bash）自动继承，无需额外配置。
+      # HTTP 源会自动从 URL 推导 PIP_TRUSTED_HOST；自签 HTTPS 需手动填。
+      # - 互联网加速：PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple
+      # - 局域网私有源：PIP_INDEX_URL=http://192.168.1.100:8080/simple（trusted-host 自动推导）
+      PIP_INDEX_URL: ${PIP_INDEX_URL:-}
+      PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-}
     # Chromium needs a real /dev/shm. Docker defaults to 64MB which causes
     # SIGBUS / "Target page closed" errors under load. 2GB is the usual
     # recommendation for Playwright / headless chrome.

mateclaw-server/Dockerfile

@@ -100,8 +100,19 @@ RUN apt-get update \
         tesseract-ocr \
         tesseract-ocr-chi-sim \
         tzdata \
+        python3-pip \
+        python-is-python3 \
+        python3-dev \
+        build-essential \
     && rm -rf /var/lib/apt/lists/*
 
+# Remove PEP 668's EXTERNALLY-MANAGED marker so pip can install packages
+# system-wide without --break-system-packages. This is a container — there is
+# no host Python environment to protect. Skill scripts and LLM-generated code
+# need to `pip install` on the fly; PEP 668 would block every install with
+# "error: externally-managed-environment".
+RUN rm -f /usr/lib/python3*/EXTERNALLY-MANAGED
+
 # Tell Playwright Java where Microsoft's image stored the browsers.
 # BrowserLauncher's BUNDLED strategy will then succeed without extra config.
 ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
@@ -118,6 +129,7 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
 ENV SPRING_PROFILES_ACTIVE=mysql
 
 COPY --from=builder /build/mateclaw-server/target/*.jar app.jar
+
 EXPOSE 18088
 EXPOSE 1455
 ENTRYPOINT ["java", "-jar", "app.jar"]

mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillScriptExecutionService.java

@@ -2,10 +2,12 @@ package vip.mate.skill.runtime;
 
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -28,11 +30,24 @@ import java.util.concurrent.TimeUnit;
 public class SkillScriptExecutionService {
 
     private static final long DEFAULT_TIMEOUT_SECONDS = 30;
-    private static final long MAX_TIMEOUT_SECONDS = 300;
+    private static final long MAX_TIMEOUT_SECONDS = 600;
     private static final int MAX_OUTPUT_BYTES = 50_000;
     private static final boolean IS_WINDOWS = System.getProperty("os.name", "")
             .toLowerCase(Locale.ROOT).contains("win");
 
+    /**
+     * Pip mirror config for desktop (non-Docker) deployments. In Docker these
+     * arrive as PIP_INDEX_URL / PIP_TRUSTED_HOST env vars (set in
+     * docker-compose) and are inherited by ProcessBuilder directly. On the
+     * desktop app Java runs on the host — the host may not have those env
+     * vars set, so we fall back to Spring config and inject them explicitly.
+     */
+    @Value("${mateclaw.pip.index-url:}")
+    private String pipIndexUrl;
+
+    @Value("${mateclaw.pip.trusted-host:}")
+    private String pipTrustedHost;
+
     /** Supported inline-code languages mapped to the temp-file extension. */
     private static final Map<String, String> LANGUAGE_EXTENSIONS = Map.of(
             "python", ".py",
@@ -225,6 +240,7 @@ public class SkillScriptExecutionService {
                     processEnv.put(e.getKey(), e.getValue());
                 }
             }
+            injectPipMirrorEnv(pb);
 
             Process process = pb.start();
 
@@ -272,6 +288,45 @@ public class SkillScriptExecutionService {
         }
     }
 
+    /**
+     * Inject pip mirror config into the subprocess environment.
+     *
+     * <p>Three layers, later ones only fill gaps left by earlier ones:
+     * <ol>
+     *   <li>Docker / system env — {@code PIP_INDEX_URL} / {@code PIP_TRUSTED_HOST}
+     *       already in the ProcessBuilder env (inherited from JVM). Nothing to do.</li>
+     *   <li>Spring config fallback — for desktop (non-Docker) deployments where
+     *       the host may not have those env vars. Injected only when absent.</li>
+     *   <li>Auto-derive {@code PIP_TRUSTED_HOST} — if the index URL is plain
+     *       HTTP and no trusted-host is set, pip blocks the download. Extract
+     *       the host from the URL so the user only needs to set one variable.</li>
+     * </ol>
+     */
+    private void injectPipMirrorEnv(ProcessBuilder pb) {
+        Map<String, String> env = pb.environment();
+
+        // Layer 2: Spring config fallback (desktop)
+        if (pipIndexUrl != null && !pipIndexUrl.isBlank()
+                && !env.containsKey("PIP_INDEX_URL")) {
+            env.put("PIP_INDEX_URL", pipIndexUrl);
+        }
+        if (pipTrustedHost != null && !pipTrustedHost.isBlank()
+                && !env.containsKey("PIP_TRUSTED_HOST")) {
+            env.put("PIP_TRUSTED_HOST", pipTrustedHost);
+        }
+
+        // Layer 3: auto-derive trusted-host for HTTP sources
+        String indexUrl = env.get("PIP_INDEX_URL");
+        if (indexUrl != null && !indexUrl.isBlank()
+                && !env.containsKey("PIP_TRUSTED_HOST")
+                && indexUrl.startsWith("http://")) {
+            String host = URI.create(indexUrl).getHost();
+            if (host != null && !host.isEmpty()) {
+                env.put("PIP_TRUSTED_HOST", host);
+            }
+        }
+    }
+
     private static String readFileTruncated(Path file, int maxBytes) {
         try {
             if (file == null || !Files.exists(file)) return "";

mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillScriptExecutionServicePipMirrorTest.java

@@ -0,0 +1,165 @@
+package vip.mate.skill.runtime;
+
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.springframework.test.util.ReflectionTestUtils;
+
+import java.nio.file.Path;
+import java.util.Locale;
+import java.util.Map;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Tests for the pip mirror env-var injection in
+ * {@link SkillScriptExecutionService}.
+ *
+ * <p>Two scenarios:
+ * <ul>
+ *   <li><b>Desktop fallback</b> — {@code PIP_INDEX_URL} absent from the
+ *       process env → Spring config ({@code mateclaw.pip.index-url}) is
+ *       injected so Python subprocesses can still find the mirror.</li>
+ *   <li><b>Docker / env-var precedence</b> — {@code PIP_INDEX_URL} already
+ *       present in the subprocess env (set via docker-compose or system env)
+ *       → Spring config does NOT override it.</li>
+ * </ul>
+ *
+ * <p>Bash-gated so the suite stays green on hosts without bash (e.g. Windows CI).
+ */
+class SkillScriptExecutionServicePipMirrorTest {
+
+    private static final boolean IS_WINDOWS =
+            System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win");
+
+    private static final String BASH_SCRIPT =
+            "echo PIP_INDEX_URL=$PIP_INDEX_URL\n" +
+            "echo PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST\n";
+
+    private SkillScriptExecutionService newService(String indexUrl, String trustedHost) {
+        SkillScriptExecutionService svc = new SkillScriptExecutionService();
+        ReflectionTestUtils.setField(svc, "pipIndexUrl", indexUrl);
+        ReflectionTestUtils.setField(svc, "pipTrustedHost", trustedHost);
+        return svc;
+    }
+
+    @Test
+    @DisplayName("desktop fallback: Spring config injected when PIP_INDEX_URL absent from env")
+    void springConfigInjectedWhenEnvAbsent(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+        // Skip if the host already has PIP_INDEX_URL set — the inherited env
+        // var would make containsKey() true, hiding the Spring fallback path.
+        assumeTrue(System.getenv("PIP_INDEX_URL") == null,
+                "PIP_INDEX_URL already set in host environment");
+
+        var svc = newService("http://192.168.1.100:8080/simple", "192.168.1.100");
+
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://192.168.1.100:8080/simple");
+        assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=192.168.1.100");
+    }
+
+    @Test
+    @DisplayName("Docker case: env var takes precedence over Spring config")
+    void envVarTakesPrecedenceOverSpringConfig(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+
+        var svc = newService("http://spring-fallback:8080/simple", "spring-fallback");
+
+        // Simulate Docker: PIP_INDEX_URL already in the subprocess env
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null,
+                Map.of("PIP_INDEX_URL", "http://docker-env:9090/simple",
+                       "PIP_TRUSTED_HOST", "docker-env"),
+                null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://docker-env:9090/simple");
+        assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=docker-env");
+        assertThat(result.getStdout()).doesNotContain("spring-fallback");
+    }
+
+    @Test
+    @DisplayName("no config: nothing injected, pip uses defaults")
+    void noPipConfigNoInjection(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+        assumeTrue(System.getenv("PIP_INDEX_URL") == null,
+                "PIP_INDEX_URL already set in host environment");
+
+        var svc = newService("", "");
+
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout()).doesNotContain("http://");
+    }
+
+    @Test
+    @DisplayName("HTTPS index-url, no trusted-host → not auto-derived")
+    void onlyIndexUrlSetHttps(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+        assumeTrue(System.getenv("PIP_INDEX_URL") == null,
+                "PIP_INDEX_URL already set in host environment");
+
+        var svc = newService("https://pypi.tuna.tsinghua.edu.cn/simple", "");
+
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout())
+                .contains("PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple");
+        // HTTPS + valid cert → no trusted-host needed, should NOT be auto-derived
+        assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=\n");
+    }
+
+    @Test
+    @DisplayName("HTTP index-url, no trusted-host → auto-derived from URL")
+    void httpAutoDeriveTrustedHost(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+        assumeTrue(System.getenv("PIP_INDEX_URL") == null,
+                "PIP_INDEX_URL already set in host environment");
+
+        var svc = newService("http://192.168.1.100:8080/simple", "");
+
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout())
+                .contains("PIP_INDEX_URL=http://192.168.1.100:8080/simple");
+        // HTTP source → trusted-host auto-derived from URL
+        assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=192.168.1.100");
+    }
+
+    @Test
+    @DisplayName("HTTP index-url with explicit trusted-host → not overwritten")
+    void httpExplicitTrustedHostNotOverwritten(@TempDir Path dir) {
+        assumeTrue(!IS_WINDOWS && hasInterpreter("bash"));
+
+        var svc = newService("http://192.168.1.100:8080/simple", "my-mirror.local");
+
+        // Simulate Docker: both env vars already set
+        var result = svc.executeCode("bash", BASH_SCRIPT, dir, null,
+                Map.of("PIP_INDEX_URL", "http://10.0.0.5:9090/simple",
+                       "PIP_TRUSTED_HOST", "10.0.0.5"),
+                null);
+
+        assertThat(result.getExitCode()).isZero();
+        assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://10.0.0.5:9090/simple");
+        assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=10.0.0.5");
+        // Neither Spring config nor auto-derive should override
+        assertThat(result.getStdout()).doesNotContain("192.168.1.100");
+        assertThat(result.getStdout()).doesNotContain("my-mirror.local");
+    }
+
+    private static boolean hasInterpreter(String name) {
+        try {
+            Process p = new ProcessBuilder(name, "--version")
+                    .redirectErrorStream(true).start();
+            return p.waitFor(10, java.util.concurrent.TimeUnit.SECONDS) && p.exitValue() == 0;
+        } catch (Exception e) {
+            return false;
+        }
+    }
+}