diff --git a/.env.example b/.env.example index a198de4d..b9ba79a1 100644 --- a/.env.example +++ b/.env.example @@ -145,6 +145,20 @@ MATECLAW_SKILL_WORKSPACE_ROOT= MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB= MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB= +# ── Python pip 镜像源（可选）────────────────────────────────────── +# skill 里 Python 脚本缺包时 pip install 走的源。默认用 PyPI 官方源。 +# pip 原生读 PIP_INDEX_URL / PIP_TRUSTED_HOST 环境变量，容器内自动继承。 +# HTTP 源会自动从 URL 推导 PIP_TRUSTED_HOST；自签 HTTPS 需手动填。 +# 互联网加速: https://pypi.tuna.tsinghua.edu.cn/simple +# 局域网私有源: http://192.168.1.100:8080/simple（trusted-host 自动推导） +#PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple +#PIP_TRUSTED_HOST= +# ── 桌面版补充（非 Docker，宿主机直接跑 Java）───────────────────── +# 桌面版不继承上面的 Docker 变量。用 Spring 配置注入 Python 子进程； +# 也可直接设系统环境变量 PIP_INDEX_URL / PIP_TRUSTED_HOST（覆盖更全）。 +#MATECLAW_PIP_INDEX_URL= +#MATECLAW_PIP_TRUSTED_HOST= + # ── Maven 镜像（国内加速）───────────────────────────────────────── # 在中国大陆构建时取消注释，将 Aliyun 仓库优先级提前，大幅提速 mvn 拉包。 # 空值（默认）使用 US Maven Central → Google CDN → Aliyun 的顺序。 diff --git a/docker-compose.yml b/docker-compose.yml index 6668f530..6a8a0411 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -162,6 +162,14 @@ services: # max-total 调多大，单次安装的峰值内存就可能吃多大。 MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_ENTRY_SIZE_MB:-1} MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB: ${MATECLAW_SKILL_UPLOAD_MAX_TOTAL_SIZE_MB:-50} + # pip 镜像源配置（可选）。skill 里的 Python 脚本缺包时 pip install 会走这个源。 + # 留空则用 PyPI 默认源（pypi.org）。pip 原生读 PIP_INDEX_URL / PIP_TRUSTED_HOST + # 环境变量，容器内所有进程（JVM、Python 子进程、bash）自动继承，无需额外配置。 + # HTTP 源会自动从 URL 推导 PIP_TRUSTED_HOST；自签 HTTPS 需手动填。 + # - 互联网加速：PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple + # - 局域网私有源：PIP_INDEX_URL=http://192.168.1.100:8080/simple（trusted-host 自动推导） + PIP_INDEX_URL: ${PIP_INDEX_URL:-} + PIP_TRUSTED_HOST: ${PIP_TRUSTED_HOST:-} # Chromium needs a real /dev/shm. Docker defaults to 64MB which causes # SIGBUS / "Target page closed" errors under load. 2GB is the usual # recommendation for Playwright / headless chrome. diff --git a/mateclaw-server/Dockerfile b/mateclaw-server/Dockerfile index 328f5494..4e9cd4b5 100644 --- a/mateclaw-server/Dockerfile +++ b/mateclaw-server/Dockerfile @@ -100,8 +100,19 @@ RUN apt-get update \ tesseract-ocr \ tesseract-ocr-chi-sim \ tzdata \ + python3-pip \ + python-is-python3 \ + python3-dev \ + build-essential \ && rm -rf /var/lib/apt/lists/* +# Remove PEP 668's EXTERNALLY-MANAGED marker so pip can install packages +# system-wide without --break-system-packages. This is a container — there is +# no host Python environment to protect. Skill scripts and LLM-generated code +# need to `pip install` on the fly; PEP 668 would block every install with +# "error: externally-managed-environment". +RUN rm -f /usr/lib/python3*/EXTERNALLY-MANAGED + # Tell Playwright Java where Microsoft's image stored the browsers. # BrowserLauncher's BUNDLED strategy will then succeed without extra config. ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \ @@ -118,6 +129,7 @@ ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \ ENV SPRING_PROFILES_ACTIVE=mysql COPY --from=builder /build/mateclaw-server/target/*.jar app.jar + EXPOSE 18088 EXPOSE 1455 ENTRYPOINT ["java", "-jar", "app.jar"] diff --git a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillScriptExecutionService.java b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillScriptExecutionService.java index daa93ae6..39c025b5 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillScriptExecutionService.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/runtime/SkillScriptExecutionService.java @@ -2,10 +2,12 @@ package vip.mate.skill.runtime; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.io.IOException; import java.io.InputStream; +import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -28,11 +30,24 @@ import java.util.concurrent.TimeUnit; public class SkillScriptExecutionService { private static final long DEFAULT_TIMEOUT_SECONDS = 30; - private static final long MAX_TIMEOUT_SECONDS = 300; + private static final long MAX_TIMEOUT_SECONDS = 600; private static final int MAX_OUTPUT_BYTES = 50_000; private static final boolean IS_WINDOWS = System.getProperty("os.name", "") .toLowerCase(Locale.ROOT).contains("win"); + /** + * Pip mirror config for desktop (non-Docker) deployments. In Docker these + * arrive as PIP_INDEX_URL / PIP_TRUSTED_HOST env vars (set in + * docker-compose) and are inherited by ProcessBuilder directly. On the + * desktop app Java runs on the host — the host may not have those env + * vars set, so we fall back to Spring config and inject them explicitly. + */ + @Value("${mateclaw.pip.index-url:}") + private String pipIndexUrl; + + @Value("${mateclaw.pip.trusted-host:}") + private String pipTrustedHost; + /** Supported inline-code languages mapped to the temp-file extension. */ private static final Map LANGUAGE_EXTENSIONS = Map.of( "python", ".py", @@ -225,6 +240,7 @@ public class SkillScriptExecutionService { processEnv.put(e.getKey(), e.getValue()); } } + injectPipMirrorEnv(pb); Process process = pb.start(); @@ -272,6 +288,45 @@ public class SkillScriptExecutionService { } } + /** + * Inject pip mirror config into the subprocess environment. + * + *

Three layers, later ones only fill gaps left by earlier ones: + *

    + *
  1. Docker / system env — {@code PIP_INDEX_URL} / {@code PIP_TRUSTED_HOST} + * already in the ProcessBuilder env (inherited from JVM). Nothing to do.
    2. + *
  2. Spring config fallback — for desktop (non-Docker) deployments where + * the host may not have those env vars. Injected only when absent.
    3. + *
  3. Auto-derive {@code PIP_TRUSTED_HOST} — if the index URL is plain + * HTTP and no trusted-host is set, pip blocks the download. Extract + * the host from the URL so the user only needs to set one variable.
    4. + *
+ */ + private void injectPipMirrorEnv(ProcessBuilder pb) { + Map env = pb.environment(); + + // Layer 2: Spring config fallback (desktop) + if (pipIndexUrl != null && !pipIndexUrl.isBlank() + && !env.containsKey("PIP_INDEX_URL")) { + env.put("PIP_INDEX_URL", pipIndexUrl); + } + if (pipTrustedHost != null && !pipTrustedHost.isBlank() + && !env.containsKey("PIP_TRUSTED_HOST")) { + env.put("PIP_TRUSTED_HOST", pipTrustedHost); + } + + // Layer 3: auto-derive trusted-host for HTTP sources + String indexUrl = env.get("PIP_INDEX_URL"); + if (indexUrl != null && !indexUrl.isBlank() + && !env.containsKey("PIP_TRUSTED_HOST") + && indexUrl.startsWith("http://")) { + String host = URI.create(indexUrl).getHost(); + if (host != null && !host.isEmpty()) { + env.put("PIP_TRUSTED_HOST", host); + } + } + } + private static String readFileTruncated(Path file, int maxBytes) { try { if (file == null || !Files.exists(file)) return ""; diff --git a/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillScriptExecutionServicePipMirrorTest.java b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillScriptExecutionServicePipMirrorTest.java new file mode 100644 index 00000000..4edce54e --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/runtime/SkillScriptExecutionServicePipMirrorTest.java @@ -0,0 +1,165 @@ +package vip.mate.skill.runtime; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.springframework.test.util.ReflectionTestUtils; + +import java.nio.file.Path; +import java.util.Locale; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Tests for the pip mirror env-var injection in + * {@link SkillScriptExecutionService}. + * + *

Two scenarios: + *

+ * + *

Bash-gated so the suite stays green on hosts without bash (e.g. Windows CI). + */ +class SkillScriptExecutionServicePipMirrorTest { + + private static final boolean IS_WINDOWS = + System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win"); + + private static final String BASH_SCRIPT = + "echo PIP_INDEX_URL=$PIP_INDEX_URL\n" + + "echo PIP_TRUSTED_HOST=$PIP_TRUSTED_HOST\n"; + + private SkillScriptExecutionService newService(String indexUrl, String trustedHost) { + SkillScriptExecutionService svc = new SkillScriptExecutionService(); + ReflectionTestUtils.setField(svc, "pipIndexUrl", indexUrl); + ReflectionTestUtils.setField(svc, "pipTrustedHost", trustedHost); + return svc; + } + + @Test + @DisplayName("desktop fallback: Spring config injected when PIP_INDEX_URL absent from env") + void springConfigInjectedWhenEnvAbsent(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + // Skip if the host already has PIP_INDEX_URL set — the inherited env + // var would make containsKey() true, hiding the Spring fallback path. + assumeTrue(System.getenv("PIP_INDEX_URL") == null, + "PIP_INDEX_URL already set in host environment"); + + var svc = newService("http://192.168.1.100:8080/simple", "192.168.1.100"); + + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://192.168.1.100:8080/simple"); + assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=192.168.1.100"); + } + + @Test + @DisplayName("Docker case: env var takes precedence over Spring config") + void envVarTakesPrecedenceOverSpringConfig(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + + var svc = newService("http://spring-fallback:8080/simple", "spring-fallback"); + + // Simulate Docker: PIP_INDEX_URL already in the subprocess env + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, + Map.of("PIP_INDEX_URL", "http://docker-env:9090/simple", + "PIP_TRUSTED_HOST", "docker-env"), + null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://docker-env:9090/simple"); + assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=docker-env"); + assertThat(result.getStdout()).doesNotContain("spring-fallback"); + } + + @Test + @DisplayName("no config: nothing injected, pip uses defaults") + void noPipConfigNoInjection(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + assumeTrue(System.getenv("PIP_INDEX_URL") == null, + "PIP_INDEX_URL already set in host environment"); + + var svc = newService("", ""); + + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()).doesNotContain("http://"); + } + + @Test + @DisplayName("HTTPS index-url, no trusted-host → not auto-derived") + void onlyIndexUrlSetHttps(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + assumeTrue(System.getenv("PIP_INDEX_URL") == null, + "PIP_INDEX_URL already set in host environment"); + + var svc = newService("https://pypi.tuna.tsinghua.edu.cn/simple", ""); + + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()) + .contains("PIP_INDEX_URL=https://pypi.tuna.tsinghua.edu.cn/simple"); + // HTTPS + valid cert → no trusted-host needed, should NOT be auto-derived + assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=\n"); + } + + @Test + @DisplayName("HTTP index-url, no trusted-host → auto-derived from URL") + void httpAutoDeriveTrustedHost(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + assumeTrue(System.getenv("PIP_INDEX_URL") == null, + "PIP_INDEX_URL already set in host environment"); + + var svc = newService("http://192.168.1.100:8080/simple", ""); + + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, Map.of(), null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()) + .contains("PIP_INDEX_URL=http://192.168.1.100:8080/simple"); + // HTTP source → trusted-host auto-derived from URL + assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=192.168.1.100"); + } + + @Test + @DisplayName("HTTP index-url with explicit trusted-host → not overwritten") + void httpExplicitTrustedHostNotOverwritten(@TempDir Path dir) { + assumeTrue(!IS_WINDOWS && hasInterpreter("bash")); + + var svc = newService("http://192.168.1.100:8080/simple", "my-mirror.local"); + + // Simulate Docker: both env vars already set + var result = svc.executeCode("bash", BASH_SCRIPT, dir, null, + Map.of("PIP_INDEX_URL", "http://10.0.0.5:9090/simple", + "PIP_TRUSTED_HOST", "10.0.0.5"), + null); + + assertThat(result.getExitCode()).isZero(); + assertThat(result.getStdout()).contains("PIP_INDEX_URL=http://10.0.0.5:9090/simple"); + assertThat(result.getStdout()).contains("PIP_TRUSTED_HOST=10.0.0.5"); + // Neither Spring config nor auto-derive should override + assertThat(result.getStdout()).doesNotContain("192.168.1.100"); + assertThat(result.getStdout()).doesNotContain("my-mirror.local"); + } + + private static boolean hasInterpreter(String name) { + try { + Process p = new ProcessBuilder(name, "--version") + .redirectErrorStream(true).start(); + return p.waitFor(10, java.util.concurrent.TimeUnit.SECONDS) && p.exitValue() == 0; + } catch (Exception e) { + return false; + } + } +}