From f8a3d0d1530abc5b5e11c2313384ca80e7cd7797 Mon Sep 17 00:00:00 2001 From: mateaix <7333791@qq.com> Date: Sat, 19 Sep 2026 21:49:06 +0800 Subject: [PATCH] fix(skills): support document and folder uploads (#609) --- .../skill/controller/SkillController.java | 73 ++++++++++++- .../vip/mate/skill/model/SkillFileEntity.java | 21 +++- .../mate/skill/service/SkillFileService.java | 60 +++++++---- .../mate/skill/workspace/SkillFileSyncer.java | 18 +++- .../h2/V202__skill_file_encoding.sql | 2 + .../kingbase/V202__skill_file_encoding.sql | 2 + .../mysql/V202__skill_file_encoding.sql | 2 + .../SkillControllerBundleFilesTest.java | 59 ++++++++++ .../SkillFileEncodingMigrationTest.java | 44 ++++++++ .../skill/service/SkillFileServiceTest.java | 32 ++++++ .../skill/workspace/SkillFileSyncerTest.java | 31 ++++++ .../src/api/__tests__/skillFiles.test.ts | 29 +++++ mateclaw-ui/src/api/index.ts | 9 ++ mateclaw-ui/src/i18n/locales/en-US.ts | 14 +++ mateclaw-ui/src/i18n/locales/zh-CN.ts | 14 +++ .../utils/__tests__/skillFileUpload.test.ts | 30 ++++++ mateclaw-ui/src/utils/skillFileUpload.ts | 20 ++++ mateclaw-ui/src/views/SkillMarket.vue | 101 +++++++++++++++++- 18 files changed, 526 insertions(+), 35 deletions(-) create mode 100644 mateclaw-server/src/main/resources/db/migration/h2/V202__skill_file_encoding.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/kingbase/V202__skill_file_encoding.sql create mode 100644 mateclaw-server/src/main/resources/db/migration/mysql/V202__skill_file_encoding.sql create mode 100644 mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileEncodingMigrationTest.java create mode 100644 mateclaw-ui/src/api/__tests__/skillFiles.test.ts create mode 100644 mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts create mode 100644 mateclaw-ui/src/utils/skillFileUpload.ts diff --git a/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java b/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java index fb75208e..7d35a818 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/controller/SkillController.java @@ -5,6 +5,13 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.http.ResponseEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.MediaType; +import org.springframework.http.ContentDisposition; +import org.springframework.web.server.ResponseStatusException; +import org.springframework.http.HttpStatus; import vip.mate.common.result.R; import vip.mate.agent.AgentService; import vip.mate.agent.binding.model.AgentSkillBinding; @@ -42,6 +49,8 @@ import vip.mate.skill.lifecycle.SkillCuratorReportStore; import vip.mate.skill.lifecycle.SkillLifecycleService; import vip.mate.skill.lifecycle.model.SkillSnapshotEntity; +import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -379,6 +388,7 @@ public class SkillController { Map item = new LinkedHashMap<>(); item.put("path", row.getFilePath()); item.put("size", row.getContentSize()); + item.put("binary", row.isBinary()); item.put("sha256", row.getSha256()); item.put("updateTime", row.getUpdateTime()); out.add(item); @@ -404,7 +414,8 @@ public class SkillController { } Map body = new LinkedHashMap<>(); body.put("path", row.getFilePath()); - body.put("content", row.getContent() == null ? "" : row.getContent()); + body.put("binary", row.isBinary()); + body.put("content", row.isBinary() ? "" : row.getContent() == null ? "" : row.getContent()); body.put("size", row.getContentSize()); body.put("sha256", row.getSha256()); body.put("updateTime", row.getUpdateTime()); @@ -429,6 +440,10 @@ public class SkillController { if (normalized == null) { return R.fail("Invalid file path — must be under scripts/, references/ or templates/, no '..'."); } + SkillFileEntity existing = skillFileService.getFile(id, normalized); + if (existing != null && existing.isBinary()) { + return R.fail("Binary files cannot be edited as text; upload a replacement instead."); + } String content = body.get("content"); if (content == null) { return R.fail("content is required (use the delete endpoint to remove a file)."); @@ -479,6 +494,58 @@ public class SkillController { return R.ok(Map.of("path", normalized, "removed", removed)); } + private static final int MAX_UPLOAD_BYTES = 10 * 1024 * 1024; + + @PostMapping(value = "/{id}/files/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + @RequireWorkspaceRole("admin") + public R> uploadBundleFile(@PathVariable Long id, + @RequestPart("file") MultipartFile file, @RequestParam String path, + @RequestParam(defaultValue = "false") boolean overwrite, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) throws IOException { + rejectVirtualSkillMutation(id); + SkillEntity skill = skillService.getSkill(id); + if (skill == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Skill not found"); + verifyResourceWorkspace(skill, workspaceId); + if (Boolean.TRUE.equals(skill.getBuiltin())) return R.fail("Builtin skill files are read-only."); + String normalized = normalizeBundlePath(path); + if (normalized == null) return R.fail("Invalid file path: " + path); + if (file.getSize() > MAX_UPLOAD_BYTES) return R.fail("File exceeds the 10 MiB limit."); + if (!overwrite && skillFileService.getFile(id, normalized) != null) { + return R.fail("File already exists; confirm replacement before uploading: " + normalized); + } + byte[] bytes; + try (var input = file.getInputStream()) { + bytes = input.readNBytes(MAX_UPLOAD_BYTES + 1); + } + if (bytes.length > MAX_UPLOAD_BYTES) return R.fail("File exceeds the 10 MiB limit."); + SkillFileEntity row = skillFileService.upsertBytes(id, normalized, bytes); + if (!skillFileSyncer.syncFile(skill, row)) { + return R.fail("File saved, but workspace synchronization failed: " + normalized + + ". Check directory permissions or conflicting paths, then upload again."); + } + skillRuntimeService.rescanSingle(skill); + return R.ok(Map.of("path", normalized, "size", row.getContentSize(), "binary", row.isBinary())); + } + + @GetMapping("/{id}/files/download") + @RequireWorkspaceRole("member") + public ResponseEntity downloadBundleFile(@PathVariable Long id, @RequestParam String path, + @RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) { + SkillEntity skill = skillService.getSkill(id); + if (skill == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Skill not found"); + verifyResourceWorkspace(skill, workspaceId); + String normalized = normalizeBundlePath(path); + if (normalized == null) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid file path"); + SkillFileEntity row = skillFileService.getFile(id, normalized); + if (row == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "File not found"); + return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM) + .header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment() + .filename(normalized.substring(normalized.lastIndexOf('/') + 1), StandardCharsets.UTF_8) + .build().toString()) + .header("X-Content-Type-Options", "nosniff") + .body(row.contentBytes()); + } + /** * Normalize a bundle-relative path and enforce the same envelope the * store and workspace cache use: forward slashes, must sit under a @@ -491,7 +558,9 @@ public class SkillController { static String normalizeBundlePath(String path) { if (path == null || path.isBlank()) return null; String p = path.strip().replace('\\', '/'); - if (p.startsWith("/") || p.contains("..") || p.contains("//") || p.endsWith("/")) return null; + if (p.length() > 512 || p.chars().anyMatch(c -> c < 32 || c == 127) + || p.contains(":") || p.contains("/./") || p.endsWith("/.") + || p.startsWith("/") || p.contains("..") || p.contains("//") || p.endsWith("/")) return null; if (!SkillBundleFiles.isDbEligible(p)) return null; int slash = p.indexOf('/'); if (slash == p.length() - 1) return null; diff --git a/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java index 6169fa25..5b5098b1 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/model/SkillFileEntity.java @@ -32,17 +32,30 @@ public class SkillFileEntity { /** * Path relative to the skill workspace root, always starting with - * {@code scripts/} or {@code references/}. Forward slashes only. + * {@code scripts/}, {@code references/} or {@code templates/}. Forward slashes only. */ private String filePath; - /** UTF-8 text content. Per-file size bounded by ZipSkillFetcher (1MB). */ + /** UTF-8 text or base64-encoded attachment bytes, according to contentEncoding. */ private String content; - /** Length of {@link #content} in bytes — kept so listings can sort/audit without loading the blob. */ + /** null on legacy rows means UTF-8. */ + private String contentEncoding; + + public boolean isBinary() { + return "base64".equals(contentEncoding); + } + + public byte[] contentBytes() { + String value = content == null ? "" : content; + return isBinary() ? java.util.Base64.getDecoder().decode(value) + : value.getBytes(java.nio.charset.StandardCharsets.UTF_8); + } + + /** Length of the original file in bytes — kept so listings can sort/audit without loading the blob. */ private Integer contentSize; - /** SHA-256 of {@link #content}; used by the syncer to skip no-op writes. */ + /** SHA-256 of the original bytes; used by the syncer to skip no-op writes. */ private String sha256; @TableField(fill = FieldFill.INSERT) diff --git a/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java index 3cea4cec..fa8d779c 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/service/SkillFileService.java @@ -46,9 +46,13 @@ public class SkillFileService { /** Compute SHA-256 hex of a UTF-8 string (used for idempotent diffs). */ public static String sha256Hex(String content) { if (content == null) content = ""; + return sha256Hex(content.getBytes(StandardCharsets.UTF_8)); + } + + public static String sha256Hex(byte[] bytes) { try { MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8)); + byte[] digest = md.digest(bytes); StringBuilder sb = new StringBuilder(digest.length * 2); for (byte b : digest) sb.append(String.format("%02x", b)); return sb.toString(); @@ -132,6 +136,7 @@ public class SkillFileService { row.setSkillId(skillId); row.setFilePath(path); row.setContent(content); + row.setContentEncoding("utf8"); row.setContentSize(size); row.setSha256(hash); row.setCreateTime(now); @@ -140,6 +145,7 @@ public class SkillFileService { written++; } else if (!hash.equals(prior.getSha256())) { prior.setContent(content); + prior.setContentEncoding("utf8"); prior.setContentSize(size); prior.setSha256(hash); prior.setUpdateTime(now); @@ -193,32 +199,40 @@ public class SkillFileService { */ @Transactional public SkillFileEntity upsertFile(Long skillId, String filePath, String content) { - String safeContent = content == null ? "" : content; - String hash = sha256Hex(safeContent); - LocalDateTime now = LocalDateTime.now(); + return upsertBytes(skillId, filePath, + (content == null ? "" : content).getBytes(StandardCharsets.UTF_8)); + } - SkillFileEntity existing = getFile(skillId, filePath); - if (existing != null) { - if (hash.equals(existing.getSha256())) { - return existing; - } - existing.setContent(safeContent); - existing.setContentSize(safeContent.getBytes(StandardCharsets.UTF_8).length); - existing.setSha256(hash); - existing.setUpdateTime(now); - mapper.updateById(existing); - return existing; + /** Preserve arbitrary bytes; only strictly valid UTF-8 without control bytes is editable text. */ + @Transactional + public SkillFileEntity upsertBytes(Long skillId, String filePath, byte[] bytes) { + String text = null; + try { + text = StandardCharsets.UTF_8.newDecoder().decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + if (text.codePoints().anyMatch(c -> c < 32 && c != '\n' && c != '\r' && c != '\t')) text = null; + } catch (java.nio.charset.CharacterCodingException binary) { + // Keep the original bytes below. } - - SkillFileEntity row = new SkillFileEntity(); - row.setSkillId(skillId); - row.setFilePath(filePath); - row.setContent(safeContent); - row.setContentSize(safeContent.getBytes(StandardCharsets.UTF_8).length); + String encoding = text == null ? "base64" : "utf8"; + String content = text == null ? java.util.Base64.getEncoder().encodeToString(bytes) : text; + String hash = sha256Hex(bytes); + SkillFileEntity row = getFile(skillId, filePath); + boolean insert = row == null; + if (!insert && hash.equals(row.getSha256())) return row; + LocalDateTime now = LocalDateTime.now(); + if (insert) { + row = new SkillFileEntity(); + row.setSkillId(skillId); + row.setFilePath(filePath); + row.setCreateTime(now); + } + row.setContent(content); + row.setContentEncoding(encoding); + row.setContentSize(bytes.length); row.setSha256(hash); - row.setCreateTime(now); row.setUpdateTime(now); - mapper.insert(row); + if (insert) mapper.insert(row); + else mapper.updateById(row); return row; } diff --git a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java index 488ca086..ac865b92 100644 --- a/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java +++ b/mateclaw-server/src/main/java/vip/mate/skill/workspace/SkillFileSyncer.java @@ -161,6 +161,12 @@ public class SkillFileSyncer { return ingested.size(); } + /** Materialize only the uploaded row; folder uploads must not re-read the full bundle per file. */ + public boolean syncFile(SkillEntity skill, SkillFileEntity row) { + Path workspace = workspaceManager.resolveConventionPath(skill.getName(), skill.getWorkspaceId()); + return materializeOne(workspace, row) != MaterializeOutcome.SKIPPED; + } + private enum MaterializeOutcome { WROTE, CURRENT, SKIPPED } private MaterializeOutcome materializeOne(Path workspaceDir, SkillFileEntity row) { @@ -183,17 +189,21 @@ public class SkillFileSyncer { } try { - String content = row.getContent() == null ? "" : row.getContent(); + // Do not follow links created by a skill script outside its workspace. + for (Path part = target; part != null && part.startsWith(workspaceDir); part = part.getParent()) { + if (Files.isSymbolicLink(part)) return MaterializeOutcome.SKIPPED; + } + byte[] content = row.contentBytes(); if (Files.exists(target)) { - String onDisk = Files.readString(target, StandardCharsets.UTF_8); + byte[] onDisk = Files.readAllBytes(target); if (SkillFileService.sha256Hex(onDisk).equals(row.getSha256())) { return MaterializeOutcome.CURRENT; } } Files.createDirectories(target.getParent()); - Files.writeString(target, content, StandardCharsets.UTF_8); + Files.write(target, content); return MaterializeOutcome.WROTE; - } catch (IOException e) { + } catch (IOException | IllegalArgumentException e) { log.warn("Failed to materialize skill_file {} → {}: {}", row.getId(), target, e.getMessage()); return MaterializeOutcome.SKIPPED; } diff --git a/mateclaw-server/src/main/resources/db/migration/h2/V202__skill_file_encoding.sql b/mateclaw-server/src/main/resources/db/migration/h2/V202__skill_file_encoding.sql new file mode 100644 index 00000000..9572be99 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/h2/V202__skill_file_encoding.sql @@ -0,0 +1,2 @@ +-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column. +ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/kingbase/V202__skill_file_encoding.sql b/mateclaw-server/src/main/resources/db/migration/kingbase/V202__skill_file_encoding.sql new file mode 100644 index 00000000..9572be99 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/kingbase/V202__skill_file_encoding.sql @@ -0,0 +1,2 @@ +-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column. +ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL; diff --git a/mateclaw-server/src/main/resources/db/migration/mysql/V202__skill_file_encoding.sql b/mateclaw-server/src/main/resources/db/migration/mysql/V202__skill_file_encoding.sql new file mode 100644 index 00000000..9572be99 --- /dev/null +++ b/mateclaw-server/src/main/resources/db/migration/mysql/V202__skill_file_encoding.sql @@ -0,0 +1,2 @@ +-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column. +ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL; diff --git a/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerBundleFilesTest.java b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerBundleFilesTest.java index 3b699ae3..505660b0 100644 --- a/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerBundleFilesTest.java +++ b/mateclaw-server/src/test/java/vip/mate/skill/controller/SkillControllerBundleFilesTest.java @@ -188,4 +188,63 @@ class SkillControllerBundleFilesTest { assertThat(resp.getMsg()).contains("read-only"); verify(fileService, never()).deleteFile(any(), any()); } + @Test + void uploadsAndDownloadsOriginalDocumentBytes() throws Exception { + when(skillService.getSkill(SID)).thenReturn(skill(false)); + byte[] bytes = {80, 75, 3, 4, 0, (byte) 255}; + var file = new org.springframework.mock.web.MockMultipartFile("file", "报告.docx", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", bytes); + var row = row("references/资料/报告.docx", java.util.Base64.getEncoder().encodeToString(bytes)); + row.setContentEncoding("base64"); + row.setContentSize(bytes.length); + when(fileService.upsertBytes(SID, row.getFilePath(), bytes)).thenReturn(row); + when(fileSyncer.syncFile(any(SkillEntity.class), any(SkillFileEntity.class))).thenReturn(true); + var result = controller.uploadBundleFile(SID, file, row.getFilePath(), false, null); + assertThat(result.getData()).containsEntry("binary", true).containsEntry("size", bytes.length); + verify(fileSyncer).syncFile(any(SkillEntity.class), any(SkillFileEntity.class)); + when(fileService.getFile(SID, row.getFilePath())).thenReturn(row); + assertThat(controller.downloadBundleFile(SID, row.getFilePath(), null).getBody()).isEqualTo(bytes); + assertThat(controller.getBundleFileContent(SID, row.getFilePath(), null).getData()) + .containsEntry("binary", true).containsEntry("content", ""); + assertThat(controller.putBundleFileContent(SID, + Map.of("path", row.getFilePath(), "content", "corrupt"), null).getMsg()).contains("Binary"); + } + + @Test + void uploadRejectsUnsafePathsReadonlyAndUnconfirmedReplacement() throws Exception { + var file = new org.springframework.mock.web.MockMultipartFile("file", "a.txt", "text/plain", new byte[]{1}); + when(skillService.getSkill(SID)).thenReturn(skill(false)); + for (String path : List.of("references/../bad", "references/./bad", "references/a\u0000b", "/tmp/a")) { + assertThat(controller.uploadBundleFile(SID, file, path, false, null).getMsg()).contains("Invalid"); + } + when(fileService.getFile(SID, "references/a.txt")).thenReturn(row("references/a.txt", "old")); + assertThat(controller.uploadBundleFile(SID, file, "references/a.txt", false, null).getMsg()).contains("already exists"); + when(skillService.getSkill(SID)).thenReturn(skill(true)); + assertThat(controller.uploadBundleFile(SID, file, "references/a.txt", true, null).getMsg()).contains("read-only"); + verify(fileService, never()).upsertBytes(any(), any(), any()); + } + + @Test + void uploadRejectsOversizedFilesBeforeReading() throws Exception { + when(skillService.getSkill(SID)).thenReturn(skill(false)); + var file = mock(org.springframework.web.multipart.MultipartFile.class); + when(file.getSize()).thenReturn(10L * 1024 * 1024 + 1); + assertThat(controller.uploadBundleFile(SID, file, "references/a.docx", false, null).getMsg()).contains("10 MiB"); + verify(file, never()).getInputStream(); + } + + @Test + void attachmentEndpointsRejectOtherWorkspacesBeforeReadingBytes() { + when(skillService.getSkill(SID)).thenReturn(skill(false)); + var file = mock(org.springframework.web.multipart.MultipartFile.class); + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + controller.uploadBundleFile(SID, file, "references/a.docx", false, 2L)) + .isInstanceOf(vip.mate.exception.MateClawException.class); + org.assertj.core.api.Assertions.assertThatThrownBy(() -> + controller.downloadBundleFile(SID, "references/a.docx", 2L)) + .isInstanceOf(vip.mate.exception.MateClawException.class); + verify(fileService, never()).getFile(any(), any()); + verify(fileService, never()).upsertBytes(any(), any(), any()); + } + } diff --git a/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileEncodingMigrationTest.java b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileEncodingMigrationTest.java new file mode 100644 index 00000000..8dd4a745 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileEncodingMigrationTest.java @@ -0,0 +1,44 @@ +package vip.mate.skill.service; + +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.DriverManager; +import java.util.Base64; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; + +class SkillFileEncodingMigrationTest { + @Test + void migrationPreservesLegacyTextAndStoresBinaryContent() throws Exception { + for (String dialect : List.of("h2", "mysql", "kingbase")) { + String migration = Files.readString(Path.of("src/main/resources/db/migration", dialect, + "V202__skill_file_encoding.sql")); + // The added column uses portable SQL. Exercise it on populated tables, + // including dialect modes; native MySQL/Kingbase still require deployment verification. + String mode = dialect.equals("kingbase") ? "PostgreSQL" : "MySQL"; + try (var connection = DriverManager.getConnection("jdbc:h2:mem:encoding_" + dialect + ";MODE=" + mode); + var statement = connection.createStatement()) { + statement.execute("CREATE TABLE mate_skill_file (id BIGINT PRIMARY KEY, content TEXT)"); + statement.execute("INSERT INTO mate_skill_file VALUES (1, '制度说明')"); + statement.execute(migration); + try (var result = statement.executeQuery("SELECT content, content_encoding FROM mate_skill_file WHERE id=1")) { + assertTrue(result.next()); + assertEquals("制度说明", result.getString(1)); + assertEquals("utf8", result.getString(2)); + } + byte[] bytes = {80, 75, 0, (byte) 255}; + try (var insert = connection.prepareStatement("INSERT INTO mate_skill_file VALUES (2, ?, 'base64')")) { + insert.setString(1, Base64.getEncoder().encodeToString(bytes)); + insert.executeUpdate(); + } + try (var result = statement.executeQuery("SELECT content FROM mate_skill_file WHERE id=2")) { + assertTrue(result.next()); + assertArrayEquals(bytes, Base64.getDecoder().decode(result.getString(1))); + } + } + } + } +} diff --git a/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java index d9c0389d..02a63d54 100644 --- a/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java +++ b/mateclaw-server/src/test/java/vip/mate/skill/service/SkillFileServiceTest.java @@ -114,4 +114,36 @@ class SkillFileServiceTest { e.setSha256(SkillFileService.sha256Hex(content)); return e; } + @Test + void binaryUploadPreservesBytesSizeAndHash() { + byte[] bytes = {80, 75, 3, 4, 0, (byte) 255, (byte) 128}; + SkillFileEntity row = service.upsertBytes(42L, "templates/报告.xlsx", bytes); + assertTrue(row.isBinary()); + assertArrayEquals(bytes, row.contentBytes()); + assertEquals(bytes.length, row.getContentSize()); + assertEquals(SkillFileService.sha256Hex(bytes), row.getSha256()); + verify(mapper).insert(row); + } + + @Test + void uploadedUtf8RemainsEditableAndLegacyRowsDecodeUnchanged() { + String text = "制度说明\n中文"; + SkillFileEntity row = service.upsertBytes(42L, "references/制度.md", + text.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + assertFalse(row.isBinary()); + assertEquals(text, row.getContent()); + row.setContentEncoding(null); + assertArrayEquals(text.getBytes(java.nio.charset.StandardCharsets.UTF_8), row.contentBytes()); + } + + @Test + void replacingBinaryWithTextClearsEncoding() { + SkillFileEntity prior = newRow(1L, "references/file", "AA=="); + prior.setContentEncoding("base64"); + when(mapper.selectOne(any())).thenReturn(prior); + SkillFileEntity row = service.upsertBytes(42L, "references/file", "hello".getBytes()); + assertFalse(row.isBinary()); + assertArrayEquals("hello".getBytes(), row.contentBytes()); + } + } diff --git a/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java index a4dc1bf2..f47e8cf6 100644 --- a/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java +++ b/mateclaw-server/src/test/java/vip/mate/skill/workspace/SkillFileSyncerTest.java @@ -167,4 +167,35 @@ class SkillFileSyncerTest { e.setSha256(SkillFileService.sha256Hex(content)); return e; } + @Test + void binaryAttachmentSurvivesRestoreAndStaleCacheReplacement() throws IOException { + SkillEntity skill = newSkill(10L, "demo"); + byte[] bytes = {80, 75, 3, 4, 0, (byte) 255}; + SkillFileEntity row = newRow(1L, 10L, "templates/报告.xlsx", ""); + row.setContent(java.util.Base64.getEncoder().encodeToString(bytes)); + row.setContentEncoding("base64"); + row.setSha256(SkillFileService.sha256Hex(bytes)); + when(mapper.selectList(any())).thenReturn(List.of(row)); + Path target = tmp.resolve("1/demo/templates/报告.xlsx"); + assertEquals(1, syncer.syncOne(skill).filesMaterialized()); + assertArrayEquals(bytes, Files.readAllBytes(target)); + assertEquals(1, syncer.syncOne(skill).filesAlreadyCurrent()); + Files.write(target, new byte[]{(byte) 255, 0, 1}); + assertEquals(1, syncer.syncOne(skill).filesMaterialized()); + assertArrayEquals(bytes, Files.readAllBytes(target)); + } + + @Test + void refusesSymlinkInsideSkillWorkspace() throws IOException { + SkillEntity skill = newSkill(10L, "demo"); + Path workspace = tmp.resolve("1/demo"); + Path outside = tmp.resolve("outside"); + Files.createDirectories(workspace); + Files.createDirectories(outside); + Files.createSymbolicLink(workspace.resolve("references"), outside); + when(mapper.selectList(any())).thenReturn(List.of(newRow(1L, 10L, "references/a.txt", "secret"))); + assertEquals(0, syncer.syncOne(skill).filesMaterialized()); + assertFalse(Files.exists(outside.resolve("a.txt"))); + } + } diff --git a/mateclaw-ui/src/api/__tests__/skillFiles.test.ts b/mateclaw-ui/src/api/__tests__/skillFiles.test.ts new file mode 100644 index 00000000..6a69df59 --- /dev/null +++ b/mateclaw-ui/src/api/__tests__/skillFiles.test.ts @@ -0,0 +1,29 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { http, skillApi } from '@/api/index' + +describe('skill attachment API', () => { + afterEach(() => vi.restoreAllMocks()) + + it('sends the original file and relative path without coercing large skill IDs', async () => { + const post = vi.spyOn(http, 'post').mockResolvedValue({} as never) + const file = new File([new Uint8Array([80, 75, 0, 255])], '模板.xlsx') + await skillApi.uploadFile('9007199254740993', file, 'templates/部门/模板.xlsx') + const [url, body] = post.mock.calls[0] + expect(url).toBe('/skills/9007199254740993/files/upload') + const form = body as FormData + expect(form.get('path')).toBe('templates/部门/模板.xlsx') + expect(form.get('overwrite')).toBe('false') + expect(await (form.get('file') as File).arrayBuffer()).toEqual(await file.arrayBuffer()) + await skillApi.uploadFile('9007199254740993', file, 'templates/部门/模板.xlsx', true) + expect((post.mock.calls[1][1] as FormData).get('overwrite')).toBe('true') + }) + + it('downloads authenticated bytes through the shared HTTP client', async () => { + const blob = new Blob([new Uint8Array([0, 255])]) + const get = vi.spyOn(http, 'get').mockResolvedValue(blob as never) + expect(await skillApi.downloadFile('9007199254740993', 'references/a.docx')).toBe(blob) + expect(get).toHaveBeenCalledWith('/skills/9007199254740993/files/download', { + params: { path: 'references/a.docx' }, responseType: 'blob', + }) + }) +}) diff --git a/mateclaw-ui/src/api/index.ts b/mateclaw-ui/src/api/index.ts index 138d236e..d840208a 100644 --- a/mateclaw-ui/src/api/index.ts +++ b/mateclaw-ui/src/api/index.ts @@ -275,6 +275,15 @@ export const skillApi = { * re-resolve the skill. */ listFiles: (id: string | number) => http.get(`/skills/${id}/files`), + uploadFile: (id: string | number, file: File, path: string, overwrite = false) => { + const form = new FormData() + form.append('file', file) + form.append('path', path) + form.append('overwrite', String(overwrite)) + return http.post(`/skills/${id}/files/upload`, form, { timeout: 120000 }) + }, + downloadFile: (id: string | number, path: string): Promise => + http.get(`/skills/${id}/files/download`, { params: { path }, responseType: 'blob' }) as unknown as Promise, getFileContent: (id: string | number, path: string) => http.get(`/skills/${id}/files/content`, { params: { path } }), saveFileContent: (id: string | number, path: string, content: string) => diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 8651d708..8478ca45 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -4305,6 +4305,20 @@ export default { fileEdit: 'Edit', fileDelete: 'Delete', fileDeleteConfirm: 'Delete "{path}"? This removes both the database record and the workspace file. Cannot be undone.', + uploadFiles: 'Upload files', + uploadFolder: 'Upload folder', + uploadDestination: 'Destination', + uploadHint: 'Supports Word, Excel and other files. Folder structure is preserved. Up to 10 MiB per file, 100 files and 50 MiB per batch. Empty folders are not uploaded.', + uploadOverwrite: 'Replace these existing files? {paths}', + uploadComplete: 'Uploaded {count} files', + uploadFailed: 'Uploaded {count} files; remaining uploads did not finish: {error}', + uploadErrors: { + batchLimit: 'Each batch is limited to 100 files and 50 MiB.', + fileLimit: 'Each file must be at most 10 MiB.', + invalidPath: 'File paths are invalid, duplicated or too long.', + }, + fileDownload: 'Download', + binaryFileHint: 'This file is preserved in its original format. Download to view it, or upload a replacement to change it.', fileSaved: 'File saved', fileDeleted: 'File deleted', fileLoadFailed: 'Failed to read file', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 04abaa5e..39a7c772 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -4397,6 +4397,20 @@ export default { fileEdit: '编辑', fileDelete: '删除', fileDeleteConfirm: '删除文件 "{path}"？此操作同时移除数据库记录和工作区文件，不可撤销。', + uploadFiles: '上传文件', + uploadFolder: '上传文件夹', + uploadDestination: '保存目录', + uploadHint: '支持 Word、Excel 等文件，文件夹会保留目录层级。单个文件不超过 10 MiB，每批最多 100 个文件、合计 50 MiB；空文件夹不会上传。', + uploadOverwrite: '以下同名文件将被替换，是否继续？{paths}', + uploadComplete: '已上传 {count} 个文件', + uploadFailed: '已上传 {count} 个文件，其余未完成：{error}', + uploadErrors: { + batchLimit: '每批最多上传 100 个文件，合计不超过 50 MiB。', + fileLimit: '单个文件不能超过 10 MiB。', + invalidPath: '文件路径无效、重复或过长，请检查文件名和目录。', + }, + fileDownload: '下载', + binaryFileHint: '此文件保留原始格式，可下载查看；如需修改，请上传替换文件。', fileSaved: '文件已保存', fileDeleted: '文件已删除', fileLoadFailed: '读取文件失败', diff --git a/mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts b/mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts new file mode 100644 index 00000000..0bf96bd5 --- /dev/null +++ b/mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { planSkillFileUploads } from '../skillFileUpload' + +function file(name: string, relative = '', size = 1): File { + const result = new File(['x'], name) + Object.defineProperty(result, 'webkitRelativePath', { value: relative }) + Object.defineProperty(result, 'size', { value: size }) + return result +} + +describe('skill file uploads', () => { + it('keeps Unicode folder structure and original files under the chosen bucket', () => { + const document = file('报告.docx', '制度/部门/报告.docx') + expect(planSkillFileUploads([document], 'references')).toEqual([ + { file: document, path: 'references/制度/部门/报告.docx' }, + ]) + expect(planSkillFileUploads([file('report.xlsx')], 'templates')[0].path).toBe('templates/report.xlsx') + }) + it('rejects traversal, absolute paths, duplicate paths and control characters', () => { + for (const path of ['../a', '/tmp/a', 'x/./a', 'x//a', 'x\\a', 'x\0a', 'C:/a']) { + expect(() => planSkillFileUploads([file('a', path)], 'references')).toThrow('invalidPath') + } + expect(() => planSkillFileUploads([file('a'), file('a')], 'references')).toThrow('invalidPath') + }) + it('bounds individual files and batches before sending any requests', () => { + expect(() => planSkillFileUploads([file('a', '', 10 * 1024 * 1024 + 1)], 'references')).toThrow('fileLimit') + expect(() => planSkillFileUploads(Array.from({ length: 101 }, (_, i) => file(String(i))), 'references')).toThrow('batchLimit') + expect(() => planSkillFileUploads(Array.from({ length: 6 }, (_, i) => file(String(i), '', 10 * 1024 * 1024)), 'references')).toThrow('batchLimit') + }) +}) diff --git a/mateclaw-ui/src/utils/skillFileUpload.ts b/mateclaw-ui/src/utils/skillFileUpload.ts new file mode 100644 index 00000000..f633a0fd --- /dev/null +++ b/mateclaw-ui/src/utils/skillFileUpload.ts @@ -0,0 +1,20 @@ +export type SkillFileBucket = 'references' | 'templates' | 'scripts' + +/** Keep the selected folder and all descendants under the chosen skill bucket. */ +export function planSkillFileUploads(files: File[], bucket: SkillFileBucket) { + if (files.length > 100 || files.reduce((n, file) => n + file.size, 0) > 50 * 1024 * 1024) { + throw new Error('batchLimit') + } + const paths = new Set() + return files.map(file => { + if (file.size > 10 * 1024 * 1024) throw new Error('fileLimit') + const relative = file.webkitRelativePath || file.name + const path = `${bucket}/${relative}` + if (path.length > 512 || relative.includes('..') || /[\\:\x00-\x1f\x7f]/.test(relative) + || relative.split('/').some(part => !part || part === '.') || paths.has(path)) { + throw new Error('invalidPath') + } + paths.add(path) + return { file, path } + }) +} diff --git a/mateclaw-ui/src/views/SkillMarket.vue b/mateclaw-ui/src/views/SkillMarket.vue index e916ef0d..b70255a2 100644 --- a/mateclaw-ui/src/views/SkillMarket.vue +++ b/mateclaw-ui/src/views/SkillMarket.vue @@ -481,6 +481,7 @@ v-if="canEditFiles && !creatingFile" class="detail-edit-btn detail-edit-primary" @click="startCreateFile" + :disabled="uploadingFiles" > + {{ t('skills.detail.fileNew') }} @@ -488,6 +489,24 @@

{{ t('skills.detail.filesHint') }}

{{ t('skills.detail.builtinFilesReadonly') }}

+
+ + + +

{{ t('skills.detail.uploadHint') }}

+
+

{{ uploadProgress || t('common.loading') }}

+
{{ activeFile.path }}
@@ -554,6 +574,7 @@

{{ t('common.loading') }}

+

{{ t('skills.detail.binaryFileHint') }}

{{ activeFileContent }}