mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-11-02 14:11:49 +08:00
fix(skills): support document and folder uploads (#609)
This commit is contained in:
parent
5c67af85fc
commit
f8a3d0d153
@ -5,6 +5,13 @@ import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.web.bind.annotation.*;
|
||||
import org.springframework.web.multipart.MultipartFile;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ContentDisposition;
|
||||
import org.springframework.web.server.ResponseStatusException;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import vip.mate.common.result.R;
|
||||
import vip.mate.agent.AgentService;
|
||||
import vip.mate.agent.binding.model.AgentSkillBinding;
|
||||
@ -42,6 +49,8 @@ import vip.mate.skill.lifecycle.SkillCuratorReportStore;
|
||||
import vip.mate.skill.lifecycle.SkillLifecycleService;
|
||||
import vip.mate.skill.lifecycle.model.SkillSnapshotEntity;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.LocalDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.LinkedHashMap;
|
||||
@ -379,6 +388,7 @@ public class SkillController {
|
||||
Map<String, Object> item = new LinkedHashMap<>();
|
||||
item.put("path", row.getFilePath());
|
||||
item.put("size", row.getContentSize());
|
||||
item.put("binary", row.isBinary());
|
||||
item.put("sha256", row.getSha256());
|
||||
item.put("updateTime", row.getUpdateTime());
|
||||
out.add(item);
|
||||
@ -404,7 +414,8 @@ public class SkillController {
|
||||
}
|
||||
Map<String, Object> body = new LinkedHashMap<>();
|
||||
body.put("path", row.getFilePath());
|
||||
body.put("content", row.getContent() == null ? "" : row.getContent());
|
||||
body.put("binary", row.isBinary());
|
||||
body.put("content", row.isBinary() ? "" : row.getContent() == null ? "" : row.getContent());
|
||||
body.put("size", row.getContentSize());
|
||||
body.put("sha256", row.getSha256());
|
||||
body.put("updateTime", row.getUpdateTime());
|
||||
@ -429,6 +440,10 @@ public class SkillController {
|
||||
if (normalized == null) {
|
||||
return R.fail("Invalid file path — must be under scripts/, references/ or templates/, no '..'.");
|
||||
}
|
||||
SkillFileEntity existing = skillFileService.getFile(id, normalized);
|
||||
if (existing != null && existing.isBinary()) {
|
||||
return R.fail("Binary files cannot be edited as text; upload a replacement instead.");
|
||||
}
|
||||
String content = body.get("content");
|
||||
if (content == null) {
|
||||
return R.fail("content is required (use the delete endpoint to remove a file).");
|
||||
@ -479,6 +494,58 @@ public class SkillController {
|
||||
return R.ok(Map.of("path", normalized, "removed", removed));
|
||||
}
|
||||
|
||||
private static final int MAX_UPLOAD_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
@PostMapping(value = "/{id}/files/upload", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
|
||||
@RequireWorkspaceRole("admin")
|
||||
public R<Map<String, Object>> uploadBundleFile(@PathVariable Long id,
|
||||
@RequestPart("file") MultipartFile file, @RequestParam String path,
|
||||
@RequestParam(defaultValue = "false") boolean overwrite,
|
||||
@RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) throws IOException {
|
||||
rejectVirtualSkillMutation(id);
|
||||
SkillEntity skill = skillService.getSkill(id);
|
||||
if (skill == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Skill not found");
|
||||
verifyResourceWorkspace(skill, workspaceId);
|
||||
if (Boolean.TRUE.equals(skill.getBuiltin())) return R.fail("Builtin skill files are read-only.");
|
||||
String normalized = normalizeBundlePath(path);
|
||||
if (normalized == null) return R.fail("Invalid file path: " + path);
|
||||
if (file.getSize() > MAX_UPLOAD_BYTES) return R.fail("File exceeds the 10 MiB limit.");
|
||||
if (!overwrite && skillFileService.getFile(id, normalized) != null) {
|
||||
return R.fail("File already exists; confirm replacement before uploading: " + normalized);
|
||||
}
|
||||
byte[] bytes;
|
||||
try (var input = file.getInputStream()) {
|
||||
bytes = input.readNBytes(MAX_UPLOAD_BYTES + 1);
|
||||
}
|
||||
if (bytes.length > MAX_UPLOAD_BYTES) return R.fail("File exceeds the 10 MiB limit.");
|
||||
SkillFileEntity row = skillFileService.upsertBytes(id, normalized, bytes);
|
||||
if (!skillFileSyncer.syncFile(skill, row)) {
|
||||
return R.fail("File saved, but workspace synchronization failed: " + normalized
|
||||
+ ". Check directory permissions or conflicting paths, then upload again.");
|
||||
}
|
||||
skillRuntimeService.rescanSingle(skill);
|
||||
return R.ok(Map.of("path", normalized, "size", row.getContentSize(), "binary", row.isBinary()));
|
||||
}
|
||||
|
||||
@GetMapping("/{id}/files/download")
|
||||
@RequireWorkspaceRole("member")
|
||||
public ResponseEntity<byte[]> downloadBundleFile(@PathVariable Long id, @RequestParam String path,
|
||||
@RequestHeader(value = "X-Workspace-Id", required = false) Long workspaceId) {
|
||||
SkillEntity skill = skillService.getSkill(id);
|
||||
if (skill == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "Skill not found");
|
||||
verifyResourceWorkspace(skill, workspaceId);
|
||||
String normalized = normalizeBundlePath(path);
|
||||
if (normalized == null) throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Invalid file path");
|
||||
SkillFileEntity row = skillFileService.getFile(id, normalized);
|
||||
if (row == null) throw new ResponseStatusException(HttpStatus.NOT_FOUND, "File not found");
|
||||
return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM)
|
||||
.header(HttpHeaders.CONTENT_DISPOSITION, ContentDisposition.attachment()
|
||||
.filename(normalized.substring(normalized.lastIndexOf('/') + 1), StandardCharsets.UTF_8)
|
||||
.build().toString())
|
||||
.header("X-Content-Type-Options", "nosniff")
|
||||
.body(row.contentBytes());
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize a bundle-relative path and enforce the same envelope the
|
||||
* store and workspace cache use: forward slashes, must sit under a
|
||||
@ -491,7 +558,9 @@ public class SkillController {
|
||||
static String normalizeBundlePath(String path) {
|
||||
if (path == null || path.isBlank()) return null;
|
||||
String p = path.strip().replace('\\', '/');
|
||||
if (p.startsWith("/") || p.contains("..") || p.contains("//") || p.endsWith("/")) return null;
|
||||
if (p.length() > 512 || p.chars().anyMatch(c -> c < 32 || c == 127)
|
||||
|| p.contains(":") || p.contains("/./") || p.endsWith("/.")
|
||||
|| p.startsWith("/") || p.contains("..") || p.contains("//") || p.endsWith("/")) return null;
|
||||
if (!SkillBundleFiles.isDbEligible(p)) return null;
|
||||
int slash = p.indexOf('/');
|
||||
if (slash == p.length() - 1) return null;
|
||||
|
||||
@ -32,17 +32,30 @@ public class SkillFileEntity {
|
||||
|
||||
/**
|
||||
* Path relative to the skill workspace root, always starting with
|
||||
* {@code scripts/} or {@code references/}. Forward slashes only.
|
||||
* {@code scripts/}, {@code references/} or {@code templates/}. Forward slashes only.
|
||||
*/
|
||||
private String filePath;
|
||||
|
||||
/** UTF-8 text content. Per-file size bounded by ZipSkillFetcher (1MB). */
|
||||
/** UTF-8 text or base64-encoded attachment bytes, according to contentEncoding. */
|
||||
private String content;
|
||||
|
||||
/** Length of {@link #content} in bytes — kept so listings can sort/audit without loading the blob. */
|
||||
/** null on legacy rows means UTF-8. */
|
||||
private String contentEncoding;
|
||||
|
||||
public boolean isBinary() {
|
||||
return "base64".equals(contentEncoding);
|
||||
}
|
||||
|
||||
public byte[] contentBytes() {
|
||||
String value = content == null ? "" : content;
|
||||
return isBinary() ? java.util.Base64.getDecoder().decode(value)
|
||||
: value.getBytes(java.nio.charset.StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/** Length of the original file in bytes — kept so listings can sort/audit without loading the blob. */
|
||||
private Integer contentSize;
|
||||
|
||||
/** SHA-256 of {@link #content}; used by the syncer to skip no-op writes. */
|
||||
/** SHA-256 of the original bytes; used by the syncer to skip no-op writes. */
|
||||
private String sha256;
|
||||
|
||||
@TableField(fill = FieldFill.INSERT)
|
||||
|
||||
@ -46,9 +46,13 @@ public class SkillFileService {
|
||||
/** Compute SHA-256 hex of a UTF-8 string (used for idempotent diffs). */
|
||||
public static String sha256Hex(String content) {
|
||||
if (content == null) content = "";
|
||||
return sha256Hex(content.getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
public static String sha256Hex(byte[] bytes) {
|
||||
try {
|
||||
MessageDigest md = MessageDigest.getInstance("SHA-256");
|
||||
byte[] digest = md.digest(content.getBytes(StandardCharsets.UTF_8));
|
||||
byte[] digest = md.digest(bytes);
|
||||
StringBuilder sb = new StringBuilder(digest.length * 2);
|
||||
for (byte b : digest) sb.append(String.format("%02x", b));
|
||||
return sb.toString();
|
||||
@ -132,6 +136,7 @@ public class SkillFileService {
|
||||
row.setSkillId(skillId);
|
||||
row.setFilePath(path);
|
||||
row.setContent(content);
|
||||
row.setContentEncoding("utf8");
|
||||
row.setContentSize(size);
|
||||
row.setSha256(hash);
|
||||
row.setCreateTime(now);
|
||||
@ -140,6 +145,7 @@ public class SkillFileService {
|
||||
written++;
|
||||
} else if (!hash.equals(prior.getSha256())) {
|
||||
prior.setContent(content);
|
||||
prior.setContentEncoding("utf8");
|
||||
prior.setContentSize(size);
|
||||
prior.setSha256(hash);
|
||||
prior.setUpdateTime(now);
|
||||
@ -193,32 +199,40 @@ public class SkillFileService {
|
||||
*/
|
||||
@Transactional
|
||||
public SkillFileEntity upsertFile(Long skillId, String filePath, String content) {
|
||||
String safeContent = content == null ? "" : content;
|
||||
String hash = sha256Hex(safeContent);
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
return upsertBytes(skillId, filePath,
|
||||
(content == null ? "" : content).getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
|
||||
SkillFileEntity existing = getFile(skillId, filePath);
|
||||
if (existing != null) {
|
||||
if (hash.equals(existing.getSha256())) {
|
||||
return existing;
|
||||
}
|
||||
existing.setContent(safeContent);
|
||||
existing.setContentSize(safeContent.getBytes(StandardCharsets.UTF_8).length);
|
||||
existing.setSha256(hash);
|
||||
existing.setUpdateTime(now);
|
||||
mapper.updateById(existing);
|
||||
return existing;
|
||||
/** Preserve arbitrary bytes; only strictly valid UTF-8 without control bytes is editable text. */
|
||||
@Transactional
|
||||
public SkillFileEntity upsertBytes(Long skillId, String filePath, byte[] bytes) {
|
||||
String text = null;
|
||||
try {
|
||||
text = StandardCharsets.UTF_8.newDecoder().decode(java.nio.ByteBuffer.wrap(bytes)).toString();
|
||||
if (text.codePoints().anyMatch(c -> c < 32 && c != '\n' && c != '\r' && c != '\t')) text = null;
|
||||
} catch (java.nio.charset.CharacterCodingException binary) {
|
||||
// Keep the original bytes below.
|
||||
}
|
||||
|
||||
SkillFileEntity row = new SkillFileEntity();
|
||||
row.setSkillId(skillId);
|
||||
row.setFilePath(filePath);
|
||||
row.setContent(safeContent);
|
||||
row.setContentSize(safeContent.getBytes(StandardCharsets.UTF_8).length);
|
||||
String encoding = text == null ? "base64" : "utf8";
|
||||
String content = text == null ? java.util.Base64.getEncoder().encodeToString(bytes) : text;
|
||||
String hash = sha256Hex(bytes);
|
||||
SkillFileEntity row = getFile(skillId, filePath);
|
||||
boolean insert = row == null;
|
||||
if (!insert && hash.equals(row.getSha256())) return row;
|
||||
LocalDateTime now = LocalDateTime.now();
|
||||
if (insert) {
|
||||
row = new SkillFileEntity();
|
||||
row.setSkillId(skillId);
|
||||
row.setFilePath(filePath);
|
||||
row.setCreateTime(now);
|
||||
}
|
||||
row.setContent(content);
|
||||
row.setContentEncoding(encoding);
|
||||
row.setContentSize(bytes.length);
|
||||
row.setSha256(hash);
|
||||
row.setCreateTime(now);
|
||||
row.setUpdateTime(now);
|
||||
mapper.insert(row);
|
||||
if (insert) mapper.insert(row);
|
||||
else mapper.updateById(row);
|
||||
return row;
|
||||
}
|
||||
|
||||
|
||||
@ -161,6 +161,12 @@ public class SkillFileSyncer {
|
||||
return ingested.size();
|
||||
}
|
||||
|
||||
/** Materialize only the uploaded row; folder uploads must not re-read the full bundle per file. */
|
||||
public boolean syncFile(SkillEntity skill, SkillFileEntity row) {
|
||||
Path workspace = workspaceManager.resolveConventionPath(skill.getName(), skill.getWorkspaceId());
|
||||
return materializeOne(workspace, row) != MaterializeOutcome.SKIPPED;
|
||||
}
|
||||
|
||||
private enum MaterializeOutcome { WROTE, CURRENT, SKIPPED }
|
||||
|
||||
private MaterializeOutcome materializeOne(Path workspaceDir, SkillFileEntity row) {
|
||||
@ -183,17 +189,21 @@ public class SkillFileSyncer {
|
||||
}
|
||||
|
||||
try {
|
||||
String content = row.getContent() == null ? "" : row.getContent();
|
||||
// Do not follow links created by a skill script outside its workspace.
|
||||
for (Path part = target; part != null && part.startsWith(workspaceDir); part = part.getParent()) {
|
||||
if (Files.isSymbolicLink(part)) return MaterializeOutcome.SKIPPED;
|
||||
}
|
||||
byte[] content = row.contentBytes();
|
||||
if (Files.exists(target)) {
|
||||
String onDisk = Files.readString(target, StandardCharsets.UTF_8);
|
||||
byte[] onDisk = Files.readAllBytes(target);
|
||||
if (SkillFileService.sha256Hex(onDisk).equals(row.getSha256())) {
|
||||
return MaterializeOutcome.CURRENT;
|
||||
}
|
||||
}
|
||||
Files.createDirectories(target.getParent());
|
||||
Files.writeString(target, content, StandardCharsets.UTF_8);
|
||||
Files.write(target, content);
|
||||
return MaterializeOutcome.WROTE;
|
||||
} catch (IOException e) {
|
||||
} catch (IOException | IllegalArgumentException e) {
|
||||
log.warn("Failed to materialize skill_file {} → {}: {}", row.getId(), target, e.getMessage());
|
||||
return MaterializeOutcome.SKIPPED;
|
||||
}
|
||||
|
||||
@ -0,0 +1,2 @@
|
||||
-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column.
|
||||
ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL;
|
||||
@ -0,0 +1,2 @@
|
||||
-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column.
|
||||
ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL;
|
||||
@ -0,0 +1,2 @@
|
||||
-- Preserve existing UTF-8 rows; binary attachments use base64 in the canonical content column.
|
||||
ALTER TABLE mate_skill_file ADD COLUMN content_encoding VARCHAR(16) DEFAULT 'utf8' NOT NULL;
|
||||
@ -188,4 +188,63 @@ class SkillControllerBundleFilesTest {
|
||||
assertThat(resp.getMsg()).contains("read-only");
|
||||
verify(fileService, never()).deleteFile(any(), any());
|
||||
}
|
||||
@Test
|
||||
void uploadsAndDownloadsOriginalDocumentBytes() throws Exception {
|
||||
when(skillService.getSkill(SID)).thenReturn(skill(false));
|
||||
byte[] bytes = {80, 75, 3, 4, 0, (byte) 255};
|
||||
var file = new org.springframework.mock.web.MockMultipartFile("file", "报告.docx",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document", bytes);
|
||||
var row = row("references/资料/报告.docx", java.util.Base64.getEncoder().encodeToString(bytes));
|
||||
row.setContentEncoding("base64");
|
||||
row.setContentSize(bytes.length);
|
||||
when(fileService.upsertBytes(SID, row.getFilePath(), bytes)).thenReturn(row);
|
||||
when(fileSyncer.syncFile(any(SkillEntity.class), any(SkillFileEntity.class))).thenReturn(true);
|
||||
var result = controller.uploadBundleFile(SID, file, row.getFilePath(), false, null);
|
||||
assertThat(result.getData()).containsEntry("binary", true).containsEntry("size", bytes.length);
|
||||
verify(fileSyncer).syncFile(any(SkillEntity.class), any(SkillFileEntity.class));
|
||||
when(fileService.getFile(SID, row.getFilePath())).thenReturn(row);
|
||||
assertThat(controller.downloadBundleFile(SID, row.getFilePath(), null).getBody()).isEqualTo(bytes);
|
||||
assertThat(controller.getBundleFileContent(SID, row.getFilePath(), null).getData())
|
||||
.containsEntry("binary", true).containsEntry("content", "");
|
||||
assertThat(controller.putBundleFileContent(SID,
|
||||
Map.of("path", row.getFilePath(), "content", "corrupt"), null).getMsg()).contains("Binary");
|
||||
}
|
||||
|
||||
@Test
|
||||
void uploadRejectsUnsafePathsReadonlyAndUnconfirmedReplacement() throws Exception {
|
||||
var file = new org.springframework.mock.web.MockMultipartFile("file", "a.txt", "text/plain", new byte[]{1});
|
||||
when(skillService.getSkill(SID)).thenReturn(skill(false));
|
||||
for (String path : List.of("references/../bad", "references/./bad", "references/a\u0000b", "/tmp/a")) {
|
||||
assertThat(controller.uploadBundleFile(SID, file, path, false, null).getMsg()).contains("Invalid");
|
||||
}
|
||||
when(fileService.getFile(SID, "references/a.txt")).thenReturn(row("references/a.txt", "old"));
|
||||
assertThat(controller.uploadBundleFile(SID, file, "references/a.txt", false, null).getMsg()).contains("already exists");
|
||||
when(skillService.getSkill(SID)).thenReturn(skill(true));
|
||||
assertThat(controller.uploadBundleFile(SID, file, "references/a.txt", true, null).getMsg()).contains("read-only");
|
||||
verify(fileService, never()).upsertBytes(any(), any(), any());
|
||||
}
|
||||
|
||||
@Test
|
||||
void uploadRejectsOversizedFilesBeforeReading() throws Exception {
|
||||
when(skillService.getSkill(SID)).thenReturn(skill(false));
|
||||
var file = mock(org.springframework.web.multipart.MultipartFile.class);
|
||||
when(file.getSize()).thenReturn(10L * 1024 * 1024 + 1);
|
||||
assertThat(controller.uploadBundleFile(SID, file, "references/a.docx", false, null).getMsg()).contains("10 MiB");
|
||||
verify(file, never()).getInputStream();
|
||||
}
|
||||
|
||||
@Test
|
||||
void attachmentEndpointsRejectOtherWorkspacesBeforeReadingBytes() {
|
||||
when(skillService.getSkill(SID)).thenReturn(skill(false));
|
||||
var file = mock(org.springframework.web.multipart.MultipartFile.class);
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(() ->
|
||||
controller.uploadBundleFile(SID, file, "references/a.docx", false, 2L))
|
||||
.isInstanceOf(vip.mate.exception.MateClawException.class);
|
||||
org.assertj.core.api.Assertions.assertThatThrownBy(() ->
|
||||
controller.downloadBundleFile(SID, "references/a.docx", 2L))
|
||||
.isInstanceOf(vip.mate.exception.MateClawException.class);
|
||||
verify(fileService, never()).getFile(any(), any());
|
||||
verify(fileService, never()).upsertBytes(any(), any(), any());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -0,0 +1,44 @@
|
||||
package vip.mate.skill.service;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
import java.sql.DriverManager;
|
||||
import java.util.Base64;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
|
||||
class SkillFileEncodingMigrationTest {
|
||||
@Test
|
||||
void migrationPreservesLegacyTextAndStoresBinaryContent() throws Exception {
|
||||
for (String dialect : List.of("h2", "mysql", "kingbase")) {
|
||||
String migration = Files.readString(Path.of("src/main/resources/db/migration", dialect,
|
||||
"V202__skill_file_encoding.sql"));
|
||||
// The added column uses portable SQL. Exercise it on populated tables,
|
||||
// including dialect modes; native MySQL/Kingbase still require deployment verification.
|
||||
String mode = dialect.equals("kingbase") ? "PostgreSQL" : "MySQL";
|
||||
try (var connection = DriverManager.getConnection("jdbc:h2:mem:encoding_" + dialect + ";MODE=" + mode);
|
||||
var statement = connection.createStatement()) {
|
||||
statement.execute("CREATE TABLE mate_skill_file (id BIGINT PRIMARY KEY, content TEXT)");
|
||||
statement.execute("INSERT INTO mate_skill_file VALUES (1, '制度说明')");
|
||||
statement.execute(migration);
|
||||
try (var result = statement.executeQuery("SELECT content, content_encoding FROM mate_skill_file WHERE id=1")) {
|
||||
assertTrue(result.next());
|
||||
assertEquals("制度说明", result.getString(1));
|
||||
assertEquals("utf8", result.getString(2));
|
||||
}
|
||||
byte[] bytes = {80, 75, 0, (byte) 255};
|
||||
try (var insert = connection.prepareStatement("INSERT INTO mate_skill_file VALUES (2, ?, 'base64')")) {
|
||||
insert.setString(1, Base64.getEncoder().encodeToString(bytes));
|
||||
insert.executeUpdate();
|
||||
}
|
||||
try (var result = statement.executeQuery("SELECT content FROM mate_skill_file WHERE id=2")) {
|
||||
assertTrue(result.next());
|
||||
assertArrayEquals(bytes, Base64.getDecoder().decode(result.getString(1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -114,4 +114,36 @@ class SkillFileServiceTest {
|
||||
e.setSha256(SkillFileService.sha256Hex(content));
|
||||
return e;
|
||||
}
|
||||
@Test
|
||||
void binaryUploadPreservesBytesSizeAndHash() {
|
||||
byte[] bytes = {80, 75, 3, 4, 0, (byte) 255, (byte) 128};
|
||||
SkillFileEntity row = service.upsertBytes(42L, "templates/报告.xlsx", bytes);
|
||||
assertTrue(row.isBinary());
|
||||
assertArrayEquals(bytes, row.contentBytes());
|
||||
assertEquals(bytes.length, row.getContentSize());
|
||||
assertEquals(SkillFileService.sha256Hex(bytes), row.getSha256());
|
||||
verify(mapper).insert(row);
|
||||
}
|
||||
|
||||
@Test
|
||||
void uploadedUtf8RemainsEditableAndLegacyRowsDecodeUnchanged() {
|
||||
String text = "制度说明\n中文";
|
||||
SkillFileEntity row = service.upsertBytes(42L, "references/制度.md",
|
||||
text.getBytes(java.nio.charset.StandardCharsets.UTF_8));
|
||||
assertFalse(row.isBinary());
|
||||
assertEquals(text, row.getContent());
|
||||
row.setContentEncoding(null);
|
||||
assertArrayEquals(text.getBytes(java.nio.charset.StandardCharsets.UTF_8), row.contentBytes());
|
||||
}
|
||||
|
||||
@Test
|
||||
void replacingBinaryWithTextClearsEncoding() {
|
||||
SkillFileEntity prior = newRow(1L, "references/file", "AA==");
|
||||
prior.setContentEncoding("base64");
|
||||
when(mapper.selectOne(any())).thenReturn(prior);
|
||||
SkillFileEntity row = service.upsertBytes(42L, "references/file", "hello".getBytes());
|
||||
assertFalse(row.isBinary());
|
||||
assertArrayEquals("hello".getBytes(), row.contentBytes());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -167,4 +167,35 @@ class SkillFileSyncerTest {
|
||||
e.setSha256(SkillFileService.sha256Hex(content));
|
||||
return e;
|
||||
}
|
||||
@Test
|
||||
void binaryAttachmentSurvivesRestoreAndStaleCacheReplacement() throws IOException {
|
||||
SkillEntity skill = newSkill(10L, "demo");
|
||||
byte[] bytes = {80, 75, 3, 4, 0, (byte) 255};
|
||||
SkillFileEntity row = newRow(1L, 10L, "templates/报告.xlsx", "");
|
||||
row.setContent(java.util.Base64.getEncoder().encodeToString(bytes));
|
||||
row.setContentEncoding("base64");
|
||||
row.setSha256(SkillFileService.sha256Hex(bytes));
|
||||
when(mapper.selectList(any())).thenReturn(List.of(row));
|
||||
Path target = tmp.resolve("1/demo/templates/报告.xlsx");
|
||||
assertEquals(1, syncer.syncOne(skill).filesMaterialized());
|
||||
assertArrayEquals(bytes, Files.readAllBytes(target));
|
||||
assertEquals(1, syncer.syncOne(skill).filesAlreadyCurrent());
|
||||
Files.write(target, new byte[]{(byte) 255, 0, 1});
|
||||
assertEquals(1, syncer.syncOne(skill).filesMaterialized());
|
||||
assertArrayEquals(bytes, Files.readAllBytes(target));
|
||||
}
|
||||
|
||||
@Test
|
||||
void refusesSymlinkInsideSkillWorkspace() throws IOException {
|
||||
SkillEntity skill = newSkill(10L, "demo");
|
||||
Path workspace = tmp.resolve("1/demo");
|
||||
Path outside = tmp.resolve("outside");
|
||||
Files.createDirectories(workspace);
|
||||
Files.createDirectories(outside);
|
||||
Files.createSymbolicLink(workspace.resolve("references"), outside);
|
||||
when(mapper.selectList(any())).thenReturn(List.of(newRow(1L, 10L, "references/a.txt", "secret")));
|
||||
assertEquals(0, syncer.syncOne(skill).filesMaterialized());
|
||||
assertFalse(Files.exists(outside.resolve("a.txt")));
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
29
mateclaw-ui/src/api/__tests__/skillFiles.test.ts
Normal file
29
mateclaw-ui/src/api/__tests__/skillFiles.test.ts
Normal file
@ -0,0 +1,29 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { http, skillApi } from '@/api/index'
|
||||
|
||||
describe('skill attachment API', () => {
|
||||
afterEach(() => vi.restoreAllMocks())
|
||||
|
||||
it('sends the original file and relative path without coercing large skill IDs', async () => {
|
||||
const post = vi.spyOn(http, 'post').mockResolvedValue({} as never)
|
||||
const file = new File([new Uint8Array([80, 75, 0, 255])], '模板.xlsx')
|
||||
await skillApi.uploadFile('9007199254740993', file, 'templates/部门/模板.xlsx')
|
||||
const [url, body] = post.mock.calls[0]
|
||||
expect(url).toBe('/skills/9007199254740993/files/upload')
|
||||
const form = body as FormData
|
||||
expect(form.get('path')).toBe('templates/部门/模板.xlsx')
|
||||
expect(form.get('overwrite')).toBe('false')
|
||||
expect(await (form.get('file') as File).arrayBuffer()).toEqual(await file.arrayBuffer())
|
||||
await skillApi.uploadFile('9007199254740993', file, 'templates/部门/模板.xlsx', true)
|
||||
expect((post.mock.calls[1][1] as FormData).get('overwrite')).toBe('true')
|
||||
})
|
||||
|
||||
it('downloads authenticated bytes through the shared HTTP client', async () => {
|
||||
const blob = new Blob([new Uint8Array([0, 255])])
|
||||
const get = vi.spyOn(http, 'get').mockResolvedValue(blob as never)
|
||||
expect(await skillApi.downloadFile('9007199254740993', 'references/a.docx')).toBe(blob)
|
||||
expect(get).toHaveBeenCalledWith('/skills/9007199254740993/files/download', {
|
||||
params: { path: 'references/a.docx' }, responseType: 'blob',
|
||||
})
|
||||
})
|
||||
})
|
||||
@ -275,6 +275,15 @@ export const skillApi = {
|
||||
* re-resolve the skill.
|
||||
*/
|
||||
listFiles: (id: string | number) => http.get(`/skills/${id}/files`),
|
||||
uploadFile: (id: string | number, file: File, path: string, overwrite = false) => {
|
||||
const form = new FormData()
|
||||
form.append('file', file)
|
||||
form.append('path', path)
|
||||
form.append('overwrite', String(overwrite))
|
||||
return http.post(`/skills/${id}/files/upload`, form, { timeout: 120000 })
|
||||
},
|
||||
downloadFile: (id: string | number, path: string): Promise<Blob> =>
|
||||
http.get(`/skills/${id}/files/download`, { params: { path }, responseType: 'blob' }) as unknown as Promise<Blob>,
|
||||
getFileContent: (id: string | number, path: string) =>
|
||||
http.get(`/skills/${id}/files/content`, { params: { path } }),
|
||||
saveFileContent: (id: string | number, path: string, content: string) =>
|
||||
|
||||
@ -4305,6 +4305,20 @@ export default {
|
||||
fileEdit: 'Edit',
|
||||
fileDelete: 'Delete',
|
||||
fileDeleteConfirm: 'Delete "{path}"? This removes both the database record and the workspace file. Cannot be undone.',
|
||||
uploadFiles: 'Upload files',
|
||||
uploadFolder: 'Upload folder',
|
||||
uploadDestination: 'Destination',
|
||||
uploadHint: 'Supports Word, Excel and other files. Folder structure is preserved. Up to 10 MiB per file, 100 files and 50 MiB per batch. Empty folders are not uploaded.',
|
||||
uploadOverwrite: 'Replace these existing files? {paths}',
|
||||
uploadComplete: 'Uploaded {count} files',
|
||||
uploadFailed: 'Uploaded {count} files; remaining uploads did not finish: {error}',
|
||||
uploadErrors: {
|
||||
batchLimit: 'Each batch is limited to 100 files and 50 MiB.',
|
||||
fileLimit: 'Each file must be at most 10 MiB.',
|
||||
invalidPath: 'File paths are invalid, duplicated or too long.',
|
||||
},
|
||||
fileDownload: 'Download',
|
||||
binaryFileHint: 'This file is preserved in its original format. Download to view it, or upload a replacement to change it.',
|
||||
fileSaved: 'File saved',
|
||||
fileDeleted: 'File deleted',
|
||||
fileLoadFailed: 'Failed to read file',
|
||||
|
||||
@ -4397,6 +4397,20 @@ export default {
|
||||
fileEdit: '编辑',
|
||||
fileDelete: '删除',
|
||||
fileDeleteConfirm: '删除文件 "{path}"?此操作同时移除数据库记录和工作区文件,不可撤销。',
|
||||
uploadFiles: '上传文件',
|
||||
uploadFolder: '上传文件夹',
|
||||
uploadDestination: '保存目录',
|
||||
uploadHint: '支持 Word、Excel 等文件,文件夹会保留目录层级。单个文件不超过 10 MiB,每批最多 100 个文件、合计 50 MiB;空文件夹不会上传。',
|
||||
uploadOverwrite: '以下同名文件将被替换,是否继续?{paths}',
|
||||
uploadComplete: '已上传 {count} 个文件',
|
||||
uploadFailed: '已上传 {count} 个文件,其余未完成:{error}',
|
||||
uploadErrors: {
|
||||
batchLimit: '每批最多上传 100 个文件,合计不超过 50 MiB。',
|
||||
fileLimit: '单个文件不能超过 10 MiB。',
|
||||
invalidPath: '文件路径无效、重复或过长,请检查文件名和目录。',
|
||||
},
|
||||
fileDownload: '下载',
|
||||
binaryFileHint: '此文件保留原始格式,可下载查看;如需修改,请上传替换文件。',
|
||||
fileSaved: '文件已保存',
|
||||
fileDeleted: '文件已删除',
|
||||
fileLoadFailed: '读取文件失败',
|
||||
|
||||
30
mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts
Normal file
30
mateclaw-ui/src/utils/__tests__/skillFileUpload.test.ts
Normal file
@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { planSkillFileUploads } from '../skillFileUpload'
|
||||
|
||||
function file(name: string, relative = '', size = 1): File {
|
||||
const result = new File(['x'], name)
|
||||
Object.defineProperty(result, 'webkitRelativePath', { value: relative })
|
||||
Object.defineProperty(result, 'size', { value: size })
|
||||
return result
|
||||
}
|
||||
|
||||
describe('skill file uploads', () => {
|
||||
it('keeps Unicode folder structure and original files under the chosen bucket', () => {
|
||||
const document = file('报告.docx', '制度/部门/报告.docx')
|
||||
expect(planSkillFileUploads([document], 'references')).toEqual([
|
||||
{ file: document, path: 'references/制度/部门/报告.docx' },
|
||||
])
|
||||
expect(planSkillFileUploads([file('report.xlsx')], 'templates')[0].path).toBe('templates/report.xlsx')
|
||||
})
|
||||
it('rejects traversal, absolute paths, duplicate paths and control characters', () => {
|
||||
for (const path of ['../a', '/tmp/a', 'x/./a', 'x//a', 'x\\a', 'x\0a', 'C:/a']) {
|
||||
expect(() => planSkillFileUploads([file('a', path)], 'references')).toThrow('invalidPath')
|
||||
}
|
||||
expect(() => planSkillFileUploads([file('a'), file('a')], 'references')).toThrow('invalidPath')
|
||||
})
|
||||
it('bounds individual files and batches before sending any requests', () => {
|
||||
expect(() => planSkillFileUploads([file('a', '', 10 * 1024 * 1024 + 1)], 'references')).toThrow('fileLimit')
|
||||
expect(() => planSkillFileUploads(Array.from({ length: 101 }, (_, i) => file(String(i))), 'references')).toThrow('batchLimit')
|
||||
expect(() => planSkillFileUploads(Array.from({ length: 6 }, (_, i) => file(String(i), '', 10 * 1024 * 1024)), 'references')).toThrow('batchLimit')
|
||||
})
|
||||
})
|
||||
20
mateclaw-ui/src/utils/skillFileUpload.ts
Normal file
20
mateclaw-ui/src/utils/skillFileUpload.ts
Normal file
@ -0,0 +1,20 @@
|
||||
export type SkillFileBucket = 'references' | 'templates' | 'scripts'
|
||||
|
||||
/** Keep the selected folder and all descendants under the chosen skill bucket. */
|
||||
export function planSkillFileUploads(files: File[], bucket: SkillFileBucket) {
|
||||
if (files.length > 100 || files.reduce((n, file) => n + file.size, 0) > 50 * 1024 * 1024) {
|
||||
throw new Error('batchLimit')
|
||||
}
|
||||
const paths = new Set<string>()
|
||||
return files.map(file => {
|
||||
if (file.size > 10 * 1024 * 1024) throw new Error('fileLimit')
|
||||
const relative = file.webkitRelativePath || file.name
|
||||
const path = `${bucket}/${relative}`
|
||||
if (path.length > 512 || relative.includes('..') || /[\\:\x00-\x1f\x7f]/.test(relative)
|
||||
|| relative.split('/').some(part => !part || part === '.') || paths.has(path)) {
|
||||
throw new Error('invalidPath')
|
||||
}
|
||||
paths.add(path)
|
||||
return { file, path }
|
||||
})
|
||||
}
|
||||
@ -481,6 +481,7 @@
|
||||
v-if="canEditFiles && !creatingFile"
|
||||
class="detail-edit-btn detail-edit-primary"
|
||||
@click="startCreateFile"
|
||||
:disabled="uploadingFiles"
|
||||
>
|
||||
+ {{ t('skills.detail.fileNew') }}
|
||||
</button>
|
||||
@ -488,6 +489,24 @@
|
||||
<p class="detail-hint">{{ t('skills.detail.filesHint') }}</p>
|
||||
<p v-if="isBuiltinDetail" class="detail-hint">{{ t('skills.detail.builtinFilesReadonly') }}</p>
|
||||
|
||||
<fieldset v-if="canEditFiles" class="skill-file-upload" :disabled="uploadingFiles || savingFile || editingFile || creatingFile">
|
||||
<label>{{ t('skills.detail.uploadDestination') }}
|
||||
<select v-model="uploadBucket" :aria-label="t('skills.detail.uploadDestination')">
|
||||
<option value="references">references/</option>
|
||||
<option value="templates">templates/</option>
|
||||
<option value="scripts">scripts/</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>{{ t('skills.detail.uploadFiles') }}
|
||||
<input type="file" multiple :aria-label="t('skills.detail.uploadFiles')" @change="uploadSkillFiles" />
|
||||
</label>
|
||||
<label>{{ t('skills.detail.uploadFolder') }}
|
||||
<input type="file" multiple webkitdirectory :aria-label="t('skills.detail.uploadFolder')" @change="uploadSkillFiles" />
|
||||
</label>
|
||||
<p class="detail-hint">{{ t('skills.detail.uploadHint') }}</p>
|
||||
</fieldset>
|
||||
<p v-if="uploadingFiles" role="status">{{ uploadProgress || t('common.loading') }}</p>
|
||||
|
||||
<!-- New-file form -->
|
||||
<div v-if="creatingFile" class="skill-file-create">
|
||||
<input
|
||||
@ -536,10 +555,11 @@
|
||||
<h4 class="detail-block-title"><code>{{ activeFile.path }}</code></h4>
|
||||
<div class="edit-actions">
|
||||
<template v-if="!editingFile">
|
||||
<button v-if="canEditFiles" class="detail-edit-btn detail-edit-primary" @click="startEditFile">
|
||||
<button class="detail-edit-btn" @click="downloadActiveFile">{{ t('skills.detail.fileDownload') }}</button>
|
||||
<button v-if="canEditFiles && !activeFile.binary" :disabled="uploadingFiles" class="detail-edit-btn detail-edit-primary" @click="startEditFile">
|
||||
{{ t('skills.detail.fileEdit') }}
|
||||
</button>
|
||||
<button v-if="canEditFiles" class="detail-edit-btn detail-edit-cancel" @click="removeActiveFile" :disabled="savingFile">
|
||||
<button v-if="canEditFiles" class="detail-edit-btn detail-edit-cancel" @click="removeActiveFile" :disabled="savingFile || uploadingFiles">
|
||||
{{ t('skills.detail.fileDelete') }}
|
||||
</button>
|
||||
</template>
|
||||
@ -554,6 +574,7 @@
|
||||
</div>
|
||||
</div>
|
||||
<p v-if="fileContentLoading" class="detail-empty">{{ t('common.loading') }}</p>
|
||||
<p v-else-if="activeFile.binary" class="detail-hint">{{ t('skills.detail.binaryFileHint') }}</p>
|
||||
<pre v-else-if="!editingFile" class="detail-pre">{{ activeFileContent }}</pre>
|
||||
<textarea
|
||||
v-else
|
||||
@ -798,6 +819,7 @@
|
||||
</template>
|
||||
|
||||
<script setup lang="ts">
|
||||
import { planSkillFileUploads, type SkillFileBucket } from '@/utils/skillFileUpload'
|
||||
import { ref, reactive, computed, onMounted, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { mcToast } from '@/composables/useMcToast'
|
||||
@ -911,11 +933,79 @@ const editBodyForm = ref<{ skillContent: string; sourceCode: string }>({
|
||||
|
||||
interface SkillBundleFile {
|
||||
path: string
|
||||
binary?: boolean
|
||||
size?: number
|
||||
sha256?: string
|
||||
updateTime?: string
|
||||
}
|
||||
|
||||
const uploadBucket = ref<SkillFileBucket>('references')
|
||||
const uploadingFiles = ref(false)
|
||||
const uploadProgress = ref('')
|
||||
|
||||
async function uploadSkillFiles(event: Event) {
|
||||
const input = event.target as HTMLInputElement
|
||||
const files = Array.from(input.files ?? [])
|
||||
input.value = ''
|
||||
if (!files.length || !detailSkill.value || uploadingFiles.value) return
|
||||
const skillId = detailSkill.value.id
|
||||
let plans
|
||||
try {
|
||||
plans = planSkillFileUploads(files, uploadBucket.value)
|
||||
} catch (error) {
|
||||
mcToast.error(t(`skills.detail.uploadErrors.${(error as Error).message}`))
|
||||
return
|
||||
}
|
||||
uploadingFiles.value = true
|
||||
let uploaded = 0
|
||||
try {
|
||||
const response: any = await skillApi.listFiles(skillId)
|
||||
const existing = new Set<string>((response?.data ?? []).map((f: SkillBundleFile) => f.path))
|
||||
const conflicts = plans.filter(item => existing.has(item.path))
|
||||
if (conflicts.length && !await mcConfirm({
|
||||
title: t('skills.detail.uploadFiles'),
|
||||
message: t('skills.detail.uploadOverwrite', { paths: conflicts.map(item => item.path).join('、') }),
|
||||
tone: 'danger',
|
||||
})) return
|
||||
for (const item of plans) {
|
||||
uploadProgress.value = `${uploaded + 1}/${plans.length}: ${item.path}`
|
||||
await skillApi.uploadFile(skillId, item.file, item.path, existing.has(item.path))
|
||||
uploaded++
|
||||
}
|
||||
mcToast.success(t('skills.detail.uploadComplete', { count: uploaded }))
|
||||
} catch (error: any) {
|
||||
mcToast.error(t('skills.detail.uploadFailed', { count: uploaded, error: error?.message || String(error) }))
|
||||
} finally {
|
||||
uploadingFiles.value = false
|
||||
uploadProgress.value = ''
|
||||
if (detailSkill.value?.id === skillId) {
|
||||
activeFile.value = null
|
||||
editingFile.value = false
|
||||
await loadDetailFiles()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function downloadActiveFile() {
|
||||
if (!detailSkill.value || !activeFile.value) return
|
||||
const path = activeFile.value.path
|
||||
try {
|
||||
const blob = await skillApi.downloadFile(detailSkill.value.id, path)
|
||||
if (blob.type.includes('application/json')) {
|
||||
const error = JSON.parse(await blob.text())
|
||||
throw new Error(error.msg || t('skills.detail.fileLoadFailed'))
|
||||
}
|
||||
const url = URL.createObjectURL(blob)
|
||||
const link = document.createElement('a')
|
||||
link.href = url
|
||||
link.download = path.split('/').pop() || 'download'
|
||||
link.click()
|
||||
setTimeout(() => URL.revokeObjectURL(url), 1000)
|
||||
} catch (error: any) {
|
||||
mcToast.error(error?.message || t('skills.detail.fileLoadFailed'))
|
||||
}
|
||||
}
|
||||
|
||||
const detailFiles = ref<SkillBundleFile[]>([])
|
||||
const detailFilesLoading = ref(false)
|
||||
const activeFile = ref<SkillBundleFile | null>(null)
|
||||
@ -961,6 +1051,7 @@ async function openFile(file: SkillBundleFile) {
|
||||
try {
|
||||
const res: any = await skillApi.getFileContent(detailSkill.value.id, file.path)
|
||||
activeFileContent.value = res?.data?.content ?? ''
|
||||
file.binary = Boolean(res?.data?.binary)
|
||||
} catch (e: any) {
|
||||
activeFileContent.value = ''
|
||||
mcToast.error(typeof e === 'string' ? e : e?.message || t('skills.detail.fileLoadFailed'))
|
||||
@ -970,6 +1061,7 @@ async function openFile(file: SkillBundleFile) {
|
||||
}
|
||||
|
||||
function startEditFile() {
|
||||
if (activeFile.value?.binary) return
|
||||
fileEditContent.value = activeFileContent.value
|
||||
editingFile.value = true
|
||||
}
|
||||
@ -2012,6 +2104,11 @@ function getSkillTypeLabel(type: string) {
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.skill-file-upload { display: flex; flex-wrap: wrap; gap: 12px; border: 1px solid var(--mc-border, #ddd); border-radius: 8px; padding: 12px; margin: 12px 0; }
|
||||
.skill-file-upload label { display: flex; flex-direction: column; gap: 6px; min-width: 0; }
|
||||
.skill-file-upload input { max-width: 100%; }
|
||||
.skill-file-upload p { flex-basis: 100%; margin: 0; }
|
||||
|
||||
.skills-page { gap: 18px; }
|
||||
.header-actions { display: flex; gap: 8px; align-items: center; flex-wrap: wrap; }
|
||||
.btn-primary { display: flex; align-items: center; gap: 6px; padding: 10px 16px; background: linear-gradient(135deg, var(--mc-primary), var(--mc-primary-hover)); color: white; border: none; border-radius: 14px; font-size: 14px; font-weight: 600; cursor: pointer; transition: background 0.15s; box-shadow: var(--mc-shadow-soft); }
|
||||
|
||||
Loading…
Reference in New Issue
Block a user