mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-11-03 22:45:32 +08:00
fix(chat): resolve uploaded images from conversation storage
Resolve web image attachments by authenticated conversation origin and stored name through ChatUploadLocationResolver. Apply the same resolution to native and recent-image inputs and validate submitted uploads. Refs #641.
This commit is contained in:
parent
17c87dfa3a
commit
6f13a70709
@ -158,6 +158,12 @@ public class AgentGraphBuilder {
|
|||||||
private final vip.mate.llm.chatmodel.DashScopeChatModelBuilder dashScopeBuilder;
|
private final vip.mate.llm.chatmodel.DashScopeChatModelBuilder dashScopeBuilder;
|
||||||
private final vip.mate.llm.routing.MultimodalRouter multimodalRouter;
|
private final vip.mate.llm.routing.MultimodalRouter multimodalRouter;
|
||||||
private final vip.mate.llm.routing.MediaCaptionService mediaCaptionService;
|
private final vip.mate.llm.routing.MediaCaptionService mediaCaptionService;
|
||||||
|
private vip.mate.workspace.core.service.ChatUploadLocationResolver chatUploadLocationResolver;
|
||||||
|
|
||||||
|
@Autowired
|
||||||
|
public void setChatUploadLocationResolver(vip.mate.workspace.core.service.ChatUploadLocationResolver resolver) {
|
||||||
|
this.chatUploadLocationResolver = resolver;
|
||||||
|
}
|
||||||
private final vip.mate.goal.service.GoalService goalService;
|
private final vip.mate.goal.service.GoalService goalService;
|
||||||
private final vip.mate.goal.service.GoalEvaluationService goalEvaluationService;
|
private final vip.mate.goal.service.GoalEvaluationService goalEvaluationService;
|
||||||
private final vip.mate.goal.service.GoalFollowupService goalFollowupService;
|
private final vip.mate.goal.service.GoalFollowupService goalFollowupService;
|
||||||
@ -541,6 +547,7 @@ public class AgentGraphBuilder {
|
|||||||
agent.goalService = goalService;
|
agent.goalService = goalService;
|
||||||
agent.multimodalRouter = multimodalRouter;
|
agent.multimodalRouter = multimodalRouter;
|
||||||
agent.mediaCaptionService = mediaCaptionService;
|
agent.mediaCaptionService = mediaCaptionService;
|
||||||
|
agent.chatUploadLocationResolver = chatUploadLocationResolver;
|
||||||
agent.userLocale = resolveLocale();
|
agent.userLocale = resolveLocale();
|
||||||
agent.temperature = runtimeModel.getTemperature();
|
agent.temperature = runtimeModel.getTemperature();
|
||||||
agent.maxTokens = runtimeModel.getMaxTokens();
|
agent.maxTokens = runtimeModel.getMaxTokens();
|
||||||
|
|||||||
@ -29,6 +29,7 @@ import vip.mate.workspace.conversation.model.MessageContentPart;
|
|||||||
import vip.mate.workspace.conversation.model.MessageEntity;
|
import vip.mate.workspace.conversation.model.MessageEntity;
|
||||||
import vip.mate.workspace.core.service.MemberFileAccess;
|
import vip.mate.workspace.core.service.MemberFileAccess;
|
||||||
import vip.mate.workspace.core.service.MemberFileIsolation;
|
import vip.mate.workspace.core.service.MemberFileIsolation;
|
||||||
|
import vip.mate.workspace.core.service.ChatUploadLocationResolver;
|
||||||
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
@ -132,6 +133,7 @@ public abstract class BaseAgent {
|
|||||||
*/
|
*/
|
||||||
protected MultimodalRouter multimodalRouter;
|
protected MultimodalRouter multimodalRouter;
|
||||||
protected MediaCaptionService mediaCaptionService;
|
protected MediaCaptionService mediaCaptionService;
|
||||||
|
protected ChatUploadLocationResolver chatUploadLocationResolver;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* RFC 48 — wired by {@link AgentGraphBuilder#build} so the agent's
|
* RFC 48 — wired by {@link AgentGraphBuilder#build} so the agent's
|
||||||
@ -1007,7 +1009,7 @@ public abstract class BaseAgent {
|
|||||||
&& !contentType.contains("svg");
|
&& !contentType.contains("svg");
|
||||||
if (!isImage) continue;
|
if (!isImage) continue;
|
||||||
MediaCaptionService.CaptionResult result = mediaCaptionService.caption(
|
MediaCaptionService.CaptionResult result = mediaCaptionService.caption(
|
||||||
decision.sidecarModel(), part, userLocale, userQuestion);
|
decision.sidecarModel(), part, userLocale, userQuestion, ChatOriginHolder.get());
|
||||||
if (result.isFailure()) {
|
if (result.isFailure()) {
|
||||||
log.warn("[{}] Sidecar caption failed for {}: {}",
|
log.warn("[{}] Sidecar caption failed for {}: {}",
|
||||||
agentName, part.getFileName(), result.failure().getMessage());
|
agentName, part.getFileName(), result.failure().getMessage());
|
||||||
@ -1103,8 +1105,8 @@ public abstract class BaseAgent {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// 解析媒体文件路径:先尝试 path,再尝试 mediaId(IM 渠道下载后存在 mediaId 中),再拼接工作目录
|
// 解析媒体文件路径:先尝试 path,再尝试 mediaId(IM 渠道下载后存在 mediaId 中),再拼接工作目录
|
||||||
Path mediaPath = resolveImagePath(part.getPath());
|
Path mediaPath = resolveAttachmentPath(part);
|
||||||
if (mediaPath == null && part.getMediaId() != null) {
|
if (mediaPath == null && part.getMediaId() != null && !isWebUpload(part)) {
|
||||||
mediaPath = resolveImagePath(part.getMediaId());
|
mediaPath = resolveImagePath(part.getMediaId());
|
||||||
}
|
}
|
||||||
if (mediaPath == null) {
|
if (mediaPath == null) {
|
||||||
@ -1362,8 +1364,8 @@ public abstract class BaseAgent {
|
|||||||
for (int k = parts.size() - 1; k >= 0; k--) {
|
for (int k = parts.size() - 1; k >= 0; k--) {
|
||||||
MessageContentPart part = parts.get(k);
|
MessageContentPart part = parts.get(k);
|
||||||
if (!isResolvableImagePart(part)) continue;
|
if (!isResolvableImagePart(part)) continue;
|
||||||
Path imgPath = resolveImagePath(part.getPath());
|
Path imgPath = resolveAttachmentPath(part);
|
||||||
if (imgPath == null && part.getMediaId() != null) imgPath = resolveImagePath(part.getMediaId());
|
if (imgPath == null && part.getMediaId() != null && !isWebUpload(part)) imgPath = resolveImagePath(part.getMediaId());
|
||||||
if (imgPath == null) continue;
|
if (imgPath == null) continue;
|
||||||
String contentType = part.getContentType();
|
String contentType = part.getContentType();
|
||||||
if (contentType == null || "image/*".equals(contentType)) contentType = "image/jpeg";
|
if (contentType == null || "image/*".equals(contentType)) contentType = "image/jpeg";
|
||||||
@ -1419,6 +1421,23 @@ public abstract class BaseAgent {
|
|||||||
return new ByteArrayResource(MemberFileAccess.read(root, path, 32 * 1024 * 1024));
|
return new ByteArrayResource(MemberFileAccess.read(root, path, 32 * 1024 * 1024));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private Path resolveAttachmentPath(MessageContentPart part) {
|
||||||
|
ChatOrigin origin = ChatOriginHolder.get();
|
||||||
|
if (chatUploadLocationResolver != null && origin != null
|
||||||
|
&& "web".equals(origin.channelType()) && part.getStoredName() != null) {
|
||||||
|
// The client-visible path is informational; the stored name is resolved
|
||||||
|
// under the authenticated conversation's actual upload root.
|
||||||
|
return chatUploadLocationResolver.resolveExistingFile(origin, part.getStoredName());
|
||||||
|
}
|
||||||
|
return resolveImagePath(part.getPath());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean isWebUpload(MessageContentPart part) {
|
||||||
|
ChatOrigin origin = ChatOriginHolder.get();
|
||||||
|
return chatUploadLocationResolver != null && origin != null
|
||||||
|
&& "web".equals(origin.channelType()) && part.getStoredName() != null;
|
||||||
|
}
|
||||||
|
|
||||||
protected Path resolveImagePath(String relativePath) {
|
protected Path resolveImagePath(String relativePath) {
|
||||||
if (relativePath == null || relativePath.isBlank()) {
|
if (relativePath == null || relativePath.isBlank()) {
|
||||||
return null;
|
return null;
|
||||||
|
|||||||
@ -645,6 +645,9 @@ public class ChatController {
|
|||||||
List<MessageContentPart> requestParts = regenerateSeed != null
|
List<MessageContentPart> requestParts = regenerateSeed != null
|
||||||
? regenerateSeed.parts()
|
? regenerateSeed.parts()
|
||||||
: normalizeRequestParts(request);
|
: normalizeRequestParts(request);
|
||||||
|
if (regenerateSeed == null) {
|
||||||
|
validateUploadedParts(conversationId, selectedTurnOrigin, requestParts);
|
||||||
|
}
|
||||||
String promptText = buildPromptText(message, requestParts);
|
String promptText = buildPromptText(message, requestParts);
|
||||||
Long originMessageId;
|
Long originMessageId;
|
||||||
if (regenerateSeed == null) {
|
if (regenerateSeed == null) {
|
||||||
@ -1199,6 +1202,9 @@ public class ChatController {
|
|||||||
return R.fail(409, "正在生成回复,请先停止或排队后续消息");
|
return R.fail(409, "正在生成回复,请先停止或排队后续消息");
|
||||||
}
|
}
|
||||||
conversationService.getOrCreateConversation(request.getConversationId(), agentId, username, workspaceId);
|
conversationService.getOrCreateConversation(request.getConversationId(), agentId, username, workspaceId);
|
||||||
|
validateUploadedParts(request.getConversationId(),
|
||||||
|
vip.mate.agent.context.ChatOrigin.web(request.getConversationId(), username, workspaceId,
|
||||||
|
null, null, requesterUserIdOf(auth)), request.getContentParts());
|
||||||
MessageEntity savedUser = conversationService.saveMessage(
|
MessageEntity savedUser = conversationService.saveMessage(
|
||||||
request.getConversationId(), "user", request.getMessage(), request.getContentParts());
|
request.getConversationId(), "user", request.getMessage(), request.getContentParts());
|
||||||
|
|
||||||
@ -2116,6 +2122,28 @@ public class ChatController {
|
|||||||
return List.of(textPart);
|
return List.of(textPart);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private void validateUploadedParts(String conversationId, vip.mate.agent.context.ChatOrigin origin,
|
||||||
|
List<MessageContentPart> parts) {
|
||||||
|
if (parts == null) return;
|
||||||
|
for (MessageContentPart part : parts) {
|
||||||
|
if (part == null || part.getStoredName() == null) continue;
|
||||||
|
Path file = uploadLocationResolver.resolveExistingFile(origin, part.getStoredName());
|
||||||
|
if (file == null) {
|
||||||
|
throw new IllegalArgumentException("聊天附件不存在或不属于当前会话");
|
||||||
|
}
|
||||||
|
// Do not let a client-supplied filesystem path become an agent prompt.
|
||||||
|
if (MemberFileIsolation.isEnabled()) {
|
||||||
|
part.setPath(toRelativeUploadPath(uploadLocationResolver.resolveUploadRoot(origin), file));
|
||||||
|
} else {
|
||||||
|
Path root = uploadLocationResolver.resolveCandidateUploadRoots(conversationId).stream()
|
||||||
|
.filter(candidate -> file.startsWith(candidate))
|
||||||
|
.findFirst()
|
||||||
|
.orElseThrow(() -> new IllegalArgumentException("聊天附件目录无效"));
|
||||||
|
part.setPath(toRelativeUploadPath(root, file));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private String buildPromptText(String message, List<MessageContentPart> parts) {
|
private String buildPromptText(String message, List<MessageContentPart> parts) {
|
||||||
if (parts == null || parts.isEmpty()) {
|
if (parts == null || parts.isEmpty()) {
|
||||||
return message != null ? message : "";
|
return message != null ? message : "";
|
||||||
|
|||||||
@ -13,6 +13,8 @@ import org.springframework.util.MimeType;
|
|||||||
import vip.mate.llm.chatmodel.ProviderChatModelFactory;
|
import vip.mate.llm.chatmodel.ProviderChatModelFactory;
|
||||||
import vip.mate.llm.model.ModelConfigEntity;
|
import vip.mate.llm.model.ModelConfigEntity;
|
||||||
import vip.mate.workspace.conversation.model.MessageContentPart;
|
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||||
|
import vip.mate.agent.context.ChatOrigin;
|
||||||
|
import vip.mate.workspace.core.service.ChatUploadLocationResolver;
|
||||||
|
|
||||||
import java.nio.file.Files;
|
import java.nio.file.Files;
|
||||||
import java.nio.file.Path;
|
import java.nio.file.Path;
|
||||||
@ -40,6 +42,12 @@ public class MediaCaptionService {
|
|||||||
|
|
||||||
private final ProviderChatModelFactory chatModelFactory;
|
private final ProviderChatModelFactory chatModelFactory;
|
||||||
private final RetryTemplate retryTemplate;
|
private final RetryTemplate retryTemplate;
|
||||||
|
private ChatUploadLocationResolver uploadLocationResolver;
|
||||||
|
|
||||||
|
@org.springframework.beans.factory.annotation.Autowired
|
||||||
|
public void setUploadLocationResolver(ChatUploadLocationResolver resolver) {
|
||||||
|
this.uploadLocationResolver = resolver;
|
||||||
|
}
|
||||||
|
|
||||||
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) {
|
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale) {
|
||||||
return caption(visionModel, imagePart, locale, null);
|
return caption(visionModel, imagePart, locale, null);
|
||||||
@ -53,11 +61,16 @@ public class MediaCaptionService {
|
|||||||
* the factual full-description prompt.
|
* the factual full-description prompt.
|
||||||
*/
|
*/
|
||||||
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale,
|
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale,
|
||||||
String userQuestion) {
|
String userQuestion) {
|
||||||
|
return caption(visionModel, imagePart, locale, userQuestion, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CaptionResult caption(ModelConfigEntity visionModel, MessageContentPart imagePart, Locale locale,
|
||||||
|
String userQuestion, ChatOrigin origin) {
|
||||||
if (visionModel == null || imagePart == null) {
|
if (visionModel == null || imagePart == null) {
|
||||||
return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null"));
|
return CaptionResult.failure(0, new IllegalArgumentException("vision model or image part is null"));
|
||||||
}
|
}
|
||||||
Path mediaPath = resolveMediaPath(imagePart);
|
Path mediaPath = resolveMediaPath(imagePart, origin);
|
||||||
if (mediaPath == null) {
|
if (mediaPath == null) {
|
||||||
return CaptionResult.failure(0, new IllegalStateException(
|
return CaptionResult.failure(0, new IllegalStateException(
|
||||||
"Image file not found for attachment: " + imagePart.getFileName()));
|
"Image file not found for attachment: " + imagePart.getFileName()));
|
||||||
@ -135,7 +148,11 @@ public class MediaCaptionService {
|
|||||||
* Mirrors {@code BaseAgent.resolveImagePath} but standalone — caption service
|
* Mirrors {@code BaseAgent.resolveImagePath} but standalone — caption service
|
||||||
* is reused outside the agent context (e.g. tests, future preflight endpoint).
|
* is reused outside the agent context (e.g. tests, future preflight endpoint).
|
||||||
*/
|
*/
|
||||||
private Path resolveMediaPath(MessageContentPart part) {
|
private Path resolveMediaPath(MessageContentPart part, ChatOrigin origin) {
|
||||||
|
if (uploadLocationResolver != null && origin != null && "web".equals(origin.channelType())
|
||||||
|
&& part.getStoredName() != null) {
|
||||||
|
return uploadLocationResolver.resolveExistingFile(origin, part.getStoredName());
|
||||||
|
}
|
||||||
Path resolved = tryResolve(part.getPath());
|
Path resolved = tryResolve(part.getPath());
|
||||||
if (resolved != null) return resolved;
|
if (resolved != null) return resolved;
|
||||||
return tryResolve(part.getMediaId());
|
return tryResolve(part.getMediaId());
|
||||||
|
|||||||
@ -0,0 +1,38 @@
|
|||||||
|
package vip.mate.llm.routing;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
import org.junit.jupiter.api.io.TempDir;
|
||||||
|
import vip.mate.agent.context.ChatOrigin;
|
||||||
|
import vip.mate.workspace.conversation.model.MessageContentPart;
|
||||||
|
import vip.mate.workspace.core.service.ChatUploadLocationResolver;
|
||||||
|
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.nio.file.Files;
|
||||||
|
import java.nio.file.Path;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||||
|
import static org.mockito.Mockito.mock;
|
||||||
|
import static org.mockito.Mockito.when;
|
||||||
|
|
||||||
|
class MediaCaptionServiceAttachmentPathTest {
|
||||||
|
@TempDir Path tempDir;
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void webUploadUsesStoredNameInsteadOfClientPath() throws Exception {
|
||||||
|
ChatUploadLocationResolver locations = mock(ChatUploadLocationResolver.class);
|
||||||
|
ChatOrigin origin = ChatOrigin.web("conversation", "alice", 7L, null);
|
||||||
|
Path image = Files.writeString(tempDir.resolve("saved.png"), "image");
|
||||||
|
when(locations.resolveExistingFile(origin, "saved.png")).thenReturn(image);
|
||||||
|
|
||||||
|
MessageContentPart part = new MessageContentPart();
|
||||||
|
part.setStoredName("saved.png");
|
||||||
|
part.setPath("chat-uploads/conversation/wrong.png");
|
||||||
|
MediaCaptionService service = new MediaCaptionService(null, null);
|
||||||
|
service.setUploadLocationResolver(locations);
|
||||||
|
Method resolve = MediaCaptionService.class.getDeclaredMethod(
|
||||||
|
"resolveMediaPath", MessageContentPart.class, ChatOrigin.class);
|
||||||
|
resolve.setAccessible(true);
|
||||||
|
|
||||||
|
assertEquals(image, resolve.invoke(service, part, origin));
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -390,4 +390,20 @@ class ChatUploadLocationResolverTest {
|
|||||||
.isEqualTo(writeDir.resolve("1777_a.png"));
|
.isEqualTo(writeDir.resolve("1777_a.png"));
|
||||||
assertThat(r.resolveExistingFile("c-e2e", "nope.png")).isNull();
|
assertThat(r.resolveExistingFile("c-e2e", "nope.png")).isNull();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
@DisplayName("attachment uploaded before conversation creation remains readable after workspace binding")
|
||||||
|
void preConversationUploadIsFoundInDefaultRoot() throws Exception {
|
||||||
|
when(conversationMapper.selectOne(any(Wrapper.class))).thenReturn(null);
|
||||||
|
ChatUploadLocationResolver r = resolver(tempDir.resolve("chat-uploads"), true);
|
||||||
|
Path writeDir = r.resolveWriteDir("new-conversation");
|
||||||
|
Files.createDirectories(writeDir);
|
||||||
|
Path image = Files.writeString(writeDir.resolve("123_image.png"), "img");
|
||||||
|
|
||||||
|
stubConversation("new-conversation", 7L, null);
|
||||||
|
when(workspaceService.getById(7L)).thenReturn(workspace(7L, tempDir.resolve("workspace").toString()));
|
||||||
|
r.invalidate("new-conversation");
|
||||||
|
|
||||||
|
assertThat(r.resolveExistingFile("new-conversation", "123_image.png")).isEqualTo(image);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -57,4 +57,16 @@ class MemberFileIntegrationTest {
|
|||||||
new ChatUploadProperties(), mock(AgentService.class));
|
new ChatUploadProperties(), mock(AgentService.class));
|
||||||
assertThrows(SecurityException.class, () -> resolver.resolveCandidateUploadRoots("missing"));
|
assertThrows(SecurityException.class, () -> resolver.resolveCandidateUploadRoots("missing"));
|
||||||
}
|
}
|
||||||
|
@Test void uploadedMediaLookupStaysInsideTheMemberDirectory() throws Exception {
|
||||||
|
var resolver = new ChatUploadLocationResolver(mock(ConversationMapper.class), mock(WorkspaceService.class),
|
||||||
|
new ChatUploadProperties(), mock(AgentService.class));
|
||||||
|
var alice = ChatOrigin.web("new", "alice", 7L, null, null, 11L);
|
||||||
|
var bob = ChatOrigin.web("new", "bob", 7L, null, null, 22L);
|
||||||
|
Path image = resolver.resolveWriteDir(alice).resolve("image.png");
|
||||||
|
Files.createDirectories(image.getParent());
|
||||||
|
Files.writeString(image, "image");
|
||||||
|
|
||||||
|
assertEquals(image, resolver.resolveExistingFile(alice, "image.png"));
|
||||||
|
assertNull(resolver.resolveExistingFile(bob, "image.png"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user