From 894c945fabd80620cc21f099ff954aca5cd0a432 Mon Sep 17 00:00:00 2001 From: taobig Date: Wed, 23 Sep 2026 23:33:04 -0400 Subject: [PATCH] fix(tts): restore Edge and DashScope read-aloud (#646) --- mateclaw-server/pom.xml | 14 +++ .../tts/provider/DashScopeTtsProvider.java | 68 ++++++------ .../mate/tts/provider/EdgeTtsProvider.java | 104 ++++++++++++------ .../provider/DashScopeTtsProviderTest.java | 78 +++++++++++++ .../tts/provider/EdgeTtsProviderTest.java | 69 ++++++++++++ .../src/components/chat/MessageBubble.vue | 76 ++++++------- .../MessageBubble.stopIndicator.test.ts | 73 +++++++++++- mateclaw-ui/src/i18n/locales/en-US.ts | 1 + mateclaw-ui/src/i18n/locales/zh-CN.ts | 1 + 9 files changed, 376 insertions(+), 108 deletions(-) create mode 100644 mateclaw-server/src/test/java/vip/mate/tts/provider/DashScopeTtsProviderTest.java create mode 100644 mateclaw-server/src/test/java/vip/mate/tts/provider/EdgeTtsProviderTest.java diff --git a/mateclaw-server/pom.xml b/mateclaw-server/pom.xml index f25d542c..31db297f 100644 --- a/mateclaw-server/pom.xml +++ b/mateclaw-server/pom.xml @@ -53,6 +53,20 @@ + + + com.alibaba + dashscope-sdk-java + 2.22.2 + + + + org.slf4j + slf4j-simple + + + + com.alibaba.cloud.ai diff --git a/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java index 0519a336..22684c8c 100644 --- a/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tts/provider/DashScopeTtsProvider.java @@ -1,9 +1,9 @@ package vip.mate.tts.provider; -import cn.hutool.http.HttpRequest; -import cn.hutool.http.HttpResponse; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam; +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat; +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer; +import java.nio.ByteBuffer; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; @@ -17,11 +17,11 @@ import vip.mate.tts.TtsResult; import java.util.List; /** - * DashScope TTS Provider — 使用 CosyVoice（OpenAI 兼容接口） + * DashScope TTS Provider — 使用 CosyVoice 原生 WebSocket SDK *

* 同步模式，直接返回音频流。 * 复用已有的 DashScope LLM provider 的 API Key。 - * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-openai-compatible + * API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-java-sdk * * @author MateClaw Team */ @@ -31,11 +31,9 @@ import java.util.List; public class DashScopeTtsProvider implements TtsProvider { private final ModelProviderService modelProviderService; - private final ObjectMapper objectMapper; - private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"; private static final String DEFAULT_MODEL = "cosyvoice-v2"; - private static final String DEFAULT_VOICE = "longxiaochun"; + private static final String DEFAULT_VOICE = "longxiaochun_v2"; @Override public String id() { @@ -69,9 +67,8 @@ public class DashScopeTtsProvider implements TtsProvider { @Override public List availableVoices() { return List.of( - "longxiaochun", "longxiaoxia", "longlaotie", "longshu", - "longhua", "longshuo", "longjielidou", "longmiao", - "longyue", "longfei", "longtong", "longxiang" + "longxiaochun_v2", "longxiaoxia_v2", "longshu_v2", + "longhua_v2", "longwan_v2", "longcheng_v2" ); } @@ -83,7 +80,7 @@ public class DashScopeTtsProvider implements TtsProvider { @Override public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) { String apiKey = getDashScopeApiKey(); - if (apiKey == null) { + if (apiKey == null || apiKey.isBlank()) { return TtsResult.failure("DashScope API Key 未配置"); } @@ -93,36 +90,33 @@ public class DashScopeTtsProvider implements TtsProvider { String voice = request.getVoice() != null && !request.getVoice().isBlank() ? request.getVoice() : DEFAULT_VOICE; - ObjectNode body = objectMapper.createObjectNode(); - body.put("model", model); - body.put("input", request.getText()); - body.put("voice", voice); - body.put("response_format", "mp3"); - if (request.getSpeed() != null && request.getSpeed() != 1.0) { - body.put("speed", request.getSpeed()); + if ((request.getVoice() == null || request.getVoice().isBlank()) + && availableVoices().contains(config.getTtsDefaultVoice() == null ? "" : config.getTtsDefaultVoice())) { + voice = config.getTtsDefaultVoice(); } - - String endpoint = BASE_URL + "/audio/speech"; - HttpResponse response = HttpRequest.post(endpoint) - .header("Authorization", "Bearer " + apiKey) - .header("Content-Type", "application/json") - .body(body.toString()) - .timeout(60_000) - .execute(); - - if (response.getStatus() == 200) { - byte[] audioData = response.bodyBytes(); + SpeechSynthesisParam param = SpeechSynthesisParam.builder() + .apiKey(apiKey) + .model(model) + .voice(voice) + .format(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS) + .speechRate(request.getSpeed() == null ? 1.0f : request.getSpeed().floatValue()) + .build(); + SpeechSynthesizer synthesizer = new SpeechSynthesizer(param, null); + try { + ByteBuffer audio = synthesizer.call(request.getText(), 60_000); + if (audio == null || !audio.hasRemaining()) { + return TtsResult.failure("DashScope TTS 未返回音频数据"); + } + byte[] audioData = new byte[audio.remaining()]; + audio.get(audioData); log.info("[DashScope TTS] Synthesized {} bytes (model={}, voice={})", audioData.length, model, voice); return TtsResult.success(audioData, "audio/mpeg", "mp3"); - } else { - String errBody = response.body(); - log.warn("[DashScope TTS] Failed: HTTP {} - {}", response.getStatus(), errBody); - return TtsResult.failure(TtsResponseDiagnostics.failureMessage( - "DashScope TTS", endpoint, response.getStatus(), errBody)); + } finally { + synthesizer.getDuplexApi().close(1000, "bye"); } } catch (Exception e) { log.error("[DashScope TTS] Error: {}", e.getMessage(), e); - return TtsResult.failure("DashScope TTS 异常: " + e.getMessage()); + return TtsResult.failure("DashScope TTS 异常: " + TtsResponseDiagnostics.snippet(e.getMessage())); } } diff --git a/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java b/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java index 168d5815..2b5fb04e 100644 --- a/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java +++ b/mateclaw-server/src/main/java/vip/mate/tts/provider/EdgeTtsProvider.java @@ -13,6 +13,13 @@ import java.net.http.HttpClient; import java.net.http.WebSocket; import java.nio.ByteBuffer; import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.util.HexFormat; +import java.util.Locale; import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; @@ -31,8 +38,9 @@ import java.util.concurrent.TimeUnit; @Component public class EdgeTtsProvider implements TtsProvider { - private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud"; + private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1"; private static final String TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4"; + private static final String EDGE_VERSION = "143.0.3650.75"; private static final String DEFAULT_VOICE_ZH = "zh-CN-XiaoxiaoNeural"; private static final String DEFAULT_VOICE_EN = "en-US-MichelleNeural"; private static final String OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3"; @@ -99,7 +107,9 @@ public class EdgeTtsProvider implements TtsProvider { private byte[] synthesizeViaWebSocket(String text, String voice, String rate) throws Exception { String requestId = UUID.randomUUID().toString().replace("-", ""); String wsUrl = WS_URL + "?TrustedClientToken=" + TRUSTED_CLIENT_TOKEN - + "&ConnectionId=" + requestId; + + "&ConnectionId=" + requestId + + "&Sec-MS-GEC=" + securityToken(Instant.now()) + + "&Sec-MS-GEC-Version=1-" + EDGE_VERSION; ByteArrayOutputStream audioBuffer = new ByteArrayOutputStream(); CompletableFuture resultFuture = new CompletableFuture<>(); @@ -110,9 +120,13 @@ public class EdgeTtsProvider implements TtsProvider { WebSocket ws = client.newWebSocketBuilder() .header("Origin", "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold") - .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36") + .connectTimeout(Duration.ofSeconds(10)) + .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + + " (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0") + .header("Cookie", "muid=" + UUID.randomUUID().toString().replace("-", "").toUpperCase(Locale.ROOT) + ";") .buildAsync(URI.create(wsUrl), new WebSocket.Listener() { private final StringBuilder textBuffer = new StringBuilder(); + private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream(); @Override public void onOpen(WebSocket webSocket) { @@ -135,13 +149,21 @@ public class EdgeTtsProvider implements TtsProvider { @Override public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { - // 二进制帧：前 2 字节是 header 长度（大端），跳过 header - byte[] bytes = new byte[data.remaining()]; - data.get(bytes); - // 查找 "Path:audio\r\n" 后的音频数据 - int headerEnd = findHeaderEnd(bytes); - if (headerEnd >= 0 && headerEnd < bytes.length) { - audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd); + byte[] fragment = new byte[data.remaining()]; + data.get(fragment); + binaryBuffer.writeBytes(fragment); + if (last) { + byte[] bytes = binaryBuffer.toByteArray(); + binaryBuffer.reset(); + int headerEnd = findHeaderEnd(bytes); + if (headerEnd >= 2 && headerEnd <= bytes.length) { + String headers = new String(bytes, 2, headerEnd - 2, StandardCharsets.UTF_8); + if (headers.contains("Path:audio\r\n")) { + audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd); + } + } else { + resultFuture.completeExceptionally(new IllegalStateException("Invalid Edge TTS audio frame")); + } } webSocket.request(1); return null; @@ -150,7 +172,8 @@ public class EdgeTtsProvider implements TtsProvider { @Override public CompletionStage onClose(WebSocket webSocket, int statusCode, String reason) { if (!resultFuture.isDone()) { - resultFuture.complete(audioBuffer.toByteArray()); + resultFuture.completeExceptionally(new IllegalStateException( + "Edge TTS connection closed before turn.end: " + statusCode)); } return null; } @@ -163,29 +186,46 @@ public class EdgeTtsProvider implements TtsProvider { } }).join(); - // 发送配置消息 - String configMsg = "Content-Type:application/json; charset=utf-8\r\n" - + "Path:speech.config\r\n\r\n" - + "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{" - + "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false}," - + "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n"; - ws.sendText(configMsg, true); + try { + String timestamp = DateTimeFormatter.ofPattern( + "EEE MMM dd yyyy HH:mm:ss 'GMT+0000 (Coordinated Universal Time)'", Locale.US) + .withZone(ZoneOffset.UTC).format(Instant.now()); + // 发送配置消息 + String configMsg = "X-Timestamp:" + timestamp + "\r\nContent-Type:application/json; charset=utf-8\r\n" + + "Path:speech.config\r\n\r\n" + + "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{" + + "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false}," + + "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n"; + ws.sendText(configMsg, true).join(); - // 发送 SSML - String escapedText = escapeXml(text); - String ssml = "" - + "" - + "" - + escapedText - + ""; + // 发送 SSML + String escapedText = escapeXml(text); + String ssml = "" + + "" + + "" + + escapedText + + ""; - String ssmlMsg = "X-RequestId:" + requestId + "\r\n" - + "Content-Type:application/ssml+xml\r\n" - + "Path:ssml\r\n\r\n" + ssml; - ws.sendText(ssmlMsg, true); + String ssmlMsg = "X-RequestId:" + requestId + "\r\n" + + "X-Timestamp:" + timestamp + "Z\r\n" + + "Content-Type:application/ssml+xml\r\n" + + "Path:ssml\r\n\r\n" + ssml; + ws.sendText(ssmlMsg, true).join(); - // 等待结果（最多 60 秒） - return resultFuture.get(60, TimeUnit.SECONDS); + // 等待结果（最多 60 秒） + return resultFuture.get(60, TimeUnit.SECONDS); + } finally { + ws.abort(); + } + } + + // Edge validates a SHA-256 of the five-minute Windows FILETIME bucket and client token. + static String securityToken(Instant now) throws Exception { + long seconds = now.getEpochSecond() + 11_644_473_600L; + long ticks = (seconds - seconds % 300) * 10_000_000L; + byte[] digest = MessageDigest.getInstance("SHA-256") + .digest((ticks + TRUSTED_CLIENT_TOKEN).getBytes(StandardCharsets.US_ASCII)); + return HexFormat.of().withUpperCase().formatHex(digest); } private int findHeaderEnd(byte[] data) { @@ -200,7 +240,7 @@ public class EdgeTtsProvider implements TtsProvider { return requestedVoice; } String configVoice = config.getTtsDefaultVoice(); - if (configVoice != null && !configVoice.isBlank()) { + if (configVoice != null && availableVoices().contains(configVoice)) { return configVoice; } // 自动语言检测：CJK 字符比例 > 30% 使用中文语音 diff --git a/mateclaw-server/src/test/java/vip/mate/tts/provider/DashScopeTtsProviderTest.java b/mateclaw-server/src/test/java/vip/mate/tts/provider/DashScopeTtsProviderTest.java new file mode 100644 index 00000000..3372a8f9 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tts/provider/DashScopeTtsProviderTest.java @@ -0,0 +1,78 @@ +package vip.mate.tts.provider; + +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer; +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam; +import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat; +import org.junit.jupiter.api.Test; +import vip.mate.llm.service.ModelProviderService; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tts.TtsRequest; + +import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +class DashScopeTtsProviderTest { + @Test + void defaultVoiceMatchesCosyVoiceV2() { + var provider = new DashScopeTtsProvider(mock(ModelProviderService.class)); + assertEquals("longxiaochun_v2", provider.defaultVoice()); + assertTrue(provider.availableVoices().contains(provider.defaultVoice())); + } + + @Test + void usesNativeSdkAndClosesConnection() { + var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS); + when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key"); + AtomicReference parameters = new AtomicReference<>(); + try (var synths = mockConstruction(SpeechSynthesizer.class, + withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) -> { + parameters.set((SpeechSynthesisParam) context.arguments().get(0)); + doReturn(ByteBuffer.wrap(new byte[]{1, 2, 3})).when(synth).call("Hello", 60000); + })) { + // A default voice from another provider must not break DashScope fallback. + var config = new SystemSettingsDTO(); + config.setTtsDefaultVoice("zh-CN-XiaoxiaoNeural"); + var result = new DashScopeTtsProvider(service).synthesize( + TtsRequest.builder().text("Hello").speed(1.5).build(), config); + assertTrue(result.isSuccess(), result.getErrorMessage()); + assertArrayEquals(new byte[]{1, 2, 3}, result.getAudioData()); + assertEquals("cosyvoice-v2", parameters.get().getModel()); + assertEquals("longxiaochun_v2", parameters.get().getVoice()); + assertEquals(1.5f, parameters.get().getSpeechRate()); + assertEquals(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS, parameters.get().getFormat()); + verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye"); + } + } + @Test + void closesConnectionWhenSynthesisFails() { + var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS); + when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key"); + try (var synths = mockConstruction(SpeechSynthesizer.class, + withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) -> + doThrow(new IllegalStateException("invalid API key")).when(synth).call("Hello", 60000))) { + var result = new DashScopeTtsProvider(service).synthesize( + TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO()); + assertFalse(result.isSuccess()); + assertTrue(result.getErrorMessage().contains("invalid API key")); + verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye"); + } + } + + @Test + void rejectsEmptyAudio() { + var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS); + when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key"); + try (var synths = mockConstruction(SpeechSynthesizer.class, + withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) -> + doReturn(ByteBuffer.allocate(0)).when(synth).call("Hello", 60000))) { + var result = new DashScopeTtsProvider(service).synthesize( + TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO()); + assertFalse(result.isSuccess()); + verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye"); + } + } + +} diff --git a/mateclaw-server/src/test/java/vip/mate/tts/provider/EdgeTtsProviderTest.java b/mateclaw-server/src/test/java/vip/mate/tts/provider/EdgeTtsProviderTest.java new file mode 100644 index 00000000..5767f523 --- /dev/null +++ b/mateclaw-server/src/test/java/vip/mate/tts/provider/EdgeTtsProviderTest.java @@ -0,0 +1,69 @@ +package vip.mate.tts.provider; + +import org.junit.jupiter.api.Test; +import vip.mate.system.model.SystemSettingsDTO; +import vip.mate.tts.TtsRequest; + +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +class EdgeTtsProviderTest { + @Test + void hashesWindowsFileTimeInFiveMinuteBuckets() throws Exception { + assertEquals("7ECB79D14E3AA576D2D79E6D487A1388156D91E614B1BE11C64226A29BC8DD8C", + EdgeTtsProvider.securityToken(java.time.Instant.EPOCH)); + assertEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH), + EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(299))); + assertNotEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH), + EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(300))); + } + + @Test + void usesCurrentHandshakeAndPreservesFragmentedAudio() { + HttpClient.Builder httpBuilder = mock(HttpClient.Builder.class, RETURNS_SELF); + HttpClient client = mock(HttpClient.class); + WebSocket.Builder wsBuilder = mock(WebSocket.Builder.class, RETURNS_SELF); + WebSocket socket = mock(WebSocket.class); + when(httpBuilder.build()).thenReturn(client); + when(client.newWebSocketBuilder()).thenReturn(wsBuilder); + AtomicReference endpoint = new AtomicReference<>(); + AtomicReference listener = new AtomicReference<>(); + when(wsBuilder.buildAsync(any(), any())).thenAnswer(inv -> { + endpoint.set(inv.getArgument(0)); + listener.set(inv.getArgument(1)); + return CompletableFuture.completedFuture(socket); + }); + when(socket.sendText(anyString(), eq(true))).thenAnswer(inv -> { + if (inv.getArgument(0).contains("Path:ssml")) { + byte[] headers = "Path:audio\r\nContent-Type:audio/mpeg\r\n".getBytes(StandardCharsets.UTF_8); + byte[] packet = ByteBuffer.allocate(2 + headers.length + 4) + .putShort((short) headers.length).put(headers).put(new byte[]{1, 2, 3, 4}).array(); + listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 0, 1)), false); + listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 1, packet.length)), true); + listener.get().onText(socket, "Path:turn.end\r\n", true); + } + return CompletableFuture.completedFuture(socket); + }); + try (var clients = mockStatic(HttpClient.class)) { + clients.when(HttpClient::newBuilder).thenReturn(httpBuilder); + var result = new EdgeTtsProvider().synthesize( + TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO()); + assertEquals("/consumer/speech/synthesize/readaloud/edge/v1", endpoint.get().getPath()); + assertTrue(endpoint.get().getQuery().matches(".*Sec-MS-GEC=[A-F0-9]{64}.*")); + assertTrue(endpoint.get().getQuery().contains("Sec-MS-GEC-Version=1-")); + assertTrue(result.isSuccess(), result.getErrorMessage()); + assertArrayEquals(new byte[]{1, 2, 3, 4}, result.getAudioData()); + verify(socket).abort(); + } + } +} diff --git a/mateclaw-ui/src/components/chat/MessageBubble.vue b/mateclaw-ui/src/components/chat/MessageBubble.vue index 7e34973d..e9154438 100644 --- a/mateclaw-ui/src/components/chat/MessageBubble.vue +++ b/mateclaw-ui/src/components/chat/MessageBubble.vue @@ -587,7 +587,7 @@ import { buildGeneratedFileNameMap, linkifyGeneratedFileUrls } from '@/utils/gen import { ensureModelViewer } from '@/utils/lazyModelViewer' import { useAuthenticatedAttachment } from '@/composables/useAuthenticatedAttachment' import { useToolLabel } from '@/composables/useToolLabel' -import { http } from '@/api' +import { http, fetchAuthenticatedBlob } from '@/api' import { copyToClipboard } from '@/utils/clipboard' import TypingCursor from './TypingCursor.vue' import { previewKindOf } from './preview/previewKind' @@ -853,59 +853,59 @@ function copyMessage() { // --- TTS 朗读 --- const ttsState = ref<'idle' | 'loading' | 'playing'>('idle') let ttsAudio: HTMLAudioElement | null = null +let ttsBlobUrl: string | null = null +let ttsRequestVersion = 0 + +function stopTts() { + ttsAudio?.pause() + if (ttsAudio) { ttsAudio.onended = null; ttsAudio.onerror = null } + ttsAudio = null + if (ttsBlobUrl) URL.revokeObjectURL(ttsBlobUrl) + ttsBlobUrl = null + ttsState.value = 'idle' +} async function handleTts() { if (ttsState.value === 'playing') { - // 停止播放 - ttsAudio?.pause() - ttsAudio = null - ttsState.value = 'idle' + stopTts() return } - + if (ttsState.value === 'loading') return const text = displayContent.value || props.message.content || '' - if (!text) return - const conversationId = props.message.conversationId - if (!conversationId) return + if (!text || !conversationId) return + const version = ++ttsRequestVersion ttsState.value = 'loading' try { - const res: any = await http.post('/tts/synthesize', { - conversationId, - text, - }) - if (res.data?.success && res.data?.audioUrl) { - // 通过认证 fetch 获取音频 blob - const audioRes = await fetch(res.data.audioUrl, { - headers: { Authorization: `Bearer ${localStorage.getItem('token') || ''}` }, - }) - const blob = await audioRes.blob() - const blobUrl = URL.createObjectURL(blob) - ttsAudio = new Audio(blobUrl) - ttsAudio.onended = () => { - ttsState.value = 'idle' - URL.revokeObjectURL(blobUrl) - ttsAudio = null - } - ttsAudio.onerror = () => { - ttsState.value = 'idle' - URL.revokeObjectURL(blobUrl) - ttsAudio = null - } - ttsState.value = 'playing' - await ttsAudio.play() - } else { - ttsState.value = 'idle' + // The shared interceptor already unwraps the raw TTS response body. + const result: any = await http.post('/tts/synthesize', { conversationId, text }, { timeout: 240_000 }) + if (version !== ttsRequestVersion) return + if (!result?.success || !result.audioUrl) { + throw new Error(result?.error || t('chat.ttsFailed')) } - } catch { - ttsState.value = 'idle' + const blob = await fetchAuthenticatedBlob(result.audioUrl) + if (version !== ttsRequestVersion) return + ttsBlobUrl = URL.createObjectURL(blob) + ttsAudio = new Audio(ttsBlobUrl) + ttsAudio.onended = stopTts + ttsAudio.onerror = () => { + stopTts() + mcToast.error(t('chat.ttsFailed')) + } + await ttsAudio.play() + if (version === ttsRequestVersion && ttsAudio) ttsState.value = 'playing' + } catch (error) { + if (version !== ttsRequestVersion) return + stopTts() + mcToast.error(error instanceof Error ? error.message : t('chat.ttsFailed')) } } onBeforeUnmount(() => { if (copyTimer) clearTimeout(copyTimer) - if (ttsAudio) { ttsAudio.pause(); ttsAudio = null } + ++ttsRequestVersion + stopTts() revokeAll() }) diff --git a/mateclaw-ui/src/components/chat/__tests__/MessageBubble.stopIndicator.test.ts b/mateclaw-ui/src/components/chat/__tests__/MessageBubble.stopIndicator.test.ts index 7613cd5e..608f1986 100644 --- a/mateclaw-ui/src/components/chat/__tests__/MessageBubble.stopIndicator.test.ts +++ b/mateclaw-ui/src/components/chat/__tests__/MessageBubble.stopIndicator.test.ts @@ -29,7 +29,7 @@ vi.mock('@/composables/useMcToast', () => ({ vi.mock('@/composables/useToolLabel', () => ({ useToolLabel: () => ({ getToolLabel: (name: string) => name }), })) -vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() } })) +vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() }, fetchAuthenticatedBlob: vi.fn() })) vi.mock('@/utils/clipboard', () => ({ copyToClipboard: vi.fn() })) vi.mock('@/utils/generatedFileLinks', () => ({ buildGeneratedFileNameMap: vi.fn(() => new Map()), @@ -58,6 +58,8 @@ vi.mock('../UserMessageContent.vue', () => ({ })) import MessageBubble from '../MessageBubble.vue' +import { http, fetchAuthenticatedBlob } from '@/api' +import { mcToast } from '@/composables/useMcToast' const apps: Array> = [] @@ -92,6 +94,9 @@ function mountMessage(message: Message, locale = 'zh-CN') { afterEach(() => { apps.splice(0).forEach(app => app.unmount()) document.body.innerHTML = '' + vi.restoreAllMocks() + vi.unstubAllGlobals() + vi.clearAllMocks() }) describe('MessageBubble stop indicator', () => { @@ -130,3 +135,69 @@ describe('MessageBubble stop indicator', () => { expect(host.querySelector('.stopped-indicator--interrupted')).not.toBeNull() }) }) + + +describe('MessageBubble TTS (#646)', () => { + function clickReadAloud() { + const host = mountMessage({ id: 'tts', conversationId: 'conv', role: 'assistant', + content: 'Hello', contentParts: [], status: 'completed' } as Message) + const button = host.querySelector('[title="chat.ttsPlay"]')! + expect(button).not.toBeNull() + button.click() + return button + } + + it('plays the unwrapped synthesis response and releases audio on stop', async () => { + vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/api/v1/chat/files/conv/tts.mp3' } as never) + vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio'])) + const play = vi.fn().mockResolvedValue(undefined) + const pause = vi.fn() + vi.stubGlobal('Audio', class { play = play; pause = pause }) + vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:tts') + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + const button = clickReadAloud() + await vi.waitFor(() => expect(play).toHaveBeenCalledOnce()) + expect(fetchAuthenticatedBlob).toHaveBeenCalledWith('/api/v1/chat/files/conv/tts.mp3') + button.click() + expect(pause).toHaveBeenCalledOnce() + expect(revoke).toHaveBeenCalledWith('blob:tts') + }) + + it('shows backend synthesis errors instead of silently doing nothing', async () => { + vi.mocked(http.post).mockResolvedValue({ success: false, error: 'DashScope: invalid voice' } as never) + clickReadAloud() + await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('DashScope: invalid voice')) + }) + + it('shows audio download failures and resets the loading state', async () => { + vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never) + vi.mocked(fetchAuthenticatedBlob).mockRejectedValue(new Error('Fetch failed: 403')) + const button = clickReadAloud() + await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Fetch failed: 403')) + expect(button.disabled).toBe(false) + }) + it('releases the audio URL when playback is rejected', async () => { + vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never) + vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio'])) + const pause = vi.fn() + vi.stubGlobal('Audio', class { pause = pause; play = vi.fn().mockRejectedValue(new Error('Playback blocked')) }) + vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:rejected') + const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + clickReadAloud() + await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Playback blocked')) + expect(pause).toHaveBeenCalledOnce() + expect(revoke).toHaveBeenCalledWith('blob:rejected') + }) + + it('does not start playback when the component unmounts during synthesis', async () => { + let finish!: (value: any) => void + vi.mocked(http.post).mockReturnValue(new Promise(resolve => { finish = resolve })) + clickReadAloud() + apps.pop()!.unmount() + finish({ success: true, audioUrl: '/audio' }) + await Promise.resolve() + await Promise.resolve() + expect(fetchAuthenticatedBlob).not.toHaveBeenCalled() + }) + +}) diff --git a/mateclaw-ui/src/i18n/locales/en-US.ts b/mateclaw-ui/src/i18n/locales/en-US.ts index 8478ca45..2436902a 100644 --- a/mateclaw-ui/src/i18n/locales/en-US.ts +++ b/mateclaw-ui/src/i18n/locales/en-US.ts @@ -497,6 +497,7 @@ export default { }, }, }, + ttsFailed: 'Speech synthesis or playback failed', ttsPlay: 'Read Aloud', ttsStop: 'Stop Reading', conversations: 'Conversations', diff --git a/mateclaw-ui/src/i18n/locales/zh-CN.ts b/mateclaw-ui/src/i18n/locales/zh-CN.ts index 39a7c772..13334ea4 100644 --- a/mateclaw-ui/src/i18n/locales/zh-CN.ts +++ b/mateclaw-ui/src/i18n/locales/zh-CN.ts @@ -497,6 +497,7 @@ export default { }, }, }, + ttsFailed: '语音合成或播放失败', ttsPlay: '朗读', ttsStop: '停止朗读', conversations: '会话列表',