fix(tts): restore Edge and DashScope read-aloud (#646)

This commit is contained in:
taobig 2026-09-23 23:33:04 -04:00
parent 9efad1a910
commit 894c945fab
9 changed files with 376 additions and 108 deletions

View File

@ -53,6 +53,20 @@
</exclusions>
</dependency>
<!-- CosyVoice uses the native WebSocket API, not OpenAI audio/speech. -->
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>dashscope-sdk-java</artifactId>
<version>2.22.2</version>
<exclusions>
<!-- Spring Boot already provides Logback. -->
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- ===== Spring AI Alibaba Graph Core (StateGraph workflow engine) ===== -->
<dependency>
<groupId>com.alibaba.cloud.ai</groupId>

View File

@ -1,9 +1,9 @@
package vip.mate.tts.provider;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer;
import java.nio.ByteBuffer;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.stereotype.Component;
@ -17,11 +17,11 @@ import vip.mate.tts.TtsResult;
import java.util.List;
/**
* DashScope TTS Provider — 使用 CosyVoice(OpenAI 兼容接口)
* DashScope TTS Provider — 使用 CosyVoice 原生 WebSocket SDK
* <p>
* 同步模式,直接返回音频流。
* 复用已有的 DashScope LLM provider 的 API Key。
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-openai-compatible
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-java-sdk
*
* @author MateClaw Team
*/
@ -31,11 +31,9 @@ import java.util.List;
public class DashScopeTtsProvider implements TtsProvider {
private final ModelProviderService modelProviderService;
private final ObjectMapper objectMapper;
private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
private static final String DEFAULT_MODEL = "cosyvoice-v2";
private static final String DEFAULT_VOICE = "longxiaochun";
private static final String DEFAULT_VOICE = "longxiaochun_v2";
@Override
public String id() {
@ -69,9 +67,8 @@ public class DashScopeTtsProvider implements TtsProvider {
@Override
public List<String> availableVoices() {
return List.of(
"longxiaochun", "longxiaoxia", "longlaotie", "longshu",
"longhua", "longshuo", "longjielidou", "longmiao",
"longyue", "longfei", "longtong", "longxiang"
"longxiaochun_v2", "longxiaoxia_v2", "longshu_v2",
"longhua_v2", "longwan_v2", "longcheng_v2"
);
}
@ -83,7 +80,7 @@ public class DashScopeTtsProvider implements TtsProvider {
@Override
public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) {
String apiKey = getDashScopeApiKey();
if (apiKey == null) {
if (apiKey == null || apiKey.isBlank()) {
return TtsResult.failure("DashScope API Key 未配置");
}
@ -93,36 +90,33 @@ public class DashScopeTtsProvider implements TtsProvider {
String voice = request.getVoice() != null && !request.getVoice().isBlank()
? request.getVoice() : DEFAULT_VOICE;
ObjectNode body = objectMapper.createObjectNode();
body.put("model", model);
body.put("input", request.getText());
body.put("voice", voice);
body.put("response_format", "mp3");
if (request.getSpeed() != null && request.getSpeed() != 1.0) {
body.put("speed", request.getSpeed());
if ((request.getVoice() == null || request.getVoice().isBlank())
&& availableVoices().contains(config.getTtsDefaultVoice() == null ? "" : config.getTtsDefaultVoice())) {
voice = config.getTtsDefaultVoice();
}
String endpoint = BASE_URL + "/audio/speech";
HttpResponse response = HttpRequest.post(endpoint)
.header("Authorization", "Bearer " + apiKey)
.header("Content-Type", "application/json")
.body(body.toString())
.timeout(60_000)
.execute();
if (response.getStatus() == 200) {
byte[] audioData = response.bodyBytes();
SpeechSynthesisParam param = SpeechSynthesisParam.builder()
.apiKey(apiKey)
.model(model)
.voice(voice)
.format(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS)
.speechRate(request.getSpeed() == null ? 1.0f : request.getSpeed().floatValue())
.build();
SpeechSynthesizer synthesizer = new SpeechSynthesizer(param, null);
try {
ByteBuffer audio = synthesizer.call(request.getText(), 60_000);
if (audio == null || !audio.hasRemaining()) {
return TtsResult.failure("DashScope TTS 未返回音频数据");
}
byte[] audioData = new byte[audio.remaining()];
audio.get(audioData);
log.info("[DashScope TTS] Synthesized {} bytes (model={}, voice={})", audioData.length, model, voice);
return TtsResult.success(audioData, "audio/mpeg", "mp3");
} else {
String errBody = response.body();
log.warn("[DashScope TTS] Failed: HTTP {} - {}", response.getStatus(), errBody);
return TtsResult.failure(TtsResponseDiagnostics.failureMessage(
"DashScope TTS", endpoint, response.getStatus(), errBody));
} finally {
synthesizer.getDuplexApi().close(1000, "bye");
}
} catch (Exception e) {
log.error("[DashScope TTS] Error: {}", e.getMessage(), e);
return TtsResult.failure("DashScope TTS 异常: " + e.getMessage());
return TtsResult.failure("DashScope TTS 异常: " + TtsResponseDiagnostics.snippet(e.getMessage()));
}
}

View File

@ -13,6 +13,13 @@ import java.net.http.HttpClient;
import java.net.http.WebSocket;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.HexFormat;
import java.util.Locale;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@ -31,8 +38,9 @@ import java.util.concurrent.TimeUnit;
@Component
public class EdgeTtsProvider implements TtsProvider {
private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud";
private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1";
private static final String TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4";
private static final String EDGE_VERSION = "143.0.3650.75";
private static final String DEFAULT_VOICE_ZH = "zh-CN-XiaoxiaoNeural";
private static final String DEFAULT_VOICE_EN = "en-US-MichelleNeural";
private static final String OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3";
@ -99,7 +107,9 @@ public class EdgeTtsProvider implements TtsProvider {
private byte[] synthesizeViaWebSocket(String text, String voice, String rate) throws Exception {
String requestId = UUID.randomUUID().toString().replace("-", "");
String wsUrl = WS_URL + "?TrustedClientToken=" + TRUSTED_CLIENT_TOKEN
+ "&ConnectionId=" + requestId;
+ "&ConnectionId=" + requestId
+ "&Sec-MS-GEC=" + securityToken(Instant.now())
+ "&Sec-MS-GEC-Version=1-" + EDGE_VERSION;
ByteArrayOutputStream audioBuffer = new ByteArrayOutputStream();
CompletableFuture<byte[]> resultFuture = new CompletableFuture<>();
@ -110,9 +120,13 @@ public class EdgeTtsProvider implements TtsProvider {
WebSocket ws = client.newWebSocketBuilder()
.header("Origin", "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold")
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
.connectTimeout(Duration.ofSeconds(10))
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+ " (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0")
.header("Cookie", "muid=" + UUID.randomUUID().toString().replace("-", "").toUpperCase(Locale.ROOT) + ";")
.buildAsync(URI.create(wsUrl), new WebSocket.Listener() {
private final StringBuilder textBuffer = new StringBuilder();
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
@Override
public void onOpen(WebSocket webSocket) {
@ -135,13 +149,21 @@ public class EdgeTtsProvider implements TtsProvider {
@Override
public CompletionStage<?> onBinary(WebSocket webSocket, ByteBuffer data, boolean last) {
// 二进制帧:前 2 字节是 header 长度(大端),跳过 header
byte[] bytes = new byte[data.remaining()];
data.get(bytes);
// 查找 "Path:audio\r\n" 后的音频数据
int headerEnd = findHeaderEnd(bytes);
if (headerEnd >= 0 && headerEnd < bytes.length) {
audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd);
byte[] fragment = new byte[data.remaining()];
data.get(fragment);
binaryBuffer.writeBytes(fragment);
if (last) {
byte[] bytes = binaryBuffer.toByteArray();
binaryBuffer.reset();
int headerEnd = findHeaderEnd(bytes);
if (headerEnd >= 2 && headerEnd <= bytes.length) {
String headers = new String(bytes, 2, headerEnd - 2, StandardCharsets.UTF_8);
if (headers.contains("Path:audio\r\n")) {
audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd);
}
} else {
resultFuture.completeExceptionally(new IllegalStateException("Invalid Edge TTS audio frame"));
}
}
webSocket.request(1);
return null;
@ -150,7 +172,8 @@ public class EdgeTtsProvider implements TtsProvider {
@Override
public CompletionStage<?> onClose(WebSocket webSocket, int statusCode, String reason) {
if (!resultFuture.isDone()) {
resultFuture.complete(audioBuffer.toByteArray());
resultFuture.completeExceptionally(new IllegalStateException(
"Edge TTS connection closed before turn.end: " + statusCode));
}
return null;
}
@ -163,29 +186,46 @@ public class EdgeTtsProvider implements TtsProvider {
}
}).join();
// 发送配置消息
String configMsg = "Content-Type:application/json; charset=utf-8\r\n"
+ "Path:speech.config\r\n\r\n"
+ "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{"
+ "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false},"
+ "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n";
ws.sendText(configMsg, true);
try {
String timestamp = DateTimeFormatter.ofPattern(
"EEE MMM dd yyyy HH:mm:ss 'GMT+0000 (Coordinated Universal Time)'", Locale.US)
.withZone(ZoneOffset.UTC).format(Instant.now());
// 发送配置消息
String configMsg = "X-Timestamp:" + timestamp + "\r\nContent-Type:application/json; charset=utf-8\r\n"
+ "Path:speech.config\r\n\r\n"
+ "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{"
+ "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false},"
+ "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n";
ws.sendText(configMsg, true).join();
// 发送 SSML
String escapedText = escapeXml(text);
String ssml = "<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>"
+ "<voice name='" + voice + "'>"
+ "<prosody pitch='+0Hz' rate='" + rate + "' volume='+0%'>"
+ escapedText
+ "</prosody></voice></speak>";
// 发送 SSML
String escapedText = escapeXml(text);
String ssml = "<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='en-US'>"
+ "<voice name='" + voice + "'>"
+ "<prosody pitch='+0Hz' rate='" + rate + "' volume='+0%'>"
+ escapedText
+ "</prosody></voice></speak>";
String ssmlMsg = "X-RequestId:" + requestId + "\r\n"
+ "Content-Type:application/ssml+xml\r\n"
+ "Path:ssml\r\n\r\n" + ssml;
ws.sendText(ssmlMsg, true);
String ssmlMsg = "X-RequestId:" + requestId + "\r\n"
+ "X-Timestamp:" + timestamp + "Z\r\n"
+ "Content-Type:application/ssml+xml\r\n"
+ "Path:ssml\r\n\r\n" + ssml;
ws.sendText(ssmlMsg, true).join();
// 等待结果(最多 60 秒)
return resultFuture.get(60, TimeUnit.SECONDS);
// 等待结果(最多 60 秒)
return resultFuture.get(60, TimeUnit.SECONDS);
} finally {
ws.abort();
}
}
// Edge validates a SHA-256 of the five-minute Windows FILETIME bucket and client token.
static String securityToken(Instant now) throws Exception {
long seconds = now.getEpochSecond() + 11_644_473_600L;
long ticks = (seconds - seconds % 300) * 10_000_000L;
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest((ticks + TRUSTED_CLIENT_TOKEN).getBytes(StandardCharsets.US_ASCII));
return HexFormat.of().withUpperCase().formatHex(digest);
}
private int findHeaderEnd(byte[] data) {
@ -200,7 +240,7 @@ public class EdgeTtsProvider implements TtsProvider {
return requestedVoice;
}
String configVoice = config.getTtsDefaultVoice();
if (configVoice != null && !configVoice.isBlank()) {
if (configVoice != null && availableVoices().contains(configVoice)) {
return configVoice;
}
// 自动语言检测:CJK 字符比例 > 30% 使用中文语音

View File

@ -0,0 +1,78 @@
package vip.mate.tts.provider;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam;
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat;
import org.junit.jupiter.api.Test;
import vip.mate.llm.service.ModelProviderService;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tts.TtsRequest;
import java.nio.ByteBuffer;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;
class DashScopeTtsProviderTest {
@Test
void defaultVoiceMatchesCosyVoiceV2() {
var provider = new DashScopeTtsProvider(mock(ModelProviderService.class));
assertEquals("longxiaochun_v2", provider.defaultVoice());
assertTrue(provider.availableVoices().contains(provider.defaultVoice()));
}
@Test
void usesNativeSdkAndClosesConnection() {
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
AtomicReference<SpeechSynthesisParam> parameters = new AtomicReference<>();
try (var synths = mockConstruction(SpeechSynthesizer.class,
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) -> {
parameters.set((SpeechSynthesisParam) context.arguments().get(0));
doReturn(ByteBuffer.wrap(new byte[]{1, 2, 3})).when(synth).call("Hello", 60000);
})) {
// A default voice from another provider must not break DashScope fallback.
var config = new SystemSettingsDTO();
config.setTtsDefaultVoice("zh-CN-XiaoxiaoNeural");
var result = new DashScopeTtsProvider(service).synthesize(
TtsRequest.builder().text("Hello").speed(1.5).build(), config);
assertTrue(result.isSuccess(), result.getErrorMessage());
assertArrayEquals(new byte[]{1, 2, 3}, result.getAudioData());
assertEquals("cosyvoice-v2", parameters.get().getModel());
assertEquals("longxiaochun_v2", parameters.get().getVoice());
assertEquals(1.5f, parameters.get().getSpeechRate());
assertEquals(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS, parameters.get().getFormat());
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
}
}
@Test
void closesConnectionWhenSynthesisFails() {
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
try (var synths = mockConstruction(SpeechSynthesizer.class,
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) ->
doThrow(new IllegalStateException("invalid API key")).when(synth).call("Hello", 60000))) {
var result = new DashScopeTtsProvider(service).synthesize(
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
assertFalse(result.isSuccess());
assertTrue(result.getErrorMessage().contains("invalid API key"));
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
}
}
@Test
void rejectsEmptyAudio() {
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
try (var synths = mockConstruction(SpeechSynthesizer.class,
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) ->
doReturn(ByteBuffer.allocate(0)).when(synth).call("Hello", 60000))) {
var result = new DashScopeTtsProvider(service).synthesize(
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
assertFalse(result.isSuccess());
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
}
}
}

View File

@ -0,0 +1,69 @@
package vip.mate.tts.provider;
import org.junit.jupiter.api.Test;
import vip.mate.system.model.SystemSettingsDTO;
import vip.mate.tts.TtsRequest;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.WebSocket;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
class EdgeTtsProviderTest {
@Test
void hashesWindowsFileTimeInFiveMinuteBuckets() throws Exception {
assertEquals("7ECB79D14E3AA576D2D79E6D487A1388156D91E614B1BE11C64226A29BC8DD8C",
EdgeTtsProvider.securityToken(java.time.Instant.EPOCH));
assertEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH),
EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(299)));
assertNotEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH),
EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(300)));
}
@Test
void usesCurrentHandshakeAndPreservesFragmentedAudio() {
HttpClient.Builder httpBuilder = mock(HttpClient.Builder.class, RETURNS_SELF);
HttpClient client = mock(HttpClient.class);
WebSocket.Builder wsBuilder = mock(WebSocket.Builder.class, RETURNS_SELF);
WebSocket socket = mock(WebSocket.class);
when(httpBuilder.build()).thenReturn(client);
when(client.newWebSocketBuilder()).thenReturn(wsBuilder);
AtomicReference<URI> endpoint = new AtomicReference<>();
AtomicReference<WebSocket.Listener> listener = new AtomicReference<>();
when(wsBuilder.buildAsync(any(), any())).thenAnswer(inv -> {
endpoint.set(inv.getArgument(0));
listener.set(inv.getArgument(1));
return CompletableFuture.completedFuture(socket);
});
when(socket.sendText(anyString(), eq(true))).thenAnswer(inv -> {
if (inv.<String>getArgument(0).contains("Path:ssml")) {
byte[] headers = "Path:audio\r\nContent-Type:audio/mpeg\r\n".getBytes(StandardCharsets.UTF_8);
byte[] packet = ByteBuffer.allocate(2 + headers.length + 4)
.putShort((short) headers.length).put(headers).put(new byte[]{1, 2, 3, 4}).array();
listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 0, 1)), false);
listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 1, packet.length)), true);
listener.get().onText(socket, "Path:turn.end\r\n", true);
}
return CompletableFuture.completedFuture(socket);
});
try (var clients = mockStatic(HttpClient.class)) {
clients.when(HttpClient::newBuilder).thenReturn(httpBuilder);
var result = new EdgeTtsProvider().synthesize(
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
assertEquals("/consumer/speech/synthesize/readaloud/edge/v1", endpoint.get().getPath());
assertTrue(endpoint.get().getQuery().matches(".*Sec-MS-GEC=[A-F0-9]{64}.*"));
assertTrue(endpoint.get().getQuery().contains("Sec-MS-GEC-Version=1-"));
assertTrue(result.isSuccess(), result.getErrorMessage());
assertArrayEquals(new byte[]{1, 2, 3, 4}, result.getAudioData());
verify(socket).abort();
}
}
}

View File

@ -587,7 +587,7 @@ import { buildGeneratedFileNameMap, linkifyGeneratedFileUrls } from '@/utils/gen
import { ensureModelViewer } from '@/utils/lazyModelViewer'
import { useAuthenticatedAttachment } from '@/composables/useAuthenticatedAttachment'
import { useToolLabel } from '@/composables/useToolLabel'
import { http } from '@/api'
import { http, fetchAuthenticatedBlob } from '@/api'
import { copyToClipboard } from '@/utils/clipboard'
import TypingCursor from './TypingCursor.vue'
import { previewKindOf } from './preview/previewKind'
@ -853,59 +853,59 @@ function copyMessage() {
// --- TTS 朗读 ---
const ttsState = ref<'idle' | 'loading' | 'playing'>('idle')
let ttsAudio: HTMLAudioElement | null = null
let ttsBlobUrl: string | null = null
let ttsRequestVersion = 0
function stopTts() {
ttsAudio?.pause()
if (ttsAudio) { ttsAudio.onended = null; ttsAudio.onerror = null }
ttsAudio = null
if (ttsBlobUrl) URL.revokeObjectURL(ttsBlobUrl)
ttsBlobUrl = null
ttsState.value = 'idle'
}
async function handleTts() {
if (ttsState.value === 'playing') {
// 停止播放
ttsAudio?.pause()
ttsAudio = null
ttsState.value = 'idle'
stopTts()
return
}
if (ttsState.value === 'loading') return
const text = displayContent.value || props.message.content || ''
if (!text) return
const conversationId = props.message.conversationId
if (!conversationId) return
if (!text || !conversationId) return
const version = ++ttsRequestVersion
ttsState.value = 'loading'
try {
const res: any = await http.post('/tts/synthesize', {
conversationId,
text,
})
if (res.data?.success && res.data?.audioUrl) {
// 通过认证 fetch 获取音频 blob
const audioRes = await fetch(res.data.audioUrl, {
headers: { Authorization: `Bearer ${localStorage.getItem('token') || ''}` },
})
const blob = await audioRes.blob()
const blobUrl = URL.createObjectURL(blob)
ttsAudio = new Audio(blobUrl)
ttsAudio.onended = () => {
ttsState.value = 'idle'
URL.revokeObjectURL(blobUrl)
ttsAudio = null
}
ttsAudio.onerror = () => {
ttsState.value = 'idle'
URL.revokeObjectURL(blobUrl)
ttsAudio = null
}
ttsState.value = 'playing'
await ttsAudio.play()
} else {
ttsState.value = 'idle'
// The shared interceptor already unwraps the raw TTS response body.
const result: any = await http.post('/tts/synthesize', { conversationId, text }, { timeout: 240_000 })
if (version !== ttsRequestVersion) return
if (!result?.success || !result.audioUrl) {
throw new Error(result?.error || t('chat.ttsFailed'))
}
} catch {
ttsState.value = 'idle'
const blob = await fetchAuthenticatedBlob(result.audioUrl)
if (version !== ttsRequestVersion) return
ttsBlobUrl = URL.createObjectURL(blob)
ttsAudio = new Audio(ttsBlobUrl)
ttsAudio.onended = stopTts
ttsAudio.onerror = () => {
stopTts()
mcToast.error(t('chat.ttsFailed'))
}
await ttsAudio.play()
if (version === ttsRequestVersion && ttsAudio) ttsState.value = 'playing'
} catch (error) {
if (version !== ttsRequestVersion) return
stopTts()
mcToast.error(error instanceof Error ? error.message : t('chat.ttsFailed'))
}
}
onBeforeUnmount(() => {
if (copyTimer) clearTimeout(copyTimer)
if (ttsAudio) { ttsAudio.pause(); ttsAudio = null }
++ttsRequestVersion
stopTts()
revokeAll()
})

View File

@ -29,7 +29,7 @@ vi.mock('@/composables/useMcToast', () => ({
vi.mock('@/composables/useToolLabel', () => ({
useToolLabel: () => ({ getToolLabel: (name: string) => name }),
}))
vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() } }))
vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() }, fetchAuthenticatedBlob: vi.fn() }))
vi.mock('@/utils/clipboard', () => ({ copyToClipboard: vi.fn() }))
vi.mock('@/utils/generatedFileLinks', () => ({
buildGeneratedFileNameMap: vi.fn(() => new Map()),
@ -58,6 +58,8 @@ vi.mock('../UserMessageContent.vue', () => ({
}))
import MessageBubble from '../MessageBubble.vue'
import { http, fetchAuthenticatedBlob } from '@/api'
import { mcToast } from '@/composables/useMcToast'
const apps: Array<ReturnType<typeof createApp>> = []
@ -92,6 +94,9 @@ function mountMessage(message: Message, locale = 'zh-CN') {
afterEach(() => {
apps.splice(0).forEach(app => app.unmount())
document.body.innerHTML = ''
vi.restoreAllMocks()
vi.unstubAllGlobals()
vi.clearAllMocks()
})
describe('MessageBubble stop indicator', () => {
@ -130,3 +135,69 @@ describe('MessageBubble stop indicator', () => {
expect(host.querySelector('.stopped-indicator--interrupted')).not.toBeNull()
})
})
describe('MessageBubble TTS (#646)', () => {
function clickReadAloud() {
const host = mountMessage({ id: 'tts', conversationId: 'conv', role: 'assistant',
content: 'Hello', contentParts: [], status: 'completed' } as Message)
const button = host.querySelector<HTMLButtonElement>('[title="chat.ttsPlay"]')!
expect(button).not.toBeNull()
button.click()
return button
}
it('plays the unwrapped synthesis response and releases audio on stop', async () => {
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/api/v1/chat/files/conv/tts.mp3' } as never)
vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio']))
const play = vi.fn().mockResolvedValue(undefined)
const pause = vi.fn()
vi.stubGlobal('Audio', class { play = play; pause = pause })
vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:tts')
const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {})
const button = clickReadAloud()
await vi.waitFor(() => expect(play).toHaveBeenCalledOnce())
expect(fetchAuthenticatedBlob).toHaveBeenCalledWith('/api/v1/chat/files/conv/tts.mp3')
button.click()
expect(pause).toHaveBeenCalledOnce()
expect(revoke).toHaveBeenCalledWith('blob:tts')
})
it('shows backend synthesis errors instead of silently doing nothing', async () => {
vi.mocked(http.post).mockResolvedValue({ success: false, error: 'DashScope: invalid voice' } as never)
clickReadAloud()
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('DashScope: invalid voice'))
})
it('shows audio download failures and resets the loading state', async () => {
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never)
vi.mocked(fetchAuthenticatedBlob).mockRejectedValue(new Error('Fetch failed: 403'))
const button = clickReadAloud()
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Fetch failed: 403'))
expect(button.disabled).toBe(false)
})
it('releases the audio URL when playback is rejected', async () => {
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never)
vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio']))
const pause = vi.fn()
vi.stubGlobal('Audio', class { pause = pause; play = vi.fn().mockRejectedValue(new Error('Playback blocked')) })
vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:rejected')
const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {})
clickReadAloud()
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Playback blocked'))
expect(pause).toHaveBeenCalledOnce()
expect(revoke).toHaveBeenCalledWith('blob:rejected')
})
it('does not start playback when the component unmounts during synthesis', async () => {
let finish!: (value: any) => void
vi.mocked(http.post).mockReturnValue(new Promise(resolve => { finish = resolve }))
clickReadAloud()
apps.pop()!.unmount()
finish({ success: true, audioUrl: '/audio' })
await Promise.resolve()
await Promise.resolve()
expect(fetchAuthenticatedBlob).not.toHaveBeenCalled()
})
})

View File

@ -497,6 +497,7 @@ export default {
},
},
},
ttsFailed: 'Speech synthesis or playback failed',
ttsPlay: 'Read Aloud',
ttsStop: 'Stop Reading',
conversations: 'Conversations',

View File

@ -497,6 +497,7 @@ export default {
},
},
},
ttsFailed: '语音合成或播放失败',
ttsPlay: '朗读',
ttsStop: '停止朗读',
conversations: '会话列表',