mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-11-02 14:11:49 +08:00
fix(tts): restore Edge and DashScope read-aloud (#646)
This commit is contained in:
taobig
parent
9efad1a910
commit
894c945fab
@ -53,6 +53,20 @@
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- CosyVoice uses the native WebSocket API, not OpenAI audio/speech. -->
|
||||
<dependency>
|
||||
<groupId>com.alibaba</groupId>
|
||||
<artifactId>dashscope-sdk-java</artifactId>
|
||||
<version>2.22.2</version>
|
||||
<exclusions>
|
||||
<!-- Spring Boot already provides Logback. -->
|
||||
<exclusion>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-simple</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
<!-- ===== Spring AI Alibaba Graph Core (StateGraph workflow engine) ===== -->
|
||||
<dependency>
|
||||
<groupId>com.alibaba.cloud.ai</groupId>
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
package vip.mate.tts.provider;
|
||||
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.node.ObjectNode;
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam;
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat;
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer;
|
||||
import java.nio.ByteBuffer;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
@ -17,11 +17,11 @@ import vip.mate.tts.TtsResult;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* DashScope TTS Provider — 使用 CosyVoice（OpenAI 兼容接口）
|
||||
* DashScope TTS Provider — 使用 CosyVoice 原生 WebSocket SDK
|
||||
* <p>
|
||||
* 同步模式，直接返回音频流。
|
||||
* 复用已有的 DashScope LLM provider 的 API Key。
|
||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-openai-compatible
|
||||
* API 文档: https://help.aliyun.com/zh/model-studio/developer-reference/cosyvoice-java-sdk
|
||||
*
|
||||
* @author MateClaw Team
|
||||
*/
|
||||
@ -31,11 +31,9 @@ import java.util.List;
|
||||
public class DashScopeTtsProvider implements TtsProvider {
|
||||
|
||||
private final ModelProviderService modelProviderService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
|
||||
private static final String DEFAULT_MODEL = "cosyvoice-v2";
|
||||
private static final String DEFAULT_VOICE = "longxiaochun";
|
||||
private static final String DEFAULT_VOICE = "longxiaochun_v2";
|
||||
|
||||
@Override
|
||||
public String id() {
|
||||
@ -69,9 +67,8 @@ public class DashScopeTtsProvider implements TtsProvider {
|
||||
@Override
|
||||
public List<String> availableVoices() {
|
||||
return List.of(
|
||||
"longxiaochun", "longxiaoxia", "longlaotie", "longshu",
|
||||
"longhua", "longshuo", "longjielidou", "longmiao",
|
||||
"longyue", "longfei", "longtong", "longxiang"
|
||||
"longxiaochun_v2", "longxiaoxia_v2", "longshu_v2",
|
||||
"longhua_v2", "longwan_v2", "longcheng_v2"
|
||||
);
|
||||
}
|
||||
|
||||
@ -83,7 +80,7 @@ public class DashScopeTtsProvider implements TtsProvider {
|
||||
@Override
|
||||
public TtsResult synthesize(TtsRequest request, SystemSettingsDTO config) {
|
||||
String apiKey = getDashScopeApiKey();
|
||||
if (apiKey == null) {
|
||||
if (apiKey == null || apiKey.isBlank()) {
|
||||
return TtsResult.failure("DashScope API Key 未配置");
|
||||
}
|
||||
|
||||
@ -93,36 +90,33 @@ public class DashScopeTtsProvider implements TtsProvider {
|
||||
String voice = request.getVoice() != null && !request.getVoice().isBlank()
|
||||
? request.getVoice() : DEFAULT_VOICE;
|
||||
|
||||
ObjectNode body = objectMapper.createObjectNode();
|
||||
body.put("model", model);
|
||||
body.put("input", request.getText());
|
||||
body.put("voice", voice);
|
||||
body.put("response_format", "mp3");
|
||||
if (request.getSpeed() != null && request.getSpeed() != 1.0) {
|
||||
body.put("speed", request.getSpeed());
|
||||
if ((request.getVoice() == null || request.getVoice().isBlank())
|
||||
&& availableVoices().contains(config.getTtsDefaultVoice() == null ? "" : config.getTtsDefaultVoice())) {
|
||||
voice = config.getTtsDefaultVoice();
|
||||
}
|
||||
|
||||
String endpoint = BASE_URL + "/audio/speech";
|
||||
HttpResponse response = HttpRequest.post(endpoint)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.header("Content-Type", "application/json")
|
||||
.body(body.toString())
|
||||
.timeout(60_000)
|
||||
.execute();
|
||||
|
||||
if (response.getStatus() == 200) {
|
||||
byte[] audioData = response.bodyBytes();
|
||||
SpeechSynthesisParam param = SpeechSynthesisParam.builder()
|
||||
.apiKey(apiKey)
|
||||
.model(model)
|
||||
.voice(voice)
|
||||
.format(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS)
|
||||
.speechRate(request.getSpeed() == null ? 1.0f : request.getSpeed().floatValue())
|
||||
.build();
|
||||
SpeechSynthesizer synthesizer = new SpeechSynthesizer(param, null);
|
||||
try {
|
||||
ByteBuffer audio = synthesizer.call(request.getText(), 60_000);
|
||||
if (audio == null || !audio.hasRemaining()) {
|
||||
return TtsResult.failure("DashScope TTS 未返回音频数据");
|
||||
}
|
||||
byte[] audioData = new byte[audio.remaining()];
|
||||
audio.get(audioData);
|
||||
log.info("[DashScope TTS] Synthesized {} bytes (model={}, voice={})", audioData.length, model, voice);
|
||||
return TtsResult.success(audioData, "audio/mpeg", "mp3");
|
||||
} else {
|
||||
String errBody = response.body();
|
||||
log.warn("[DashScope TTS] Failed: HTTP {} - {}", response.getStatus(), errBody);
|
||||
return TtsResult.failure(TtsResponseDiagnostics.failureMessage(
|
||||
"DashScope TTS", endpoint, response.getStatus(), errBody));
|
||||
} finally {
|
||||
synthesizer.getDuplexApi().close(1000, "bye");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[DashScope TTS] Error: {}", e.getMessage(), e);
|
||||
return TtsResult.failure("DashScope TTS 异常: " + e.getMessage());
|
||||
return TtsResult.failure("DashScope TTS 异常: " + TtsResponseDiagnostics.snippet(e.getMessage()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -13,6 +13,13 @@ import java.net.http.HttpClient;
|
||||
import java.net.http.WebSocket;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.HexFormat;
|
||||
import java.util.Locale;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
@ -31,8 +38,9 @@ import java.util.concurrent.TimeUnit;
|
||||
@Component
|
||||
public class EdgeTtsProvider implements TtsProvider {
|
||||
|
||||
private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud";
|
||||
private static final String WS_URL = "wss://speech.platform.bing.com/consumer/speech/synthesize/readaloud/edge/v1";
|
||||
private static final String TRUSTED_CLIENT_TOKEN = "6A5AA1D4EAFF4E9FB37E23D68491D6F4";
|
||||
private static final String EDGE_VERSION = "143.0.3650.75";
|
||||
private static final String DEFAULT_VOICE_ZH = "zh-CN-XiaoxiaoNeural";
|
||||
private static final String DEFAULT_VOICE_EN = "en-US-MichelleNeural";
|
||||
private static final String OUTPUT_FORMAT = "audio-24khz-48kbitrate-mono-mp3";
|
||||
@ -99,7 +107,9 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
private byte[] synthesizeViaWebSocket(String text, String voice, String rate) throws Exception {
|
||||
String requestId = UUID.randomUUID().toString().replace("-", "");
|
||||
String wsUrl = WS_URL + "?TrustedClientToken=" + TRUSTED_CLIENT_TOKEN
|
||||
+ "&ConnectionId=" + requestId;
|
||||
+ "&ConnectionId=" + requestId
|
||||
+ "&Sec-MS-GEC=" + securityToken(Instant.now())
|
||||
+ "&Sec-MS-GEC-Version=1-" + EDGE_VERSION;
|
||||
|
||||
ByteArrayOutputStream audioBuffer = new ByteArrayOutputStream();
|
||||
CompletableFuture<byte[]> resultFuture = new CompletableFuture<>();
|
||||
@ -110,9 +120,13 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
|
||||
WebSocket ws = client.newWebSocketBuilder()
|
||||
.header("Origin", "chrome-extension://jdiccldimpdaibmpdkjnbmckianbfold")
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36")
|
||||
.connectTimeout(Duration.ofSeconds(10))
|
||||
.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
||||
+ " (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36 Edg/143.0.0.0")
|
||||
.header("Cookie", "muid=" + UUID.randomUUID().toString().replace("-", "").toUpperCase(Locale.ROOT) + ";")
|
||||
.buildAsync(URI.create(wsUrl), new WebSocket.Listener() {
|
||||
private final StringBuilder textBuffer = new StringBuilder();
|
||||
private final ByteArrayOutputStream binaryBuffer = new ByteArrayOutputStream();
|
||||
|
||||
@Override
|
||||
public void onOpen(WebSocket webSocket) {
|
||||
@ -135,14 +149,22 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
|
||||
@Override
|
||||
public CompletionStage<?> onBinary(WebSocket webSocket, ByteBuffer data, boolean last) {
|
||||
// 二进制帧：前 2 字节是 header 长度（大端），跳过 header
|
||||
byte[] bytes = new byte[data.remaining()];
|
||||
data.get(bytes);
|
||||
// 查找 "Path:audio\r\n" 后的音频数据
|
||||
byte[] fragment = new byte[data.remaining()];
|
||||
data.get(fragment);
|
||||
binaryBuffer.writeBytes(fragment);
|
||||
if (last) {
|
||||
byte[] bytes = binaryBuffer.toByteArray();
|
||||
binaryBuffer.reset();
|
||||
int headerEnd = findHeaderEnd(bytes);
|
||||
if (headerEnd >= 0 && headerEnd < bytes.length) {
|
||||
if (headerEnd >= 2 && headerEnd <= bytes.length) {
|
||||
String headers = new String(bytes, 2, headerEnd - 2, StandardCharsets.UTF_8);
|
||||
if (headers.contains("Path:audio\r\n")) {
|
||||
audioBuffer.write(bytes, headerEnd, bytes.length - headerEnd);
|
||||
}
|
||||
} else {
|
||||
resultFuture.completeExceptionally(new IllegalStateException("Invalid Edge TTS audio frame"));
|
||||
}
|
||||
}
|
||||
webSocket.request(1);
|
||||
return null;
|
||||
}
|
||||
@ -150,7 +172,8 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
@Override
|
||||
public CompletionStage<?> onClose(WebSocket webSocket, int statusCode, String reason) {
|
||||
if (!resultFuture.isDone()) {
|
||||
resultFuture.complete(audioBuffer.toByteArray());
|
||||
resultFuture.completeExceptionally(new IllegalStateException(
|
||||
"Edge TTS connection closed before turn.end: " + statusCode));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@ -163,13 +186,17 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
}
|
||||
}).join();
|
||||
|
||||
try {
|
||||
String timestamp = DateTimeFormatter.ofPattern(
|
||||
"EEE MMM dd yyyy HH:mm:ss 'GMT+0000 (Coordinated Universal Time)'", Locale.US)
|
||||
.withZone(ZoneOffset.UTC).format(Instant.now());
|
||||
// 发送配置消息
|
||||
String configMsg = "Content-Type:application/json; charset=utf-8\r\n"
|
||||
String configMsg = "X-Timestamp:" + timestamp + "\r\nContent-Type:application/json; charset=utf-8\r\n"
|
||||
+ "Path:speech.config\r\n\r\n"
|
||||
+ "{\"context\":{\"synthesis\":{\"audio\":{\"metadataoptions\":{"
|
||||
+ "\"sentenceBoundaryEnabled\":false,\"wordBoundaryEnabled\":false},"
|
||||
+ "\"outputFormat\":\"" + OUTPUT_FORMAT + "\"}}}}\r\n";
|
||||
ws.sendText(configMsg, true);
|
||||
ws.sendText(configMsg, true).join();
|
||||
|
||||
// 发送 SSML
|
||||
String escapedText = escapeXml(text);
|
||||
@ -180,12 +207,25 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
+ "</prosody></voice></speak>";
|
||||
|
||||
String ssmlMsg = "X-RequestId:" + requestId + "\r\n"
|
||||
+ "X-Timestamp:" + timestamp + "Z\r\n"
|
||||
+ "Content-Type:application/ssml+xml\r\n"
|
||||
+ "Path:ssml\r\n\r\n" + ssml;
|
||||
ws.sendText(ssmlMsg, true);
|
||||
ws.sendText(ssmlMsg, true).join();
|
||||
|
||||
// 等待结果（最多 60 秒）
|
||||
return resultFuture.get(60, TimeUnit.SECONDS);
|
||||
} finally {
|
||||
ws.abort();
|
||||
}
|
||||
}
|
||||
|
||||
// Edge validates a SHA-256 of the five-minute Windows FILETIME bucket and client token.
|
||||
static String securityToken(Instant now) throws Exception {
|
||||
long seconds = now.getEpochSecond() + 11_644_473_600L;
|
||||
long ticks = (seconds - seconds % 300) * 10_000_000L;
|
||||
byte[] digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest((ticks + TRUSTED_CLIENT_TOKEN).getBytes(StandardCharsets.US_ASCII));
|
||||
return HexFormat.of().withUpperCase().formatHex(digest);
|
||||
}
|
||||
|
||||
private int findHeaderEnd(byte[] data) {
|
||||
@ -200,7 +240,7 @@ public class EdgeTtsProvider implements TtsProvider {
|
||||
return requestedVoice;
|
||||
}
|
||||
String configVoice = config.getTtsDefaultVoice();
|
||||
if (configVoice != null && !configVoice.isBlank()) {
|
||||
if (configVoice != null && availableVoices().contains(configVoice)) {
|
||||
return configVoice;
|
||||
}
|
||||
// 自动语言检测：CJK 字符比例 > 30% 使用中文语音
|
||||
|
||||
@ -0,0 +1,78 @@
|
||||
package vip.mate.tts.provider;
|
||||
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesizer;
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisParam;
|
||||
import com.alibaba.dashscope.audio.ttsv2.SpeechSynthesisAudioFormat;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.tts.TtsRequest;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class DashScopeTtsProviderTest {
|
||||
@Test
|
||||
void defaultVoiceMatchesCosyVoiceV2() {
|
||||
var provider = new DashScopeTtsProvider(mock(ModelProviderService.class));
|
||||
assertEquals("longxiaochun_v2", provider.defaultVoice());
|
||||
assertTrue(provider.availableVoices().contains(provider.defaultVoice()));
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesNativeSdkAndClosesConnection() {
|
||||
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
|
||||
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
|
||||
AtomicReference<SpeechSynthesisParam> parameters = new AtomicReference<>();
|
||||
try (var synths = mockConstruction(SpeechSynthesizer.class,
|
||||
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) -> {
|
||||
parameters.set((SpeechSynthesisParam) context.arguments().get(0));
|
||||
doReturn(ByteBuffer.wrap(new byte[]{1, 2, 3})).when(synth).call("Hello", 60000);
|
||||
})) {
|
||||
// A default voice from another provider must not break DashScope fallback.
|
||||
var config = new SystemSettingsDTO();
|
||||
config.setTtsDefaultVoice("zh-CN-XiaoxiaoNeural");
|
||||
var result = new DashScopeTtsProvider(service).synthesize(
|
||||
TtsRequest.builder().text("Hello").speed(1.5).build(), config);
|
||||
assertTrue(result.isSuccess(), result.getErrorMessage());
|
||||
assertArrayEquals(new byte[]{1, 2, 3}, result.getAudioData());
|
||||
assertEquals("cosyvoice-v2", parameters.get().getModel());
|
||||
assertEquals("longxiaochun_v2", parameters.get().getVoice());
|
||||
assertEquals(1.5f, parameters.get().getSpeechRate());
|
||||
assertEquals(SpeechSynthesisAudioFormat.MP3_24000HZ_MONO_256KBPS, parameters.get().getFormat());
|
||||
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
|
||||
}
|
||||
}
|
||||
@Test
|
||||
void closesConnectionWhenSynthesisFails() {
|
||||
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
|
||||
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
|
||||
try (var synths = mockConstruction(SpeechSynthesizer.class,
|
||||
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) ->
|
||||
doThrow(new IllegalStateException("invalid API key")).when(synth).call("Hello", 60000))) {
|
||||
var result = new DashScopeTtsProvider(service).synthesize(
|
||||
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
|
||||
assertFalse(result.isSuccess());
|
||||
assertTrue(result.getErrorMessage().contains("invalid API key"));
|
||||
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void rejectsEmptyAudio() {
|
||||
var service = mock(ModelProviderService.class, RETURNS_DEEP_STUBS);
|
||||
when(service.getProviderConfig("dashscope").getApiKey()).thenReturn("test-key");
|
||||
try (var synths = mockConstruction(SpeechSynthesizer.class,
|
||||
withSettings().defaultAnswer(RETURNS_DEEP_STUBS), (synth, context) ->
|
||||
doReturn(ByteBuffer.allocate(0)).when(synth).call("Hello", 60000))) {
|
||||
var result = new DashScopeTtsProvider(service).synthesize(
|
||||
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
|
||||
assertFalse(result.isSuccess());
|
||||
verify(synths.constructed().getFirst().getDuplexApi()).close(1000, "bye");
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
package vip.mate.tts.provider;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
import vip.mate.tts.TtsRequest;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.WebSocket;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.Arrays;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.*;
|
||||
import static org.mockito.ArgumentMatchers.*;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
class EdgeTtsProviderTest {
|
||||
@Test
|
||||
void hashesWindowsFileTimeInFiveMinuteBuckets() throws Exception {
|
||||
assertEquals("7ECB79D14E3AA576D2D79E6D487A1388156D91E614B1BE11C64226A29BC8DD8C",
|
||||
EdgeTtsProvider.securityToken(java.time.Instant.EPOCH));
|
||||
assertEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH),
|
||||
EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(299)));
|
||||
assertNotEquals(EdgeTtsProvider.securityToken(java.time.Instant.EPOCH),
|
||||
EdgeTtsProvider.securityToken(java.time.Instant.ofEpochSecond(300)));
|
||||
}
|
||||
|
||||
@Test
|
||||
void usesCurrentHandshakeAndPreservesFragmentedAudio() {
|
||||
HttpClient.Builder httpBuilder = mock(HttpClient.Builder.class, RETURNS_SELF);
|
||||
HttpClient client = mock(HttpClient.class);
|
||||
WebSocket.Builder wsBuilder = mock(WebSocket.Builder.class, RETURNS_SELF);
|
||||
WebSocket socket = mock(WebSocket.class);
|
||||
when(httpBuilder.build()).thenReturn(client);
|
||||
when(client.newWebSocketBuilder()).thenReturn(wsBuilder);
|
||||
AtomicReference<URI> endpoint = new AtomicReference<>();
|
||||
AtomicReference<WebSocket.Listener> listener = new AtomicReference<>();
|
||||
when(wsBuilder.buildAsync(any(), any())).thenAnswer(inv -> {
|
||||
endpoint.set(inv.getArgument(0));
|
||||
listener.set(inv.getArgument(1));
|
||||
return CompletableFuture.completedFuture(socket);
|
||||
});
|
||||
when(socket.sendText(anyString(), eq(true))).thenAnswer(inv -> {
|
||||
if (inv.<String>getArgument(0).contains("Path:ssml")) {
|
||||
byte[] headers = "Path:audio\r\nContent-Type:audio/mpeg\r\n".getBytes(StandardCharsets.UTF_8);
|
||||
byte[] packet = ByteBuffer.allocate(2 + headers.length + 4)
|
||||
.putShort((short) headers.length).put(headers).put(new byte[]{1, 2, 3, 4}).array();
|
||||
listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 0, 1)), false);
|
||||
listener.get().onBinary(socket, ByteBuffer.wrap(Arrays.copyOfRange(packet, 1, packet.length)), true);
|
||||
listener.get().onText(socket, "Path:turn.end\r\n", true);
|
||||
}
|
||||
return CompletableFuture.completedFuture(socket);
|
||||
});
|
||||
try (var clients = mockStatic(HttpClient.class)) {
|
||||
clients.when(HttpClient::newBuilder).thenReturn(httpBuilder);
|
||||
var result = new EdgeTtsProvider().synthesize(
|
||||
TtsRequest.builder().text("Hello").build(), new SystemSettingsDTO());
|
||||
assertEquals("/consumer/speech/synthesize/readaloud/edge/v1", endpoint.get().getPath());
|
||||
assertTrue(endpoint.get().getQuery().matches(".*Sec-MS-GEC=[A-F0-9]{64}.*"));
|
||||
assertTrue(endpoint.get().getQuery().contains("Sec-MS-GEC-Version=1-"));
|
||||
assertTrue(result.isSuccess(), result.getErrorMessage());
|
||||
assertArrayEquals(new byte[]{1, 2, 3, 4}, result.getAudioData());
|
||||
verify(socket).abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -587,7 +587,7 @@ import { buildGeneratedFileNameMap, linkifyGeneratedFileUrls } from '@/utils/gen
|
||||
import { ensureModelViewer } from '@/utils/lazyModelViewer'
|
||||
import { useAuthenticatedAttachment } from '@/composables/useAuthenticatedAttachment'
|
||||
import { useToolLabel } from '@/composables/useToolLabel'
|
||||
import { http } from '@/api'
|
||||
import { http, fetchAuthenticatedBlob } from '@/api'
|
||||
import { copyToClipboard } from '@/utils/clipboard'
|
||||
import TypingCursor from './TypingCursor.vue'
|
||||
import { previewKindOf } from './preview/previewKind'
|
||||
@ -853,59 +853,59 @@ function copyMessage() {
|
||||
// --- TTS 朗读 ---
|
||||
const ttsState = ref<'idle' | 'loading' | 'playing'>('idle')
|
||||
let ttsAudio: HTMLAudioElement | null = null
|
||||
let ttsBlobUrl: string | null = null
|
||||
let ttsRequestVersion = 0
|
||||
|
||||
function stopTts() {
|
||||
ttsAudio?.pause()
|
||||
if (ttsAudio) { ttsAudio.onended = null; ttsAudio.onerror = null }
|
||||
ttsAudio = null
|
||||
if (ttsBlobUrl) URL.revokeObjectURL(ttsBlobUrl)
|
||||
ttsBlobUrl = null
|
||||
ttsState.value = 'idle'
|
||||
}
|
||||
|
||||
async function handleTts() {
|
||||
if (ttsState.value === 'playing') {
|
||||
// 停止播放
|
||||
ttsAudio?.pause()
|
||||
ttsAudio = null
|
||||
ttsState.value = 'idle'
|
||||
stopTts()
|
||||
return
|
||||
}
|
||||
|
||||
if (ttsState.value === 'loading') return
|
||||
const text = displayContent.value || props.message.content || ''
|
||||
if (!text) return
|
||||
|
||||
const conversationId = props.message.conversationId
|
||||
if (!conversationId) return
|
||||
if (!text || !conversationId) return
|
||||
|
||||
const version = ++ttsRequestVersion
|
||||
ttsState.value = 'loading'
|
||||
try {
|
||||
const res: any = await http.post('/tts/synthesize', {
|
||||
conversationId,
|
||||
text,
|
||||
})
|
||||
if (res.data?.success && res.data?.audioUrl) {
|
||||
// 通过认证 fetch 获取音频 blob
|
||||
const audioRes = await fetch(res.data.audioUrl, {
|
||||
headers: { Authorization: `Bearer ${localStorage.getItem('token') || ''}` },
|
||||
})
|
||||
const blob = await audioRes.blob()
|
||||
const blobUrl = URL.createObjectURL(blob)
|
||||
ttsAudio = new Audio(blobUrl)
|
||||
ttsAudio.onended = () => {
|
||||
ttsState.value = 'idle'
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
ttsAudio = null
|
||||
// The shared interceptor already unwraps the raw TTS response body.
|
||||
const result: any = await http.post('/tts/synthesize', { conversationId, text }, { timeout: 240_000 })
|
||||
if (version !== ttsRequestVersion) return
|
||||
if (!result?.success || !result.audioUrl) {
|
||||
throw new Error(result?.error || t('chat.ttsFailed'))
|
||||
}
|
||||
const blob = await fetchAuthenticatedBlob(result.audioUrl)
|
||||
if (version !== ttsRequestVersion) return
|
||||
ttsBlobUrl = URL.createObjectURL(blob)
|
||||
ttsAudio = new Audio(ttsBlobUrl)
|
||||
ttsAudio.onended = stopTts
|
||||
ttsAudio.onerror = () => {
|
||||
ttsState.value = 'idle'
|
||||
URL.revokeObjectURL(blobUrl)
|
||||
ttsAudio = null
|
||||
stopTts()
|
||||
mcToast.error(t('chat.ttsFailed'))
|
||||
}
|
||||
ttsState.value = 'playing'
|
||||
await ttsAudio.play()
|
||||
} else {
|
||||
ttsState.value = 'idle'
|
||||
}
|
||||
} catch {
|
||||
ttsState.value = 'idle'
|
||||
if (version === ttsRequestVersion && ttsAudio) ttsState.value = 'playing'
|
||||
} catch (error) {
|
||||
if (version !== ttsRequestVersion) return
|
||||
stopTts()
|
||||
mcToast.error(error instanceof Error ? error.message : t('chat.ttsFailed'))
|
||||
}
|
||||
}
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
if (copyTimer) clearTimeout(copyTimer)
|
||||
if (ttsAudio) { ttsAudio.pause(); ttsAudio = null }
|
||||
++ttsRequestVersion
|
||||
stopTts()
|
||||
revokeAll()
|
||||
})
|
||||
|
||||
|
||||
@ -29,7 +29,7 @@ vi.mock('@/composables/useMcToast', () => ({
|
||||
vi.mock('@/composables/useToolLabel', () => ({
|
||||
useToolLabel: () => ({ getToolLabel: (name: string) => name }),
|
||||
}))
|
||||
vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() } }))
|
||||
vi.mock('@/api', () => ({ http: { get: vi.fn(), post: vi.fn() }, fetchAuthenticatedBlob: vi.fn() }))
|
||||
vi.mock('@/utils/clipboard', () => ({ copyToClipboard: vi.fn() }))
|
||||
vi.mock('@/utils/generatedFileLinks', () => ({
|
||||
buildGeneratedFileNameMap: vi.fn(() => new Map()),
|
||||
@ -58,6 +58,8 @@ vi.mock('../UserMessageContent.vue', () => ({
|
||||
}))
|
||||
|
||||
import MessageBubble from '../MessageBubble.vue'
|
||||
import { http, fetchAuthenticatedBlob } from '@/api'
|
||||
import { mcToast } from '@/composables/useMcToast'
|
||||
|
||||
const apps: Array<ReturnType<typeof createApp>> = []
|
||||
|
||||
@ -92,6 +94,9 @@ function mountMessage(message: Message, locale = 'zh-CN') {
|
||||
afterEach(() => {
|
||||
apps.splice(0).forEach(app => app.unmount())
|
||||
document.body.innerHTML = ''
|
||||
vi.restoreAllMocks()
|
||||
vi.unstubAllGlobals()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
describe('MessageBubble stop indicator', () => {
|
||||
@ -130,3 +135,69 @@ describe('MessageBubble stop indicator', () => {
|
||||
expect(host.querySelector('.stopped-indicator--interrupted')).not.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
describe('MessageBubble TTS (#646)', () => {
|
||||
function clickReadAloud() {
|
||||
const host = mountMessage({ id: 'tts', conversationId: 'conv', role: 'assistant',
|
||||
content: 'Hello', contentParts: [], status: 'completed' } as Message)
|
||||
const button = host.querySelector<HTMLButtonElement>('[title="chat.ttsPlay"]')!
|
||||
expect(button).not.toBeNull()
|
||||
button.click()
|
||||
return button
|
||||
}
|
||||
|
||||
it('plays the unwrapped synthesis response and releases audio on stop', async () => {
|
||||
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/api/v1/chat/files/conv/tts.mp3' } as never)
|
||||
vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio']))
|
||||
const play = vi.fn().mockResolvedValue(undefined)
|
||||
const pause = vi.fn()
|
||||
vi.stubGlobal('Audio', class { play = play; pause = pause })
|
||||
vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:tts')
|
||||
const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {})
|
||||
const button = clickReadAloud()
|
||||
await vi.waitFor(() => expect(play).toHaveBeenCalledOnce())
|
||||
expect(fetchAuthenticatedBlob).toHaveBeenCalledWith('/api/v1/chat/files/conv/tts.mp3')
|
||||
button.click()
|
||||
expect(pause).toHaveBeenCalledOnce()
|
||||
expect(revoke).toHaveBeenCalledWith('blob:tts')
|
||||
})
|
||||
|
||||
it('shows backend synthesis errors instead of silently doing nothing', async () => {
|
||||
vi.mocked(http.post).mockResolvedValue({ success: false, error: 'DashScope: invalid voice' } as never)
|
||||
clickReadAloud()
|
||||
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('DashScope: invalid voice'))
|
||||
})
|
||||
|
||||
it('shows audio download failures and resets the loading state', async () => {
|
||||
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never)
|
||||
vi.mocked(fetchAuthenticatedBlob).mockRejectedValue(new Error('Fetch failed: 403'))
|
||||
const button = clickReadAloud()
|
||||
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Fetch failed: 403'))
|
||||
expect(button.disabled).toBe(false)
|
||||
})
|
||||
it('releases the audio URL when playback is rejected', async () => {
|
||||
vi.mocked(http.post).mockResolvedValue({ success: true, audioUrl: '/audio' } as never)
|
||||
vi.mocked(fetchAuthenticatedBlob).mockResolvedValue(new Blob(['audio']))
|
||||
const pause = vi.fn()
|
||||
vi.stubGlobal('Audio', class { pause = pause; play = vi.fn().mockRejectedValue(new Error('Playback blocked')) })
|
||||
vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:rejected')
|
||||
const revoke = vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {})
|
||||
clickReadAloud()
|
||||
await vi.waitFor(() => expect(mcToast.error).toHaveBeenCalledWith('Playback blocked'))
|
||||
expect(pause).toHaveBeenCalledOnce()
|
||||
expect(revoke).toHaveBeenCalledWith('blob:rejected')
|
||||
})
|
||||
|
||||
it('does not start playback when the component unmounts during synthesis', async () => {
|
||||
let finish!: (value: any) => void
|
||||
vi.mocked(http.post).mockReturnValue(new Promise(resolve => { finish = resolve }))
|
||||
clickReadAloud()
|
||||
apps.pop()!.unmount()
|
||||
finish({ success: true, audioUrl: '/audio' })
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
expect(fetchAuthenticatedBlob).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
})
|
||||
|
||||
@ -497,6 +497,7 @@ export default {
|
||||
},
|
||||
},
|
||||
},
|
||||
ttsFailed: 'Speech synthesis or playback failed',
|
||||
ttsPlay: 'Read Aloud',
|
||||
ttsStop: 'Stop Reading',
|
||||
conversations: 'Conversations',
|
||||
|
||||
@ -497,6 +497,7 @@ export default {
|
||||
},
|
||||
},
|
||||
},
|
||||
ttsFailed: '语音合成或播放失败',
|
||||
ttsPlay: '朗读',
|
||||
ttsStop: '停止朗读',
|
||||
conversations: '会话列表',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user