mirror of
https://gitee.com/mateos/mateclaw.git
synced 2026-09-13 03:13:41 +08:00
feat(stt): DashScope realtime voice + language-aware routing + TalkMode polish
- DashScope paraformer-realtime-v2 WebSocket streaming - Language-aware provider routing: Whisper for English, Paraformer for Chinese - PCM WAV recording replaces WebM (provider filename bug + diagnostics) - TalkMode push-to-talk fixes (audio drop, WS connecting race) - Vite dev proxy WebSocket upgrade fix - WebSocket binary buffer 8KB → 8MB (Tomcat default truncated voice clips) - Audio chunk pacing at 100ms (DashScope returned 0 chars otherwise) - Resolved language hint propagation + raw frame logging - V46 seed idempotency fix on UI-toggled STT row - Diagnostic cleanup after debugging session
This commit is contained in:
matevip
parent
4d7c6593c4
commit
b7c911f01d
@ -118,7 +118,10 @@ public class TalkModeWebSocketHandler extends AbstractWebSocketHandler {
|
||||
sendJson(session, Map.of("type", "state", "state", "processing"));
|
||||
|
||||
// 2. STT: 音频转文字
|
||||
Map<String, Object> sttResult = sttService.transcribe(audioData, "audio.webm", "audio/webm", null);
|
||||
// 前端用 WavRecorder（Web Audio API + 手写 PCM WAV 编码）— 见
|
||||
// mateclaw-ui/src/utils/wavEncoder.ts. WebM/Opus 被 DashScope
|
||||
// Paraformer 拒收，WAV 是所有 STT provider 都接受的最大公约数。
|
||||
Map<String, Object> sttResult = sttService.transcribe(audioData, "audio.wav", "audio/wav", null);
|
||||
if (!Boolean.TRUE.equals(sttResult.get("success"))) {
|
||||
sendJson(session, Map.of("type", "error", "message", "Speech recognition failed: " + sttResult.get("error")));
|
||||
sendJson(session, Map.of("type", "state", "state", "idle"));
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
package vip.mate.config;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.web.socket.config.annotation.EnableWebSocket;
|
||||
import org.springframework.web.socket.config.annotation.WebSocketConfigurer;
|
||||
import org.springframework.web.socket.config.annotation.WebSocketHandlerRegistry;
|
||||
import org.springframework.web.socket.server.standard.ServletServerContainerFactoryBean;
|
||||
import vip.mate.channel.web.TalkModeWebSocketHandler;
|
||||
|
||||
/**
|
||||
@ -19,6 +21,19 @@ import vip.mate.channel.web.TalkModeWebSocketHandler;
|
||||
@RequiredArgsConstructor
|
||||
public class WebSocketConfig implements WebSocketConfigurer {
|
||||
|
||||
/**
|
||||
* Max binary frame the TalkMode WebSocket accepts. Recordings come over
|
||||
* as 16 kHz / 16-bit / mono PCM WAV — roughly 32 KB per second of audio.
|
||||
* 8 MB headroom covers ~4 minutes of speech, which is more than any
|
||||
* realistic single-turn voice message. Default is 8 KB which used to
|
||||
* crash the connection (CloseStatus 1009 "Message too big") on every
|
||||
* non-trivial recording.
|
||||
*/
|
||||
private static final int MAX_BINARY_BUFFER_BYTES = 8 * 1024 * 1024;
|
||||
|
||||
/** Text frames stay reasonably small (init / state / transcript JSON). */
|
||||
private static final int MAX_TEXT_BUFFER_BYTES = 64 * 1024;
|
||||
|
||||
private final TalkModeWebSocketHandler talkModeHandler;
|
||||
|
||||
@Override
|
||||
@ -26,4 +41,21 @@ public class WebSocketConfig implements WebSocketConfigurer {
|
||||
registry.addHandler(talkModeHandler, "/api/v1/talk/ws")
|
||||
.setAllowedOrigins("*");
|
||||
}
|
||||
|
||||
/**
|
||||
* Tomcat-level buffer override. Without this Spring's
|
||||
* {@link org.springframework.web.socket.handler.AbstractWebSocketHandler}
|
||||
* still buffers the whole binary message before dispatching to the handler
|
||||
* — capped at 8 KB by default — and a single voice clip blows past that
|
||||
* limit on the very first send. Bumping the binary buffer is simpler than
|
||||
* implementing partial-message handling, given the bounded size of
|
||||
* speech-to-text clips. See discussion in the V46 STT bug-fix series.
|
||||
*/
|
||||
@Bean
|
||||
public ServletServerContainerFactoryBean createWebSocketContainer() {
|
||||
ServletServerContainerFactoryBean container = new ServletServerContainerFactoryBean();
|
||||
container.setMaxBinaryMessageBufferSize(MAX_BINARY_BUFFER_BYTES);
|
||||
container.setMaxTextMessageBufferSize(MAX_TEXT_BUFFER_BYTES);
|
||||
return container;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,90 @@
|
||||
package vip.mate.stt;
|
||||
|
||||
import java.util.Locale;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Filename inference helper for the OpenAI Whisper STT path.
|
||||
*
|
||||
* <p>Whisper's {@code /v1/audio/transcriptions} endpoint reads the
|
||||
* <em>filename extension</em> on the multipart {@code file} part to infer
|
||||
* audio format; the {@code Content-Type} alone isn't enough because
|
||||
* Hutool's 3-arg {@code form(name, bytes, fileName)} overload deduces
|
||||
* the multipart Content-Type from the extension we pass. Hence this
|
||||
* helper picks an extension that matches the actual bytes.
|
||||
*
|
||||
* <p>Previous bug (pre-fix): both providers hardcoded {@code "audio.ogg"}
|
||||
* as the default filename even when the upstream content was WebM/Opus,
|
||||
* which DashScope's HTTP path then tried to decode as Ogg and 400'd.
|
||||
* That bug + DashScope's HTTP STT are both gone now (DashScope went to
|
||||
* WebSocket); this class survives because Whisper still cares.
|
||||
*/
|
||||
public final class AudioMimeTypes {
|
||||
|
||||
/** Fallback when neither contentType nor filename gives us a hint. */
|
||||
private static final String DEFAULT_EXTENSION = "wav";
|
||||
|
||||
/** content-type → conventional file extension. Ordered for documentation only. */
|
||||
private static final Map<String, String> CONTENT_TYPE_TO_EXTENSION = Map.ofEntries(
|
||||
Map.entry("audio/wav", "wav"),
|
||||
Map.entry("audio/wave", "wav"),
|
||||
Map.entry("audio/x-wav", "wav"),
|
||||
Map.entry("audio/mpeg", "mp3"),
|
||||
Map.entry("audio/mp3", "mp3"),
|
||||
Map.entry("audio/mp4", "m4a"),
|
||||
Map.entry("audio/m4a", "m4a"),
|
||||
Map.entry("audio/x-m4a", "m4a"),
|
||||
Map.entry("audio/aac", "aac"),
|
||||
Map.entry("audio/flac", "flac"),
|
||||
Map.entry("audio/ogg", "ogg"),
|
||||
Map.entry("audio/webm", "webm"),
|
||||
Map.entry("audio/amr", "amr"));
|
||||
|
||||
/** file extension → conventional content-type (the inverse, for filename-first cases). */
|
||||
private static final Map<String, String> EXTENSION_TO_CONTENT_TYPE = Map.ofEntries(
|
||||
Map.entry("wav", "audio/wav"),
|
||||
Map.entry("mp3", "audio/mpeg"),
|
||||
Map.entry("m4a", "audio/mp4"),
|
||||
Map.entry("mp4", "audio/mp4"),
|
||||
Map.entry("aac", "audio/aac"),
|
||||
Map.entry("flac", "audio/flac"),
|
||||
Map.entry("ogg", "audio/ogg"),
|
||||
Map.entry("webm", "audio/webm"),
|
||||
Map.entry("amr", "audio/amr"));
|
||||
|
||||
private AudioMimeTypes() {}
|
||||
|
||||
/**
|
||||
* Choose a filename for the upload. Order of precedence:
|
||||
* <ol>
|
||||
* <li>The caller-supplied filename, when it has a known audio extension.</li>
|
||||
* <li>A name synthesised from the content-type, e.g. {@code audio/mpeg → audio.mp3}.</li>
|
||||
* <li>{@code audio.wav} as a final fallback (WAV is universally accepted).</li>
|
||||
* </ol>
|
||||
*/
|
||||
public static String resolveFileName(String fileName, String contentType) {
|
||||
if (fileName != null && !fileName.isBlank() && extensionOf(fileName) != null) {
|
||||
return fileName;
|
||||
}
|
||||
String extension = extensionForContentType(contentType);
|
||||
return "audio." + (extension != null ? extension : DEFAULT_EXTENSION);
|
||||
}
|
||||
|
||||
/** Extract the lower-cased extension (without the dot), or null. Package-private for tests. */
|
||||
static String extensionOf(String fileName) {
|
||||
if (fileName == null) return null;
|
||||
int dot = fileName.lastIndexOf('.');
|
||||
if (dot < 0 || dot == fileName.length() - 1) return null;
|
||||
String ext = fileName.substring(dot + 1).toLowerCase(Locale.ROOT);
|
||||
return EXTENSION_TO_CONTENT_TYPE.containsKey(ext) ? ext : null;
|
||||
}
|
||||
|
||||
/** Map a content-type to its conventional extension, or null when unknown. */
|
||||
static String extensionForContentType(String contentType) {
|
||||
if (contentType == null) return null;
|
||||
// Strip parameters: "audio/webm; codecs=opus" → "audio/webm"
|
||||
int semi = contentType.indexOf(';');
|
||||
String base = (semi >= 0 ? contentType.substring(0, semi) : contentType).trim().toLowerCase(Locale.ROOT);
|
||||
return CONTENT_TYPE_TO_EXTENSION.get(base);
|
||||
}
|
||||
}
|
||||
@ -3,7 +3,16 @@ package vip.mate.stt;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
|
||||
/**
|
||||
* STT 语音识别提供商接口
|
||||
* STT 语音识别提供商接口.
|
||||
*
|
||||
* <p>Auto-detect ordering uses ascending priority (low number = preferred).
|
||||
* Most providers can use the default {@link #autoDetectOrder()} value, but
|
||||
* providers with strong language bias should override
|
||||
* {@link #autoDetectOrder(String)} so the registry picks the right primary
|
||||
* for the user's locale: OpenAI Whisper is the canonical English path,
|
||||
* DashScope Paraformer is the canonical Chinese path. Mixing them up at
|
||||
* dispatch time costs accuracy AND latency (the wrong primary tends to
|
||||
* produce garbage that the fallback can't easily compensate for).
|
||||
*/
|
||||
public interface SttProvider {
|
||||
String id();
|
||||
@ -12,6 +21,23 @@ public interface SttProvider {
|
||||
int autoDetectOrder();
|
||||
boolean isAvailable(SystemSettingsDTO config);
|
||||
|
||||
/**
|
||||
* Per-language priority hook. Default implementation returns the
|
||||
* language-agnostic {@link #autoDetectOrder()} value, so existing
|
||||
* providers stay backwards-compatible. Override when the provider has
|
||||
* a known language strength — e.g. OpenAI Whisper returns a smaller
|
||||
* number for {@code "en"} than for {@code "zh"} to win the auto-pick
|
||||
* for English users.
|
||||
*
|
||||
* @param language IETF / ISO-639 language hint, possibly {@code null}.
|
||||
* Implementations should match conservatively (prefix
|
||||
* match on {@code zh}, {@code en}, etc.) and gracefully
|
||||
* fall back to {@link #autoDetectOrder()} on unknown.
|
||||
*/
|
||||
default int autoDetectOrder(String language) {
|
||||
return autoDetectOrder();
|
||||
}
|
||||
|
||||
/**
|
||||
* 转写音频
|
||||
*/
|
||||
|
||||
@ -11,41 +11,85 @@ import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* STT 提供商注册表
|
||||
* STT 提供商注册表.
|
||||
*
|
||||
* <p>Provider selection is two-stage:
|
||||
* <ol>
|
||||
* <li>If the user pinned a specific provider in settings, that wins.</li>
|
||||
* <li>Otherwise sort all available providers by
|
||||
* {@link SttProvider#autoDetectOrder(String)} — the per-language
|
||||
* hook lets Whisper win on English while Paraformer wins on Chinese.
|
||||
* Falls back to the language-agnostic order when no language hint
|
||||
* is supplied.</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>The registry is constructed once at startup and the unsorted provider
|
||||
* list is held — sorting happens on demand because the order depends on the
|
||||
* incoming language hint.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class SttProviderRegistry {
|
||||
|
||||
private final List<SttProvider> sortedProviders;
|
||||
private final List<SttProvider> providers;
|
||||
private final Map<String, SttProvider> providerMap;
|
||||
|
||||
public SttProviderRegistry(List<SttProvider> providers) {
|
||||
this.sortedProviders = providers.stream()
|
||||
.sorted(Comparator.comparingInt(SttProvider::autoDetectOrder))
|
||||
.toList();
|
||||
this.providers = List.copyOf(providers);
|
||||
this.providerMap = providers.stream()
|
||||
.collect(Collectors.toMap(SttProvider::id, Function.identity()));
|
||||
log.info("注册 STT 提供商 {} 个: {}", sortedProviders.size(),
|
||||
sortedProviders.stream().map(p -> p.id() + "(order=" + p.autoDetectOrder() + ")").toList());
|
||||
log.info("注册 STT 提供商 {} 个: {}", providers.size(),
|
||||
providers.stream()
|
||||
.map(p -> p.id() + "(default-order=" + p.autoDetectOrder() + ")")
|
||||
.toList());
|
||||
}
|
||||
|
||||
/** Backwards-compatible no-language overload. Prefer the (config, language) form. */
|
||||
public SttProvider resolve(SystemSettingsDTO config) {
|
||||
return resolve(config, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the primary provider given the user's settings + a language hint.
|
||||
* Returns null when nothing is available — callers should treat that as
|
||||
* "no API key configured anywhere" and surface the actionable error.
|
||||
*/
|
||||
public SttProvider resolve(SystemSettingsDTO config, String language) {
|
||||
String configuredId = config.getSttProvider();
|
||||
if (configuredId != null && !configuredId.isBlank() && !"auto".equals(configuredId)) {
|
||||
SttProvider p = providerMap.get(configuredId);
|
||||
if (p != null && p.isAvailable(config)) return p;
|
||||
// Configured-but-unavailable falls through to auto so the user
|
||||
// still gets a result if any other provider has its key set.
|
||||
}
|
||||
for (SttProvider p : sortedProviders) {
|
||||
for (SttProvider p : sortedByLanguage(language)) {
|
||||
if (p.isAvailable(config)) return p;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Backwards-compatible no-language overload. */
|
||||
public List<SttProvider> fallbackCandidates(SystemSettingsDTO config, String excludeId) {
|
||||
return sortedProviders.stream()
|
||||
return fallbackCandidates(config, excludeId, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Available providers other than {@code excludeId}, ordered by their
|
||||
* priority for the given language. The fallback list always uses the
|
||||
* language-aware order — if Whisper failed on Chinese, Paraformer is
|
||||
* the right next-best, not whatever happened to come next in the
|
||||
* default order.
|
||||
*/
|
||||
public List<SttProvider> fallbackCandidates(SystemSettingsDTO config, String excludeId, String language) {
|
||||
return sortedByLanguage(language).stream()
|
||||
.filter(p -> !p.id().equals(excludeId))
|
||||
.filter(p -> p.isAvailable(config))
|
||||
.toList();
|
||||
}
|
||||
|
||||
private List<SttProvider> sortedByLanguage(String language) {
|
||||
return providers.stream()
|
||||
.sorted(Comparator.comparingInt(p -> p.autoDetectOrder(language)))
|
||||
.toList();
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,6 +26,15 @@ public class SttService {
|
||||
return Map.of("success", false, "error", "STT 功能未启用，请在系统设置中开启");
|
||||
}
|
||||
|
||||
// Per-call dispatch trace — without this the only signal that STT
|
||||
// is even being attempted is the eventual provider success/failure
|
||||
// log, which makes "no audio reached us" indistinguishable from
|
||||
// "audio reached us but provider rejected it".
|
||||
int bytes = audioData != null ? audioData.length : 0;
|
||||
log.info("[STT] dispatch bytes={} fileName={} contentType={} language={} provider={}",
|
||||
bytes, fileName, contentType, language,
|
||||
config.getSttProvider() != null ? config.getSttProvider() : "auto");
|
||||
|
||||
SttRequest request = SttRequest.builder()
|
||||
.audioData(audioData)
|
||||
.fileName(fileName)
|
||||
@ -45,26 +54,61 @@ public class SttService {
|
||||
}
|
||||
|
||||
private SttResult transcribeWithFallback(SttRequest request, SystemSettingsDTO config) {
|
||||
SttProvider primary = providerRegistry.resolve(config);
|
||||
if (primary == null) {
|
||||
return SttResult.failure("没有可用的 STT Provider，请检查配置");
|
||||
// Language hint resolution order:
|
||||
// 1. Caller-supplied request.language (explicit per-call override)
|
||||
// 2. UI language from system settings (zh-CN / en-US)
|
||||
// 3. null — registry falls back to the language-agnostic order
|
||||
// Both registry primary-pick and fallback-candidate ordering use the
|
||||
// same hint; without it Paraformer/Whisper would frequently swap
|
||||
// priorities mid-fallback for the same conversation.
|
||||
//
|
||||
// Critical: write the resolved hint BACK into the request, so
|
||||
// providers (especially DashScope's run-task language_hints field)
|
||||
// see it. Pre-fix, providers got null even though we routed by
|
||||
// zh-CN — DashScope's auto-detect would then sit through 2-3s of
|
||||
// Chinese audio and emit zero result-generated events.
|
||||
String languageHint = request.getLanguage();
|
||||
if (languageHint == null || languageHint.isBlank()) {
|
||||
languageHint = config.getLanguage();
|
||||
if (languageHint != null && !languageHint.isBlank()) {
|
||||
request.setLanguage(languageHint);
|
||||
}
|
||||
}
|
||||
SttProvider primary = providerRegistry.resolve(config, languageHint);
|
||||
if (primary == null) {
|
||||
// Most common cause: no provider has its API key configured. Tell
|
||||
// the operator that explicitly so they don't dig through provider
|
||||
// logs looking for the real reason.
|
||||
log.warn("[STT] no provider available — check DashScope / OpenAI API keys in 模型管理");
|
||||
return SttResult.failure("没有可用的 STT Provider，请在模型管理中配置 DashScope 或 OpenAI API Key");
|
||||
}
|
||||
log.info("[STT] primary provider resolved: {} (language={})", primary.id(), languageHint);
|
||||
|
||||
SttResult result = primary.transcribe(request, config);
|
||||
if (result.isSuccess()) return result;
|
||||
if (result.isSuccess()) {
|
||||
log.info("[STT] success via {} ({} chars)", primary.id(), result.getText() != null ? result.getText().length() : 0);
|
||||
return result;
|
||||
}
|
||||
log.warn("[STT] primary {} failed: {}", primary.id(), result.getErrorMessage());
|
||||
|
||||
List<String> errors = new ArrayList<>();
|
||||
errors.add(primary.id() + ": " + result.getErrorMessage());
|
||||
|
||||
if (Boolean.TRUE.equals(config.getSttFallbackEnabled())) {
|
||||
for (SttProvider fb : providerRegistry.fallbackCandidates(config, primary.id())) {
|
||||
log.info("[STT] Trying fallback provider: {}", fb.id());
|
||||
for (SttProvider fb : providerRegistry.fallbackCandidates(config, primary.id(), languageHint)) {
|
||||
log.info("[STT] trying fallback provider: {}", fb.id());
|
||||
result = fb.transcribe(request, config);
|
||||
if (result.isSuccess()) return result;
|
||||
if (result.isSuccess()) {
|
||||
log.info("[STT] fallback success via {} ({} chars)", fb.id(),
|
||||
result.getText() != null ? result.getText().length() : 0);
|
||||
return result;
|
||||
}
|
||||
log.warn("[STT] fallback {} failed: {}", fb.id(), result.getErrorMessage());
|
||||
errors.add(fb.id() + ": " + result.getErrorMessage());
|
||||
}
|
||||
}
|
||||
|
||||
log.error("[STT] all providers failed — errors: {}", errors);
|
||||
return SttResult.failure("所有 STT Provider 均失败\n" + String.join("\n", errors));
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,69 @@
|
||||
package vip.mate.stt;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
import java.nio.ByteOrder;
|
||||
|
||||
/**
|
||||
* Strip the RIFF/WAVE header off a WAV blob to expose raw PCM samples.
|
||||
*
|
||||
* <p>DashScope's realtime ASR expects the {@code parameters.format = "pcm"}
|
||||
* input as **bare 16-bit signed little-endian PCM**, not WAV. The frontend
|
||||
* (see {@code mateclaw-ui/src/utils/wavEncoder.ts}) emits a 16 kHz mono
|
||||
* 16-bit WAV with the canonical 44-byte header — this helper unwraps it.
|
||||
*
|
||||
* <p>Why not just send the WAV: DashScope rejects with "format mismatch"
|
||||
* because the first 44 bytes look like garbage when interpreted as PCM
|
||||
* samples — they're the RIFF magic + format chunk metadata.
|
||||
*
|
||||
* <p>Limitations: handles only the canonical 44-byte WAV layout produced by
|
||||
* MateClaw's WavRecorder. WAVs with extra chunks (LIST, JUNK, …) before the
|
||||
* data chunk would need a chunk-walking parser. We don't currently accept
|
||||
* arbitrary uploads, so the tighter scope is fine; if this changes,
|
||||
* extend {@link #extract} to scan for the {@code "data"} chunk header
|
||||
* instead of assuming offset 36.
|
||||
*/
|
||||
public final class WavPcmExtractor {
|
||||
|
||||
/** Bytes before the "data" chunk in a canonical mono 16-bit PCM WAV. */
|
||||
public static final int CANONICAL_HEADER_BYTES = 44;
|
||||
|
||||
/** Sample rate field offset in the canonical WAV header. */
|
||||
private static final int OFFSET_SAMPLE_RATE = 24;
|
||||
|
||||
private WavPcmExtractor() {}
|
||||
|
||||
/**
|
||||
* Extract raw PCM bytes from a WAV blob. Throws when the input is too short
|
||||
* or the magic header bytes don't look like RIFF/WAVE — better to fail loud
|
||||
* here than ship garbage to DashScope and chase a confusing error code.
|
||||
*/
|
||||
public static byte[] extract(byte[] wavBytes) {
|
||||
if (wavBytes == null || wavBytes.length < CANONICAL_HEADER_BYTES) {
|
||||
throw new IllegalArgumentException(
|
||||
"WAV input too short: " + (wavBytes == null ? 0 : wavBytes.length) + " bytes");
|
||||
}
|
||||
if (wavBytes[0] != 'R' || wavBytes[1] != 'I' || wavBytes[2] != 'F' || wavBytes[3] != 'F'
|
||||
|| wavBytes[8] != 'W' || wavBytes[9] != 'A' || wavBytes[10] != 'V' || wavBytes[11] != 'E') {
|
||||
throw new IllegalArgumentException("Not a WAV (missing RIFF/WAVE magic)");
|
||||
}
|
||||
byte[] pcm = new byte[wavBytes.length - CANONICAL_HEADER_BYTES];
|
||||
System.arraycopy(wavBytes, CANONICAL_HEADER_BYTES, pcm, 0, pcm.length);
|
||||
return pcm;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the sample rate from a WAV header. Used by callers that need to
|
||||
* tell DashScope the actual rate of the audio (the API requires the rate
|
||||
* up front in the {@code run-task} message — getting it wrong produces
|
||||
* recognisable but distorted transcripts).
|
||||
*/
|
||||
public static int sampleRate(byte[] wavBytes) {
|
||||
if (wavBytes == null || wavBytes.length < CANONICAL_HEADER_BYTES) {
|
||||
throw new IllegalArgumentException("WAV input too short for sample-rate read");
|
||||
}
|
||||
return ByteBuffer.wrap(wavBytes, OFFSET_SAMPLE_RATE, 4)
|
||||
.order(ByteOrder.LITTLE_ENDIAN)
|
||||
.getInt();
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,7 +1,5 @@
|
||||
package vip.mate.stt.provider;
|
||||
|
||||
import cn.hutool.http.HttpRequest;
|
||||
import cn.hutool.http.HttpResponse;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
@ -11,63 +9,486 @@ import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.stt.SttProvider;
|
||||
import vip.mate.stt.SttRequest;
|
||||
import vip.mate.stt.SttResult;
|
||||
import vip.mate.stt.WavPcmExtractor;
|
||||
import vip.mate.system.model.SystemSettingsDTO;
|
||||
|
||||
import java.net.URI;
|
||||
import java.net.http.HttpClient;
|
||||
import java.net.http.WebSocket;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.time.Duration;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
import java.util.concurrent.CompletionStage;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
/**
|
||||
* DashScope STT Provider — Paraformer（OpenAI 兼容接口）
|
||||
* <p>
|
||||
* 复用模型管理中的 DashScope API Key。中文识别效果优秀。
|
||||
* DashScope STT Provider — Paraformer Realtime via WebSocket.
|
||||
*
|
||||
* <p>DashScope's only sync-callable STT path is the realtime WebSocket API
|
||||
* — there is no <code>/audio/transcriptions</code> endpoint on either the
|
||||
* native or OpenAI-compatible HTTP surface (verified empirically, returns
|
||||
* 404). The earlier sync-HTTP version of this provider was speculative and
|
||||
* has been replaced by this one.
|
||||
*
|
||||
* <h2>Wire protocol</h2>
|
||||
* Documented at <i>Aliyun DashScope Realtime ASR</i>. Message exchange:
|
||||
* <ol>
|
||||
* <li>Open WS to {@value #WS_URL} with {@code Authorization: bearer
|
||||
* <api-key>} header.</li>
|
||||
* <li>Client sends a {@code run-task} text frame with task_id +
|
||||
* paraformer-realtime-v2 model + format/sample-rate parameters.</li>
|
||||
* <li>Server replies with {@code task-started} text frame.</li>
|
||||
* <li>Client streams raw 16-bit PCM bytes as binary frames (chunked at
|
||||
* ~100ms each = {@value #CHUNK_BYTES} bytes for 16 kHz mono).</li>
|
||||
* <li>Server emits {@code result-generated} events as transcripts come
|
||||
* in. Each event carries a sentence keyed by {@code begin_time};
|
||||
* later events with the same {@code begin_time} update the same
|
||||
* sentence (interim → final).</li>
|
||||
* <li>Client sends {@code finish-task} text frame; server replies with
|
||||
* {@code task-finished}; both sides close.</li>
|
||||
* </ol>
|
||||
*
|
||||
* <p>The {@link SttProvider} interface is sync — we bridge the async WS
|
||||
* conversation to a blocking call via {@link CountDownLatch} (run-task ack
|
||||
* + task-finished ack) plus an overall hard timeout. The whole transcribe
|
||||
* call returns either a full transcript or a domain-typed
|
||||
* {@link SttResult#failure} after at most {@value #OVERALL_TIMEOUT_MS}ms.
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class DashScopeSttProvider implements SttProvider {
|
||||
|
||||
/** DashScope WS endpoint for realtime inference (audio/text/multimodal). */
|
||||
static final URI WS_URL = URI.create("wss://dashscope.aliyuncs.com/api-ws/v1/inference/");
|
||||
|
||||
/** Default model — paraformer-realtime-v2 is the canonical 2024+ realtime ASR. */
|
||||
static final String DEFAULT_MODEL = "paraformer-realtime-v2";
|
||||
|
||||
/** Default sample rate in Hz. Must match the actual WAV — the helper reads it. */
|
||||
static final int DEFAULT_SAMPLE_RATE_HZ = 16_000;
|
||||
|
||||
/** ~100ms of 16 kHz / 16-bit / mono PCM. DashScope recommends 100-300ms chunks. */
|
||||
static final int CHUNK_BYTES = 3200;
|
||||
|
||||
/**
|
||||
* How long to sleep between chunks. Paraformer-Realtime expects audio to
|
||||
* arrive at roughly the natural recording rate; if we dump the whole clip
|
||||
* in tens of milliseconds the server discards the stream and replies with
|
||||
* task-finished + zero result-generated events. The official Python SDK
|
||||
* does the same with {@code time.sleep(0.1)} between chunks. Matches
|
||||
* {@link #CHUNK_BYTES} (100ms of audio → 100ms wall sleep).
|
||||
*/
|
||||
static final long CHUNK_PACING_MS = 100L;
|
||||
|
||||
/** How long to wait for the WS handshake + task-started ack before giving up. */
|
||||
static final long TASK_STARTED_TIMEOUT_MS = 10_000L;
|
||||
|
||||
/** Overall budget for a single transcribe — beyond this we abort the WS. */
|
||||
static final long OVERALL_TIMEOUT_MS = 60_000L;
|
||||
|
||||
private final ModelProviderService modelProviderService;
|
||||
private final ObjectMapper objectMapper;
|
||||
|
||||
private static final String BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1";
|
||||
private static final String DEFAULT_MODEL = "paraformer-v2";
|
||||
/** Shared HttpClient — JDK's WebSocket builder doesn't reuse the underlying
|
||||
* connection pool when you allocate a fresh client per call, so making
|
||||
* this a field saves a connection-pool spin-up on every transcribe. */
|
||||
private final HttpClient httpClient = HttpClient.newHttpClient();
|
||||
|
||||
@Override public String id() { return "dashscope"; }
|
||||
@Override public String label() { return "DashScope (Paraformer)"; }
|
||||
@Override public String label() { return "DashScope (Paraformer Realtime)"; }
|
||||
@Override public boolean requiresCredential() { return true; }
|
||||
@Override public int autoDetectOrder() { return 150; }
|
||||
|
||||
/**
|
||||
* Per-language priority. Paraformer is the strongest mainstream Chinese
|
||||
* STT, so push it ahead of Whisper on zh — see {@link SttProvider} javadoc
|
||||
* for the routing rationale.
|
||||
*/
|
||||
@Override
|
||||
public int autoDetectOrder(String language) {
|
||||
if (language == null) return autoDetectOrder();
|
||||
String lang = language.toLowerCase();
|
||||
if (lang.startsWith("zh")) return 60;
|
||||
return autoDetectOrder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable(SystemSettingsDTO config) {
|
||||
try { return modelProviderService.isProviderConfigured("dashscope"); }
|
||||
catch (Exception e) { return false; }
|
||||
try {
|
||||
return modelProviderService.isProviderConfigured("dashscope");
|
||||
} catch (Exception e) {
|
||||
log.warn("[DashScope STT] availability check failed: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public SttResult transcribe(SttRequest request, SystemSettingsDTO config) {
|
||||
try {
|
||||
String apiKey = modelProviderService.getProviderConfig("dashscope").getApiKey();
|
||||
if (apiKey == null) return SttResult.failure("DashScope API Key 未配置");
|
||||
|
||||
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
|
||||
String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg";
|
||||
|
||||
HttpResponse response = HttpRequest.post(BASE_URL + "/audio/transcriptions")
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.form("model", model)
|
||||
.form("file", request.getAudioData(), request.getContentType(), fileName)
|
||||
.timeout(60_000)
|
||||
.execute();
|
||||
|
||||
if (response.getStatus() == 200) {
|
||||
JsonNode result = objectMapper.readTree(response.body());
|
||||
String text = result.path("text").asText("");
|
||||
log.info("[DashScope STT] Transcribed {} chars (model={})", text.length(), model);
|
||||
return SttResult.success(text);
|
||||
} else {
|
||||
log.warn("[DashScope STT] Failed: HTTP {} - {}", response.getStatus(), response.body());
|
||||
return SttResult.failure("DashScope STT 失败: HTTP " + response.getStatus());
|
||||
if (apiKey == null || apiKey.isBlank()) {
|
||||
return SttResult.failure("DashScope API Key 未配置");
|
||||
}
|
||||
byte[] audio = request.getAudioData();
|
||||
if (audio == null || audio.length < WavPcmExtractor.CANONICAL_HEADER_BYTES) {
|
||||
return SttResult.failure("音频为空或过短");
|
||||
}
|
||||
byte[] pcm = WavPcmExtractor.extract(audio);
|
||||
int sampleRate = WavPcmExtractor.sampleRate(audio);
|
||||
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
|
||||
String taskId = UUID.randomUUID().toString().replace("-", "");
|
||||
|
||||
// Peak/RMS check — the silence path is a failure mode worth its
|
||||
// own log line so users can tell "mic captured nothing" from
|
||||
// "DashScope rejected real audio". Successful calls log peak/rms
|
||||
// at DEBUG only; a healthy call shouldn't produce a per-request
|
||||
// INFO log every time the user holds the talk button.
|
||||
int[] peakRms = computePcmPeakRms(pcm);
|
||||
if (peakRms[0] == 0) {
|
||||
log.warn("[DashScope STT] PCM is silent (peak=0, bytes={}) — check mic permission / frontend recording",
|
||||
pcm.length);
|
||||
return SttResult.failure(
|
||||
"音频为静音（PCM peak=0）— 检查麦克风权限或前端录制实现");
|
||||
}
|
||||
log.debug("[DashScope STT] PCM stats — bytes={} samples={} peak={} rms={} sampleRate={}",
|
||||
pcm.length, pcm.length / 2, peakRms[0], peakRms[1], sampleRate);
|
||||
|
||||
DashScopeSession session = new DashScopeSession(taskId, objectMapper);
|
||||
WebSocket ws;
|
||||
try {
|
||||
ws = httpClient.newWebSocketBuilder()
|
||||
.header("Authorization", "bearer " + apiKey)
|
||||
.connectTimeout(Duration.ofMillis(TASK_STARTED_TIMEOUT_MS))
|
||||
.buildAsync(WS_URL, session)
|
||||
.get(TASK_STARTED_TIMEOUT_MS, TimeUnit.MILLISECONDS);
|
||||
} catch (TimeoutException e) {
|
||||
return SttResult.failure("DashScope WS 握手超时");
|
||||
}
|
||||
|
||||
try {
|
||||
// 1. run-task. Envelope dumped at DEBUG only — the JSON is
|
||||
// identical across calls modulo task_id + language hint, so
|
||||
// logging it on every transcribe just clutters logs.
|
||||
String runTask = buildRunTask(taskId, model, sampleRate, request.getLanguage());
|
||||
log.debug("[DashScope STT] run-task envelope: {}", runTask);
|
||||
ws.sendText(runTask, true).get(TASK_STARTED_TIMEOUT_MS, TimeUnit.MILLISECONDS);
|
||||
|
||||
// 2. wait for task-started ack
|
||||
if (!session.awaitTaskStarted(TASK_STARTED_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return SttResult.failure("DashScope task-started 超时");
|
||||
}
|
||||
if (session.failed()) {
|
||||
return SttResult.failure("DashScope: " + session.errorMessage());
|
||||
}
|
||||
|
||||
// 3. stream PCM chunks at real-time pace. Paraformer-Realtime
|
||||
// is built for live mic input and silently drops audio when it
|
||||
// arrives faster than wall-clock — symptom is 0 chars
|
||||
// transcribed even though the protocol completes successfully
|
||||
// (no task-failed). Sleep 100ms between 100ms chunks so total
|
||||
// send time ≈ audio duration, matching what DashScope's own
|
||||
// SDK examples do (time.sleep(0.1) per chunk).
|
||||
int chunksSent = 0;
|
||||
long sendStart = System.currentTimeMillis();
|
||||
for (int offset = 0; offset < pcm.length; offset += CHUNK_BYTES) {
|
||||
int len = Math.min(CHUNK_BYTES, pcm.length - offset);
|
||||
ByteBuffer chunk = ByteBuffer.wrap(pcm, offset, len);
|
||||
ws.sendBinary(chunk, true).get(TASK_STARTED_TIMEOUT_MS, TimeUnit.MILLISECONDS);
|
||||
chunksSent++;
|
||||
Thread.sleep(CHUNK_PACING_MS);
|
||||
// Cheap fail-fast: if the server already said we're done /
|
||||
// failed mid-stream, stop sending so we don't waste seconds
|
||||
// sleeping on a dead connection.
|
||||
if (session.failed() || session.taskFinishedRaised()) break;
|
||||
}
|
||||
long sendDuration = System.currentTimeMillis() - sendStart;
|
||||
log.debug("[DashScope STT] streamed {} chunks ({} bytes) in {} ms",
|
||||
chunksSent, pcm.length, sendDuration);
|
||||
|
||||
// 4. finish-task
|
||||
ws.sendText(buildFinishTask(taskId), true).get(TASK_STARTED_TIMEOUT_MS, TimeUnit.MILLISECONDS);
|
||||
|
||||
// 5. wait for task-finished
|
||||
if (!session.awaitTaskFinished(OVERALL_TIMEOUT_MS, TimeUnit.MILLISECONDS)) {
|
||||
return SttResult.failure("DashScope task-finished 超时");
|
||||
}
|
||||
if (session.failed()) {
|
||||
return SttResult.failure("DashScope: " + session.errorMessage());
|
||||
}
|
||||
|
||||
String text = session.aggregatedText();
|
||||
log.info("[DashScope STT] Transcribed {} chars from {} result-events "
|
||||
+ "(model={}, sampleRate={}, pcmBytes={})",
|
||||
text.length(), session.resultEventCount(), model, sampleRate, pcm.length);
|
||||
if (text.isEmpty() && session.resultEventCount() == 0) {
|
||||
// Distinct failure mode: protocol completed cleanly but
|
||||
// server never sent a single result-generated event.
|
||||
// Almost always means the audio was discarded for
|
||||
// pacing/format reasons. Surface as a typed failure so
|
||||
// the fallback chain (Whisper) can still try.
|
||||
return SttResult.failure(
|
||||
"DashScope 收到 0 个识别事件——可能是音频格式或节奏问题");
|
||||
}
|
||||
return SttResult.success(text);
|
||||
} finally {
|
||||
// Best-effort close. abort() is fire-and-forget; we don't need to wait.
|
||||
try {
|
||||
ws.sendClose(WebSocket.NORMAL_CLOSURE, "done");
|
||||
} catch (Exception ignored) {
|
||||
ws.abort();
|
||||
}
|
||||
}
|
||||
} catch (TimeoutException e) {
|
||||
log.warn("[DashScope STT] timeout: {}", e.getMessage());
|
||||
return SttResult.failure("DashScope STT 超时: " + e.getMessage());
|
||||
} catch (ExecutionException e) {
|
||||
Throwable cause = e.getCause() != null ? e.getCause() : e;
|
||||
log.error("[DashScope STT] WS error: {}", cause.getMessage(), cause);
|
||||
return SttResult.failure("DashScope STT WS 错误: " + cause.getMessage());
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
return SttResult.failure("DashScope STT 被中断");
|
||||
} catch (Exception e) {
|
||||
log.error("[DashScope STT] Error: {}", e.getMessage(), e);
|
||||
return SttResult.failure("DashScope STT 异常: " + e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/* ====================================================================== */
|
||||
/* Wire-format helpers (package-private for unit testing). */
|
||||
/* ====================================================================== */
|
||||
|
||||
String buildRunTask(String taskId, String model, int sampleRate, String language) throws Exception {
|
||||
Map<String, Object> parameters = new LinkedHashMap<>();
|
||||
parameters.put("format", "pcm");
|
||||
parameters.put("sample_rate", sampleRate);
|
||||
// Language hint when supplied — paraformer-realtime-v2 supports
|
||||
// "zh", "en", "ja", "ko" via language_hints. Skip when null/blank
|
||||
// to let the model auto-detect.
|
||||
if (language != null && !language.isBlank()) {
|
||||
// Strip locale suffix (zh-CN → zh).
|
||||
String hint = language.toLowerCase();
|
||||
int dash = hint.indexOf('-');
|
||||
if (dash > 0) hint = hint.substring(0, dash);
|
||||
parameters.put("language_hints", new String[]{hint});
|
||||
}
|
||||
|
||||
Map<String, Object> payload = Map.of(
|
||||
"task_group", "audio",
|
||||
"task", "asr",
|
||||
"function", "recognition",
|
||||
"model", model,
|
||||
"parameters", parameters,
|
||||
"input", Map.of());
|
||||
Map<String, Object> message = Map.of(
|
||||
"header", Map.of(
|
||||
"action", "run-task",
|
||||
"task_id", taskId,
|
||||
"streaming", "duplex"),
|
||||
"payload", payload);
|
||||
return objectMapper.writeValueAsString(message);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute peak (max absolute value) and RMS for 16-bit signed
|
||||
* little-endian PCM bytes. Returns {peak, rms} as ints for log-friendly
|
||||
* formatting. Both metrics are in raw int16 units (-32768..32767).
|
||||
*
|
||||
* <p>Reference values for 16-bit PCM at typical recording levels:
|
||||
* <ul>
|
||||
* <li>Silence / muted mic: peak ≤ 5, rms ≤ 2</li>
|
||||
* <li>Quiet speech: peak ≈ 1000-5000, rms ≈ 200-1000</li>
|
||||
* <li>Normal speech: peak ≈ 5000-20000, rms ≈ 1000-5000</li>
|
||||
* <li>Loud / close-mic: peak ≈ 20000-32000, rms ≈ 5000-15000</li>
|
||||
* </ul>
|
||||
*/
|
||||
static int[] computePcmPeakRms(byte[] pcm) {
|
||||
if (pcm == null || pcm.length < 2) {
|
||||
return new int[]{0, 0};
|
||||
}
|
||||
int peak = 0;
|
||||
long sumSq = 0;
|
||||
int sampleCount = pcm.length / 2;
|
||||
for (int i = 0; i < sampleCount; i++) {
|
||||
// Little-endian 16-bit signed: low byte first.
|
||||
int lo = pcm[i * 2] & 0xFF;
|
||||
int hi = pcm[i * 2 + 1]; // signed
|
||||
int sample = (hi << 8) | lo;
|
||||
int abs = Math.abs(sample);
|
||||
if (abs > peak) peak = abs;
|
||||
sumSq += (long) sample * sample;
|
||||
}
|
||||
int rms = (int) Math.sqrt((double) sumSq / sampleCount);
|
||||
return new int[]{peak, rms};
|
||||
}
|
||||
|
||||
String buildFinishTask(String taskId) throws Exception {
|
||||
Map<String, Object> message = Map.of(
|
||||
"header", Map.of(
|
||||
"action", "finish-task",
|
||||
"task_id", taskId,
|
||||
"streaming", "duplex"),
|
||||
"payload", Map.of("input", Map.of()));
|
||||
return objectMapper.writeValueAsString(message);
|
||||
}
|
||||
|
||||
/* ====================================================================== */
|
||||
/* WebSocket.Listener: collects events and signals task-started/finished. */
|
||||
/* ====================================================================== */
|
||||
|
||||
/**
|
||||
* State machine for one DashScope ASR conversation. Package-private so
|
||||
* unit tests can drive it with synthetic JSON without hitting the network.
|
||||
*/
|
||||
static class DashScopeSession implements WebSocket.Listener {
|
||||
private final String taskId;
|
||||
private final ObjectMapper mapper;
|
||||
private final CountDownLatch taskStarted = new CountDownLatch(1);
|
||||
private final CountDownLatch taskFinished = new CountDownLatch(1);
|
||||
|
||||
/**
|
||||
* Sentence buffer keyed by begin_time. DashScope emits multiple
|
||||
* {@code result-generated} events for the same sentence as it gets
|
||||
* refined (interim → final); each new event for a given begin_time
|
||||
* supersedes the previous text. LinkedHashMap preserves arrival
|
||||
* order, which roughly matches speech order, for the final concat.
|
||||
*/
|
||||
private final Map<Long, String> sentencesByBeginTime = new LinkedHashMap<>();
|
||||
|
||||
/** Buffer for fragmented text frames (WS allows partial messages). */
|
||||
private final StringBuilder textFrameBuf = new StringBuilder();
|
||||
|
||||
private final AtomicReference<String> errorMessage = new AtomicReference<>();
|
||||
|
||||
/** Counts result-generated events — distinguishes "server got our audio
|
||||
* but recognised nothing" (>0 events, all empty text) from "server
|
||||
* saw zero audio frames" (0 events). Helps diagnose pacing /
|
||||
* format issues. */
|
||||
private int resultEventCount;
|
||||
|
||||
DashScopeSession(String taskId, ObjectMapper mapper) {
|
||||
this.taskId = taskId;
|
||||
this.mapper = mapper;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<?> onText(WebSocket webSocket, CharSequence data, boolean last) {
|
||||
textFrameBuf.append(data);
|
||||
if (last) {
|
||||
handleMessage(textFrameBuf.toString());
|
||||
textFrameBuf.setLength(0);
|
||||
}
|
||||
webSocket.request(1);
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void onError(WebSocket webSocket, Throwable error) {
|
||||
errorMessage.compareAndSet(null, "WS error: " + error.getMessage());
|
||||
taskStarted.countDown();
|
||||
taskFinished.countDown();
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletionStage<?> onClose(WebSocket webSocket, int statusCode, String reason) {
|
||||
// If the server closes before task-finished, unblock waiters.
|
||||
if (taskFinished.getCount() > 0) {
|
||||
errorMessage.compareAndSet(null,
|
||||
"WS closed before task-finished (status=" + statusCode + ", reason=" + reason + ")");
|
||||
}
|
||||
taskStarted.countDown();
|
||||
taskFinished.countDown();
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Package-private hook so unit tests can drive {@link DashScopeSession} without a real WebSocket. */
|
||||
void handleMessage(String json) {
|
||||
// Always trace the raw frame at DEBUG — this is invaluable when
|
||||
// the protocol completes "successfully" but produces no
|
||||
// transcripts. Without seeing every frame it's impossible to
|
||||
// tell whether DashScope sent us a status-update / warning we
|
||||
// ignored, or just stayed silent between task-started and
|
||||
// task-finished.
|
||||
log.debug("[DashScope STT] frame: {}", json);
|
||||
try {
|
||||
JsonNode node = mapper.readTree(json);
|
||||
String event = node.path("header").path("event").asText();
|
||||
switch (event) {
|
||||
case "task-started" -> taskStarted.countDown();
|
||||
case "result-generated" -> {
|
||||
resultEventCount++;
|
||||
JsonNode sentence = node.path("payload").path("output").path("sentence");
|
||||
if (sentence.isObject()) {
|
||||
long beginTime = sentence.path("begin_time").asLong(0L);
|
||||
String text = sentence.path("text").asText("");
|
||||
// Always overwrite — later events for the same begin_time
|
||||
// carry the more-final transcript.
|
||||
sentencesByBeginTime.put(beginTime, text);
|
||||
}
|
||||
}
|
||||
case "task-finished" -> taskFinished.countDown();
|
||||
case "task-failed" -> {
|
||||
String msg = node.path("header").path("error_message").asText("unknown");
|
||||
String code = node.path("header").path("error_code").asText("");
|
||||
errorMessage.compareAndSet(null,
|
||||
code.isEmpty() ? msg : (code + " — " + msg));
|
||||
// Wake both latches so the caller can return the typed
|
||||
// failure instead of timing out for the full budget.
|
||||
taskStarted.countDown();
|
||||
taskFinished.countDown();
|
||||
}
|
||||
// Anything else (status updates, model warnings, beta
|
||||
// events) gets surfaced at INFO so it shows up without
|
||||
// turning DEBUG on. If DashScope rolls out a new event
|
||||
// type we should know about, this catches it.
|
||||
default -> log.info("[DashScope STT] unhandled event '{}' frame={}", event, json);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.warn("[DashScope STT] failed to parse WS message: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
boolean awaitTaskStarted(long timeout, TimeUnit unit) throws InterruptedException {
|
||||
return taskStarted.await(timeout, unit);
|
||||
}
|
||||
|
||||
boolean awaitTaskFinished(long timeout, TimeUnit unit) throws InterruptedException {
|
||||
return taskFinished.await(timeout, unit);
|
||||
}
|
||||
|
||||
boolean failed() {
|
||||
return errorMessage.get() != null;
|
||||
}
|
||||
|
||||
String errorMessage() {
|
||||
return errorMessage.get();
|
||||
}
|
||||
|
||||
/** True once task-finished has been observed — used by the sender
|
||||
* loop to bail out early instead of pacing through dead-WS sleeps. */
|
||||
boolean taskFinishedRaised() {
|
||||
return taskFinished.getCount() == 0;
|
||||
}
|
||||
|
||||
int resultEventCount() {
|
||||
return resultEventCount;
|
||||
}
|
||||
|
||||
String aggregatedText() {
|
||||
// Concat in begin_time order. Different sentences typically don't
|
||||
// need separator characters because Chinese text streams are
|
||||
// already glued; for safety against missed punctuation we leave
|
||||
// a soft join ("") rather than space — Whisper-style space
|
||||
// joining produces odd-looking Chinese transcripts.
|
||||
StringBuilder sb = new StringBuilder();
|
||||
sentencesByBeginTime.values().forEach(sb::append);
|
||||
return sb.toString();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ import lombok.RequiredArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.stereotype.Component;
|
||||
import vip.mate.llm.service.ModelProviderService;
|
||||
import vip.mate.stt.AudioMimeTypes;
|
||||
import vip.mate.stt.SttProvider;
|
||||
import vip.mate.stt.SttRequest;
|
||||
import vip.mate.stt.SttResult;
|
||||
@ -33,10 +34,33 @@ public class OpenAiSttProvider implements SttProvider {
|
||||
@Override public boolean requiresCredential() { return true; }
|
||||
@Override public int autoDetectOrder() { return 100; }
|
||||
|
||||
/**
|
||||
* Whisper is the canonical English STT and noticeably weaker on Chinese
|
||||
* (it tends to produce simplified-character output even for traditional
|
||||
* input, and short Chinese clips frequently transcribe to gibberish).
|
||||
* Boost Whisper's priority for English/Japanese/Korean (where it leads),
|
||||
* and de-prioritise it for Chinese so DashScope (Paraformer) wins the
|
||||
* auto-pick.
|
||||
*/
|
||||
@Override
|
||||
public int autoDetectOrder(String language) {
|
||||
if (language == null) return autoDetectOrder();
|
||||
String lang = language.toLowerCase();
|
||||
if (lang.startsWith("zh")) return 250; // pushed below DashScope Paraformer
|
||||
if (lang.startsWith("en")
|
||||
|| lang.startsWith("ja")
|
||||
|| lang.startsWith("ko")) return 80; // pulled above DashScope
|
||||
return autoDetectOrder();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean isAvailable(SystemSettingsDTO config) {
|
||||
try { return modelProviderService.isProviderConfigured("openai"); }
|
||||
catch (Exception e) { return false; }
|
||||
try {
|
||||
return modelProviderService.isProviderConfigured("openai");
|
||||
} catch (Exception e) {
|
||||
log.warn("[OpenAI STT] availability check failed: {}", e.getMessage());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -48,12 +72,18 @@ public class OpenAiSttProvider implements SttProvider {
|
||||
|
||||
String url = (baseUrl != null ? baseUrl : "https://api.openai.com") + "/v1/audio/transcriptions";
|
||||
String model = request.getModel() != null ? request.getModel() : DEFAULT_MODEL;
|
||||
String fileName = request.getFileName() != null ? request.getFileName() : "audio.ogg";
|
||||
// AudioMimeTypes ensures the filename extension matches the
|
||||
// actual bytes (audio.wav, audio.mp3, etc.), which Hutool then
|
||||
// uses to infer the multipart Content-Type. Don't pass
|
||||
// contentType to .form() explicitly — Hutool has no
|
||||
// form(String,byte[],String,String) overload, and the wrong
|
||||
// dispatch crashes with ClassCastException on byte[] → Object[].
|
||||
String fileName = AudioMimeTypes.resolveFileName(request.getFileName(), request.getContentType());
|
||||
|
||||
HttpResponse response = HttpRequest.post(url)
|
||||
.header("Authorization", "Bearer " + apiKey)
|
||||
.form("model", model)
|
||||
.form("file", request.getAudioData(), request.getContentType(), fileName)
|
||||
.form("file", request.getAudioData(), fileName)
|
||||
.timeout(60_000)
|
||||
.execute();
|
||||
|
||||
|
||||
@ -329,6 +329,22 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre
|
||||
KEY (id)
|
||||
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW());
|
||||
|
||||
-- Speech-to-text (STT) defaults — enabled out of the box so users only need to configure an API key.
|
||||
-- Skip-if-exists on setting_key (NOT MERGE BY id) so we don't trip the
|
||||
-- UNIQUE index when an existing user has the row at a runtime-assigned id
|
||||
-- from toggling the UI before this seed shipped. V46 covers the same.
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', 'Enable speech-to-text (TalkMode mic input)', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT provider: auto / openai / dashscope', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', 'Try alternate STT provider when the primary fails', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
|
||||
-- Built-in tool: Date & Time
|
||||
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||
KEY (id)
|
||||
|
||||
@ -368,6 +368,26 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr
|
||||
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG instance base URL (auto-configured in Docker)', NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||
|
||||
-- Speech-to-text (STT) defaults — enabled out of the box so users only need to configure an API key.
|
||||
-- Skip-if-exists keyed on setting_key (FROM DUAL ... WHERE NOT EXISTS) so
|
||||
-- we don't override a value the user explicitly set before this seed shipped,
|
||||
-- and don't trip the UNIQUE index on setting_key when their row is at a
|
||||
-- runtime-assigned id. V46 migration uses the same idiom.
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', 'Enable speech-to-text (TalkMode mic input)', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT provider: auto / openai / dashscope', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', 'Try alternate STT provider when the primary fails', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
|
||||
-- Built-in tool: Date & Time
|
||||
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||
VALUES (1000000001, 'DateTimeTool', 'Date & Time', 'Get current date and time information', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
||||
|
||||
@ -368,6 +368,25 @@ INSERT INTO mate_system_setting (id, setting_key, setting_value, description, cr
|
||||
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址（Docker 部署时自动配置）', NOW(), NOW())
|
||||
ON DUPLICATE KEY UPDATE setting_key=VALUES(setting_key), setting_value=VALUES(setting_value), description=VALUES(description), update_time=VALUES(update_time);
|
||||
|
||||
-- 语音识别（STT）默认配置 —— 默认启用，用户只需在模型管理中配置 OpenAI / DashScope API Key 即可使用
|
||||
-- 用 setting_key 的 skip-if-exists 写法（FROM DUAL ... WHERE NOT EXISTS），
|
||||
-- 既不强行覆盖用户显式设过的值，也不会撞 setting_key UNIQUE 索引。
|
||||
-- V46 迁移走的是同一套语义。
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', '启用语音识别（TalkMode 麦克风输入）', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT 提供商：auto / openai / dashscope', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', '主 provider 失败时自动尝试备选 provider', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
|
||||
-- 内置工具：日期时间
|
||||
INSERT INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||
VALUES (1000000001, 'DateTimeTool', '日期时间', '获取当前日期和时间信息', 'builtin', 'dateTimeTool', '🕐', TRUE, TRUE, NOW(), NOW(), 0)
|
||||
|
||||
@ -333,6 +333,22 @@ MERGE INTO mate_system_setting (id, setting_key, setting_value, description, cre
|
||||
KEY (id)
|
||||
VALUES (1000000013, 'searxngBaseUrl', '', 'SearXNG 实例地址（Docker 部署时自动配置）', NOW(), NOW());
|
||||
|
||||
-- 语音识别（STT）默认配置 —— 默认启用，用户只需在模型管理中配置 OpenAI / DashScope API Key 即可使用
|
||||
-- 用 setting_key 的 skip-if-exists 写法（不再走 MERGE BY id），避免与
|
||||
-- 旧版 UI 已经写入的运行时 snowflake id 在 setting_key UNIQUE 索引上撞车。
|
||||
-- V46 迁移走的是同一套语义。
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', '启用语音识别（TalkMode 麦克风输入）', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT 提供商：auto / openai / dashscope', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', '主 provider 失败时自动尝试备选 provider', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
|
||||
-- 内置工具：日期时间
|
||||
MERGE INTO mate_tool (id, name, display_name, description, tool_type, bean_name, icon, enabled, builtin, create_time, update_time, deleted)
|
||||
KEY (id)
|
||||
|
||||
@ -0,0 +1,31 @@
|
||||
-- Default-enable STT on existing deployments. New installs run V46 first
|
||||
-- (table empty, INSERT succeeds), then DatabaseBootstrapRunner re-runs the
|
||||
-- same statements from data-{en,zh,mysql-en,mysql-zh}.sql with the same
|
||||
-- skip-if-exists semantics, so this is idempotent across both paths.
|
||||
--
|
||||
-- Why default to enabled: STT requires a recording UI gesture to even be
|
||||
-- exercised, so leaving it off by default just makes "the mic button does
|
||||
-- nothing" the most-reported support issue. Once enabled, it still does
|
||||
-- nothing dangerous unless the user has also configured an OpenAI / DashScope
|
||||
-- API key — those are the real gating credentials.
|
||||
--
|
||||
-- Idiom: INSERT ... SELECT ... WHERE NOT EXISTS, keyed on setting_key.
|
||||
-- Earlier versions of this migration used MERGE INTO ... KEY (id), which
|
||||
-- crashed on deployments where the user had already toggled STT in the UI:
|
||||
-- their row landed at a runtime-assigned snowflake id, then this migration
|
||||
-- tried to insert a fresh row at id=1000000020 and tripped the UNIQUE index
|
||||
-- on setting_key. Skip-if-exists preserves whatever value the user picked
|
||||
-- (don't override an explicit "off" with "on"). See
|
||||
-- https://git.mate.vip/mate/MateClaw issue noted 2026-04-26.
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', 'Enable speech-to-text (TalkMode mic input)', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT provider: auto / openai / dashscope', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', 'Try alternate STT provider when the primary fails', NOW(), NOW()
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
@ -0,0 +1,24 @@
|
||||
-- Default-enable STT on existing deployments. See the h2/ counterpart for
|
||||
-- the "why enabled by default" rationale and the bug history that drove
|
||||
-- the skip-if-exists idiom. Same V number is used in h2/ for cross-dialect
|
||||
-- parity.
|
||||
--
|
||||
-- MySQL doesn't allow `INSERT ... SELECT ... WHERE NOT EXISTS` without a
|
||||
-- FROM clause, so we synthesise one with `FROM DUAL`. The end result is
|
||||
-- the same: insert when the setting_key is absent, no-op when it's already
|
||||
-- there.
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000020, 'sttEnabled', 'true', 'Enable speech-to-text (TalkMode mic input)', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttEnabled');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000021, 'sttProvider', 'auto', 'STT provider: auto / openai / dashscope', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttProvider');
|
||||
|
||||
INSERT INTO mate_system_setting (id, setting_key, setting_value, description, create_time, update_time)
|
||||
SELECT 1000000022, 'sttFallbackEnabled', 'true', 'Try alternate STT provider when the primary fails', NOW(), NOW()
|
||||
FROM DUAL
|
||||
WHERE NOT EXISTS (SELECT 1 FROM mate_system_setting WHERE setting_key = 'sttFallbackEnabled');
|
||||
@ -37,13 +37,13 @@
|
||||
<button
|
||||
class="talk-ptt"
|
||||
:class="{ active: state === 'listening' }"
|
||||
:disabled="state === 'processing' || state === 'speaking'"
|
||||
:disabled="!canRecord"
|
||||
@mousedown="startListening"
|
||||
@mouseup="stopListening"
|
||||
@touchstart.prevent="startListening"
|
||||
@touchend.prevent="stopListening"
|
||||
>
|
||||
{{ state === 'listening' ? t('talk.releaseToSend') : t('talk.holdToTalk') }}
|
||||
{{ pttLabel }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
@ -55,6 +55,7 @@ import { ref, computed, onMounted, onBeforeUnmount } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { ElMessage } from 'element-plus'
|
||||
import { CloseBold, Loading, Microphone, Service } from '@element-plus/icons-vue'
|
||||
import { WavRecorder } from '@/utils/wavEncoder'
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
@ -68,23 +69,53 @@ const emit = defineEmits<{
|
||||
close: []
|
||||
}>()
|
||||
|
||||
type TalkState = 'idle' | 'listening' | 'processing' | 'speaking'
|
||||
/**
|
||||
* Connection-aware state machine. Pre-fix the modal opened in 'idle' which
|
||||
* lit the PTT button green even while the WebSocket was still in
|
||||
* CONNECTING (readyState=0). Users hit the button immediately, recorded a
|
||||
* clip, and stopListening saw ws.readyState !== OPEN — the audio went into
|
||||
* the void. Now PTT is disabled until the backend sends {type:"ready"}.
|
||||
*/
|
||||
type TalkState = 'connecting' | 'idle' | 'listening' | 'processing' | 'speaking' | 'failed'
|
||||
|
||||
const state = ref<TalkState>('idle')
|
||||
const state = ref<TalkState>('connecting')
|
||||
const transcript = ref<Array<{ role: 'user' | 'assistant'; text: string }>>([])
|
||||
|
||||
let ws: WebSocket | null = null
|
||||
let mediaRecorder: MediaRecorder | null = null
|
||||
let audioChunks: Blob[] = []
|
||||
let recorder: WavRecorder | null = null
|
||||
/**
|
||||
* Persistent warmed-up recorder kept alive for the modal's lifetime so the
|
||||
* press-and-hold gesture doesn't race a first-time mic permission dialog.
|
||||
* The dialog steals focus → mouseup fires on the dialog instead of the PTT
|
||||
* button → stopListening never runs → recording is stuck on. Warming up
|
||||
* front-loads the permission prompt and reuses the MediaStream.
|
||||
*/
|
||||
let warmRecorder: WavRecorder | null = null
|
||||
let audioContext: AudioContext | null = null
|
||||
|
||||
/**
|
||||
* Button-enabled predicate. Allows:
|
||||
* - 'idle' — fresh / between recordings (the normal case)
|
||||
* - 'listening' — already recording, the release event must reach us
|
||||
* - 'failed' — clicking restarts the WS connection (retry path)
|
||||
*/
|
||||
const canRecord = computed(() =>
|
||||
state.value === 'idle' || state.value === 'listening' || state.value === 'failed')
|
||||
const pttLabel = computed(() => {
|
||||
if (state.value === 'connecting') return t('talk.connecting')
|
||||
if (state.value === 'failed') return t('talk.retry')
|
||||
return state.value === 'listening' ? t('talk.releaseToSend') : t('talk.holdToTalk')
|
||||
})
|
||||
|
||||
const stateClass = computed(() => 'talk-state--' + state.value)
|
||||
const stateLabel = computed(() => {
|
||||
switch (state.value) {
|
||||
case 'connecting': return t('talk.connecting')
|
||||
case 'idle': return t('talk.ready')
|
||||
case 'listening': return t('talk.listening')
|
||||
case 'processing': return t('talk.processing')
|
||||
case 'speaking': return t('talk.speaking')
|
||||
case 'failed': return t('talk.connectionError')
|
||||
default: return ''
|
||||
}
|
||||
})
|
||||
@ -92,11 +123,20 @@ const stateLabel = computed(() => {
|
||||
onMounted(() => {
|
||||
if (props.agentId) {
|
||||
connectWebSocket()
|
||||
// Pre-acquire mic permission so the press-and-hold path skips the
|
||||
// permission dialog. Best-effort — failure here just means the user
|
||||
// gets the dialog on first PTT press (i.e. previous behaviour).
|
||||
warmRecorder = new WavRecorder()
|
||||
warmRecorder.warmUp().catch(err => {
|
||||
console.debug('[TalkMode] mic warm-up failed (will prompt on first PTT)', err)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
onBeforeUnmount(() => {
|
||||
disconnectWebSocket()
|
||||
warmRecorder?.releaseWarmUp()
|
||||
warmRecorder = null
|
||||
})
|
||||
|
||||
function connectWebSocket() {
|
||||
@ -104,10 +144,13 @@ function connectWebSocket() {
|
||||
const token = localStorage.getItem('token')
|
||||
const wsUrl = `${protocol}//${location.host}/api/v1/talk/ws${token ? '?token=' + token : ''}`
|
||||
|
||||
state.value = 'connecting'
|
||||
ws = new WebSocket(wsUrl)
|
||||
|
||||
ws.onopen = () => {
|
||||
// Send init message
|
||||
// Send init message. Stay in 'connecting' until the backend's
|
||||
// {type:"ready"} ack lands — the WS being open isn't the same as the
|
||||
// backend session being initialised.
|
||||
ws?.send(JSON.stringify({
|
||||
type: 'init',
|
||||
agentId: props.agentId,
|
||||
@ -127,6 +170,7 @@ function connectWebSocket() {
|
||||
const data = JSON.parse(event.data)
|
||||
switch (data.type) {
|
||||
case 'ready':
|
||||
// Only NOW does the PTT button unlock — see canRecord computed.
|
||||
state.value = 'idle'
|
||||
break
|
||||
case 'state':
|
||||
@ -151,14 +195,40 @@ function connectWebSocket() {
|
||||
}
|
||||
}
|
||||
|
||||
ws.onclose = () => {
|
||||
state.value = 'idle'
|
||||
ws.onerror = (e) => {
|
||||
console.warn('[TalkMode] WS error', e)
|
||||
if (state.value === 'connecting') {
|
||||
state.value = 'failed'
|
||||
ElMessage.error(t('talk.connectionError'))
|
||||
}
|
||||
}
|
||||
|
||||
ws.onclose = (e) => {
|
||||
console.debug('[TalkMode] WS closed', 'code=', e.code, 'reason=', e.reason)
|
||||
// Distinguish close-before-init (the user got a failed handshake) from
|
||||
// close-after-success (the modal was just closed). A premature close
|
||||
// leaves the user staring at "Ready" with no working button — flag
|
||||
// it so the retry path actually runs.
|
||||
if (state.value === 'connecting') {
|
||||
state.value = 'failed'
|
||||
} else if (state.value !== 'failed') {
|
||||
state.value = 'idle'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Manual reconnect — wired to the PTT button when state==='failed'. */
|
||||
function retryConnection() {
|
||||
ws?.close()
|
||||
ws = null
|
||||
connectWebSocket()
|
||||
}
|
||||
|
||||
function disconnectWebSocket() {
|
||||
if (mediaRecorder && mediaRecorder.state !== 'inactive') {
|
||||
mediaRecorder.stop()
|
||||
// Force-stop any in-flight recording so released the mic on close.
|
||||
if (recorder?.isActive()) {
|
||||
recorder.stop().catch(() => {})
|
||||
recorder = null
|
||||
}
|
||||
ws?.close()
|
||||
ws = null
|
||||
@ -167,50 +237,74 @@ function disconnectWebSocket() {
|
||||
}
|
||||
|
||||
async function startListening() {
|
||||
// Failed-state click is a retry, not a recording start. The button label
|
||||
// already says "Retry" via pttLabel, so this matches the user's intent.
|
||||
if (state.value === 'failed') {
|
||||
retryConnection()
|
||||
return
|
||||
}
|
||||
if (state.value !== 'idle') return
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
audioChunks = []
|
||||
mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' })
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
if (event.data.size > 0) {
|
||||
audioChunks.push(event.data)
|
||||
}
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
// Stop all tracks
|
||||
stream.getTracks().forEach(t => t.stop())
|
||||
|
||||
if (audioChunks.length === 0) {
|
||||
state.value = 'idle'
|
||||
return
|
||||
}
|
||||
|
||||
// Combine chunks and send
|
||||
const audioBlob = new Blob(audioChunks, { type: 'audio/webm' })
|
||||
const arrayBuffer = await audioBlob.arrayBuffer()
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(arrayBuffer)
|
||||
state.value = 'processing'
|
||||
} else {
|
||||
state.value = 'idle'
|
||||
}
|
||||
}
|
||||
|
||||
mediaRecorder.start()
|
||||
// Web Audio API + manual WAV encode (utils/wavEncoder.ts) — replaces
|
||||
// MediaRecorder/WebM. DashScope Paraformer rejects webm; WAV is the
|
||||
// lowest common denominator every STT provider accepts.
|
||||
//
|
||||
// Reuse the warmed-up recorder so we skip the permission dialog if
|
||||
// it was successfully acquired in onMounted. If warm-up failed (or
|
||||
// is still pending), fall back to a fresh recorder.
|
||||
recorder = warmRecorder ?? new WavRecorder()
|
||||
warmRecorder = null // ownership transferred for the duration of recording
|
||||
await recorder.start()
|
||||
state.value = 'listening'
|
||||
} catch {
|
||||
console.debug('[TalkMode] listening started')
|
||||
} catch (err) {
|
||||
console.warn('[TalkMode] startListening failed', err)
|
||||
ElMessage.error(t('talk.micError'))
|
||||
state.value = 'idle'
|
||||
recorder = null
|
||||
}
|
||||
}
|
||||
|
||||
function stopListening() {
|
||||
if (state.value !== 'listening' || !mediaRecorder) return
|
||||
mediaRecorder.stop()
|
||||
async function stopListening() {
|
||||
// Belt-and-braces: also stop when recorder exists but state hasn't caught
|
||||
// up (race with the slow path of startListening). Without this, a fast
|
||||
// press-then-release leaves the recorder running invisibly.
|
||||
if (!recorder) {
|
||||
if (state.value === 'listening') state.value = 'idle'
|
||||
return
|
||||
}
|
||||
const result = await recorder.stop()
|
||||
recorder = null
|
||||
// Re-warm for the next press so subsequent PTTs also skip permission.
|
||||
warmRecorder = new WavRecorder()
|
||||
warmRecorder.warmUp().catch(() => {})
|
||||
|
||||
if (!result) {
|
||||
// No audio captured — the most common cause is ScriptProcessor never
|
||||
// firing (suspended AudioContext) or the recording was so short no
|
||||
// sample buffer landed. Surface a clear hint instead of a silent idle.
|
||||
console.warn('[TalkMode] stop returned no audio (recording too short or context suspended)')
|
||||
ElMessage.warning(t('talk.tooShort') || '录音过短，请按住按钮多说几秒')
|
||||
state.value = 'idle'
|
||||
return
|
||||
}
|
||||
const arrayBuffer = await result.blob.arrayBuffer()
|
||||
console.debug('[TalkMode] stop produced wav',
|
||||
'bytes=', arrayBuffer.byteLength,
|
||||
'duration=', result.durationSeconds, 's',
|
||||
'wsReadyState=', ws?.readyState)
|
||||
if (ws && ws.readyState === WebSocket.OPEN) {
|
||||
ws.send(arrayBuffer)
|
||||
state.value = 'processing'
|
||||
} else {
|
||||
// Distinct from the empty-recording case: capture worked, but the WS
|
||||
// dropped before we got here. Tell the user instead of silently going
|
||||
// idle — they'd otherwise blame the mic.
|
||||
console.warn('[TalkMode] WS not open at stop time, readyState=', ws?.readyState)
|
||||
ElMessage.error(t('talk.connectionError'))
|
||||
state.value = 'idle'
|
||||
}
|
||||
}
|
||||
|
||||
async function playAudio(blob: Blob) {
|
||||
|
||||
@ -485,7 +485,7 @@ export default {
|
||||
sttProvider: 'Select preferred STT provider. Auto mode picks the first available one.',
|
||||
sttFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
|
||||
openaiSttInfo: 'Reuses OpenAI API Key from Model Management. Whisper model, supports multilingual auto-detection.',
|
||||
dashscopeSttInfo: 'Reuses DashScope API Key from Model Management. Paraformer model, excellent Chinese recognition.',
|
||||
dashscopeSttInfo: 'Reuses DashScope API Key from Model Management. Paraformer Realtime over WebSocket — strong Chinese recognition, sub-second latency.',
|
||||
musicEnabled: 'Enable to let Agent use the music generation tool. Google Lyria reuses Google Key.',
|
||||
musicProvider: 'Select preferred music provider. Auto mode prioritizes Google Lyria.',
|
||||
musicFallbackEnabled: 'Automatically try other configured providers if the preferred one fails.',
|
||||
@ -1817,16 +1817,19 @@ export default {
|
||||
talk: {
|
||||
title: 'Talk Mode',
|
||||
ready: 'Ready',
|
||||
connecting: 'Connecting...',
|
||||
listening: 'Listening...',
|
||||
processing: 'Thinking...',
|
||||
speaking: 'Speaking...',
|
||||
holdToTalk: 'Hold to talk',
|
||||
releaseToSend: 'Release to send',
|
||||
retry: 'Tap to retry',
|
||||
you: 'You',
|
||||
ai: 'AI',
|
||||
micError: 'Cannot access microphone',
|
||||
connectionError: 'Connection error',
|
||||
connectionError: 'Connection error — tap the button to retry',
|
||||
playbackError: 'Audio playback failed',
|
||||
tooShort: 'Recording too short — hold the button while speaking',
|
||||
},
|
||||
browser: {
|
||||
timeline: {
|
||||
|
||||
@ -481,7 +481,7 @@ export default {
|
||||
sttProvider: '选择首选 STT Provider，auto 模式自动选择可用的 Provider。',
|
||||
sttFallbackEnabled: '首选 Provider 失败时自动尝试其他已配置的 Provider。',
|
||||
openaiSttInfo: '复用模型管理中的 OpenAI API Key。使用 Whisper 模型，支持多语言自动识别。',
|
||||
dashscopeSttInfo: '复用模型管理中的 DashScope API Key。使用 Paraformer 模型，中文识别效果优秀。',
|
||||
dashscopeSttInfo: '复用模型管理中的 DashScope API Key。使用 Paraformer Realtime（WebSocket 流式），中文识别效果优秀，亚秒级延迟。',
|
||||
// 音乐生成
|
||||
musicEnabled: '开启后 Agent 可通过 music_generate 工具生成音乐。Google Lyria 复用 Google Key。',
|
||||
musicProvider: '选择首选音乐 Provider，auto 模式优先使用 Google Lyria。',
|
||||
@ -1827,16 +1827,19 @@ export default {
|
||||
talk: {
|
||||
title: '语音模式',
|
||||
ready: '就绪',
|
||||
connecting: '连接中...',
|
||||
listening: '收听中...',
|
||||
processing: '思考中...',
|
||||
speaking: '回复中...',
|
||||
holdToTalk: '按住说话',
|
||||
releaseToSend: '松开发送',
|
||||
retry: '点击重新连接',
|
||||
you: '你',
|
||||
ai: 'AI',
|
||||
micError: '无法访问麦克风',
|
||||
connectionError: '连接错误',
|
||||
connectionError: '连接错误，点击按钮重试',
|
||||
playbackError: '音频播放失败',
|
||||
tooShort: '录音过短，请按住按钮多说几秒',
|
||||
},
|
||||
browser: {
|
||||
timeline: {
|
||||
|
||||
277
mateclaw-ui/src/utils/wavEncoder.ts
Normal file
277
mateclaw-ui/src/utils/wavEncoder.ts
Normal file
@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Browser-side recorder that captures microphone audio via the Web Audio API
|
||||
* and encodes it directly to a 16-bit PCM WAV blob.
|
||||
*
|
||||
* <p>Why this exists: MediaRecorder produces WebM/Opus, which DashScope
|
||||
* Paraformer rejects (it accepts wav/mp3/m4a/flac/aac/amr/ogg-Vorbis only).
|
||||
* OpenAI Whisper claims WebM support but is finicky with the codecs string
|
||||
* MediaRecorder picks. WAV is the lowest common denominator that every STT
|
||||
* provider accepts without server-side transcoding.
|
||||
*
|
||||
* <p>Trade-off: WAV files are ~10x larger than Opus. For typical
|
||||
* conversational STT (5-30s clips at 16 kHz / 16-bit / mono) that's
|
||||
* 160 KB - 1 MB — fine to send over a websocket.
|
||||
*
|
||||
* <p>Reference: openclaw {@code src/media/audio-transcode.ts} solves the same
|
||||
* problem with server-side ffmpeg. Doing it client-side avoids the ffmpeg
|
||||
* dependency on the MateClaw server and works the same way.
|
||||
*/
|
||||
|
||||
/** Target sample rate for recordings. 16 kHz is the standard for STT models. */
|
||||
const TARGET_SAMPLE_RATE = 16_000;
|
||||
|
||||
/** WAV / PCM constants. */
|
||||
const NUM_CHANNELS = 1;
|
||||
const BITS_PER_SAMPLE = 16;
|
||||
const RIFF_HEADER_SIZE = 44;
|
||||
|
||||
/**
|
||||
* Result of a finished recording.
|
||||
*/
|
||||
export interface WavRecording {
|
||||
/** WAV-encoded audio bytes ready to upload. */
|
||||
blob: Blob;
|
||||
/** Convenience MIME type — always "audio/wav". */
|
||||
mimeType: 'audio/wav';
|
||||
/** Recording duration in seconds (rounded to ms). */
|
||||
durationSeconds: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Live microphone recorder backed by Web Audio API. Call {@link start} once,
|
||||
* later call {@link stop} to receive a {@link WavRecording}. After stop the
|
||||
* underlying MediaStream tracks are released.
|
||||
*
|
||||
* <p>The recorder captures Float32 samples from a {@code ScriptProcessorNode},
|
||||
* accumulates them, then encodes a 16-bit PCM mono WAV in {@link stop}.
|
||||
* ScriptProcessorNode is deprecated in favour of AudioWorklet, but it's still
|
||||
* universally supported and zero-config — sufficient for short STT clips.
|
||||
*
|
||||
* <p>Browser support: every browser MateClaw targets ships AudioContext +
|
||||
* ScriptProcessorNode. Safari requires a user gesture to call
|
||||
* {@code getUserMedia}; the caller should already be doing this.
|
||||
*/
|
||||
export class WavRecorder {
|
||||
private audioContext: AudioContext | null = null;
|
||||
private mediaStream: MediaStream | null = null;
|
||||
private source: MediaStreamAudioSourceNode | null = null;
|
||||
private processor: ScriptProcessorNode | null = null;
|
||||
/** Silent sink node — keeps the processor graph alive without echoing through speakers. */
|
||||
private silentSink: GainNode | null = null;
|
||||
private chunks: Float32Array[] = [];
|
||||
/** Sample rate the AudioContext is actually running at — may differ from TARGET_SAMPLE_RATE. */
|
||||
private inputSampleRate = TARGET_SAMPLE_RATE;
|
||||
private startTimeMs = 0;
|
||||
|
||||
/**
|
||||
* Pre-acquire microphone permission and warm the audio graph without
|
||||
* starting capture. Called when the TalkMode modal opens so the
|
||||
* press-and-hold gesture later doesn't race a first-time permission
|
||||
* dialog (the dialog steals focus → mouseup fires on the dialog →
|
||||
* stopListening never runs → recording is stuck on forever).
|
||||
*
|
||||
* <p>Safe to call multiple times. Subsequent calls return immediately.
|
||||
*/
|
||||
async warmUp(): Promise<void> {
|
||||
if (this.mediaStream) return;
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Begin capturing from the microphone. Throws if mic access is denied.
|
||||
* Already-started recorders are idempotent — calling start twice is a no-op.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.audioContext) return;
|
||||
|
||||
// sampleRate hint: browsers honour it on Chrome/Edge but Safari may
|
||||
// ignore it and run at the device default. We resample manually in
|
||||
// stop() to be safe.
|
||||
const ctx = new AudioContext();
|
||||
this.audioContext = ctx;
|
||||
this.inputSampleRate = ctx.sampleRate;
|
||||
// Reuse the warmed-up stream when present so we skip the permission
|
||||
// dialog on the press-and-hold path.
|
||||
if (!this.mediaStream) {
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
}
|
||||
// Modern Chromium AudioContexts created from a user gesture may still
|
||||
// arrive suspended if mic permission was prompted asynchronously.
|
||||
// Force-resume so onaudioprocess actually fires.
|
||||
if (ctx.state === 'suspended') {
|
||||
await ctx.resume();
|
||||
}
|
||||
this.source = ctx.createMediaStreamSource(this.mediaStream);
|
||||
this.processor = ctx.createScriptProcessor(4096, 1, 1);
|
||||
this.processor.onaudioprocess = (e) => {
|
||||
const channel = e.inputBuffer.getChannelData(0);
|
||||
// Defensive copy — the underlying buffer is reused on the next callback.
|
||||
this.chunks.push(new Float32Array(channel));
|
||||
};
|
||||
// Wire source → processor → silent gain → destination. The gain=0
|
||||
// node muzzles the echo through the speakers but keeps the chain
|
||||
// attached to destination, which Chrome requires to fire
|
||||
// onaudioprocess. Connecting processor directly to destination
|
||||
// would echo the mic input back through speakers (feedback) AND
|
||||
// some browsers stop calling onaudioprocess if they decide the
|
||||
// chain "produces no audible output" — the explicit GainNode
|
||||
// makes that decision unambiguous.
|
||||
const silentSink = ctx.createGain();
|
||||
silentSink.gain.value = 0;
|
||||
this.silentSink = silentSink;
|
||||
this.source.connect(this.processor);
|
||||
this.processor.connect(silentSink);
|
||||
silentSink.connect(ctx.destination);
|
||||
this.startTimeMs = Date.now();
|
||||
// Diagnostic — paste from devtools console when the recording silently
|
||||
// produces 0 bytes. Includes sample rate so we can confirm Safari
|
||||
// is at 44.1kHz vs Chrome's 48kHz.
|
||||
console.debug('[WavRecorder] started',
|
||||
'sampleRate=', ctx.sampleRate,
|
||||
'state=', ctx.state);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop capturing, encode the buffered audio to WAV, and release resources.
|
||||
* Returns null when nothing was captured (e.g. start failed silently).
|
||||
*/
|
||||
async stop(): Promise<WavRecording | null> {
|
||||
if (!this.audioContext) return null;
|
||||
|
||||
const durationSeconds = Math.round((Date.now() - this.startTimeMs)) / 1000;
|
||||
const sampleRate = this.inputSampleRate;
|
||||
const collected = this.chunks;
|
||||
|
||||
this.processor?.disconnect();
|
||||
this.source?.disconnect();
|
||||
this.silentSink?.disconnect();
|
||||
this.mediaStream?.getTracks().forEach((t) => t.stop());
|
||||
await this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
this.mediaStream = null;
|
||||
this.source = null;
|
||||
this.processor = null;
|
||||
this.silentSink = null;
|
||||
this.chunks = [];
|
||||
|
||||
// Diagnostic — chunk count + total samples. If chunks=0, the
|
||||
// ScriptProcessor never fired (suspended context, browser
|
||||
// optimised the graph away, etc). With this log the user can paste
|
||||
// the devtools output and we can tell from a single line.
|
||||
const totalSamples = collected.reduce((n, c) => n + c.length, 0);
|
||||
console.debug('[WavRecorder] stopped',
|
||||
'durationSec=', durationSeconds,
|
||||
'chunks=', collected.length,
|
||||
'totalSamples=', totalSamples);
|
||||
|
||||
if (collected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const merged = mergeFloat32(collected);
|
||||
// Resample to 16 kHz if we captured at a higher rate (Safari often runs
|
||||
// the AudioContext at 44.1 kHz). 16 kHz is what Whisper / Paraformer
|
||||
// expect, and shrinks the WAV by ~3x at no quality cost for speech.
|
||||
const resampled = sampleRate === TARGET_SAMPLE_RATE
|
||||
? merged
|
||||
: downsample(merged, sampleRate, TARGET_SAMPLE_RATE);
|
||||
const blob = encodeWav(resampled, TARGET_SAMPLE_RATE);
|
||||
return { blob, mimeType: 'audio/wav', durationSeconds };
|
||||
}
|
||||
|
||||
/** True when the recorder is actively capturing. */
|
||||
isActive(): boolean {
|
||||
return this.audioContext !== null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the warmed-up MediaStream without starting a recording.
|
||||
* Pair with {@link warmUp} on modal close.
|
||||
*/
|
||||
releaseWarmUp(): void {
|
||||
if (this.audioContext) return; // active recording owns the stream
|
||||
this.mediaStream?.getTracks().forEach((t) => t.stop());
|
||||
this.mediaStream = null;
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------ */
|
||||
/* Internals */
|
||||
/* ------------------------------------------------------------------ */
|
||||
|
||||
function mergeFloat32(chunks: Float32Array[]): Float32Array {
|
||||
let total = 0;
|
||||
for (const c of chunks) total += c.length;
|
||||
const out = new Float32Array(total);
|
||||
let offset = 0;
|
||||
for (const c of chunks) {
|
||||
out.set(c, offset);
|
||||
offset += c.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decimating linear-interpolation resampler. Adequate for 16 kHz speech —
|
||||
* the accuracy gap vs polyphase resampling is inaudible to Whisper /
|
||||
* Paraformer at the input sample rates we see in practice (44.1k → 16k).
|
||||
*/
|
||||
function downsample(samples: Float32Array, fromRate: number, toRate: number): Float32Array {
|
||||
if (fromRate === toRate) return samples;
|
||||
const ratio = fromRate / toRate;
|
||||
const newLength = Math.floor(samples.length / ratio);
|
||||
const out = new Float32Array(newLength);
|
||||
for (let i = 0; i < newLength; i++) {
|
||||
const srcIndex = i * ratio;
|
||||
const lo = Math.floor(srcIndex);
|
||||
const hi = Math.min(lo + 1, samples.length - 1);
|
||||
const frac = srcIndex - lo;
|
||||
out[i] = samples[lo] * (1 - frac) + samples[hi] * frac;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a WAV (RIFF / WAVE) blob from Float32 PCM samples in the [-1.0, 1.0]
|
||||
* range. Output is 16-bit signed PCM, mono.
|
||||
*/
|
||||
function encodeWav(samples: Float32Array, sampleRate: number): Blob {
|
||||
const dataSize = samples.length * (BITS_PER_SAMPLE / 8);
|
||||
const buffer = new ArrayBuffer(RIFF_HEADER_SIZE + dataSize);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
// RIFF header — fixed shape; getting any of these wrong makes the WAV
|
||||
// unreadable by every consumer (browsers, ffmpeg, STT servers).
|
||||
writeAscii(view, 0, 'RIFF');
|
||||
view.setUint32(4, 36 + dataSize, true); // file size minus the 8-byte RIFF preamble
|
||||
writeAscii(view, 8, 'WAVE');
|
||||
|
||||
// fmt chunk
|
||||
writeAscii(view, 12, 'fmt ');
|
||||
view.setUint32(16, 16, true); // PCM fmt chunk size
|
||||
view.setUint16(20, 1, true); // audio format = 1 (PCM)
|
||||
view.setUint16(22, NUM_CHANNELS, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * NUM_CHANNELS * (BITS_PER_SAMPLE / 8), true); // byte rate
|
||||
view.setUint16(32, NUM_CHANNELS * (BITS_PER_SAMPLE / 8), true); // block align
|
||||
view.setUint16(34, BITS_PER_SAMPLE, true);
|
||||
|
||||
// data chunk
|
||||
writeAscii(view, 36, 'data');
|
||||
view.setUint32(40, dataSize, true);
|
||||
|
||||
// PCM samples — clamp to [-1, 1] before scaling to int16 to avoid wrap-around
|
||||
// distortion on hot signals.
|
||||
let offset = RIFF_HEADER_SIZE;
|
||||
for (let i = 0; i < samples.length; i++, offset += 2) {
|
||||
const clamped = Math.max(-1, Math.min(1, samples[i]));
|
||||
const intVal = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff;
|
||||
view.setInt16(offset, intVal, true);
|
||||
}
|
||||
return new Blob([buffer], { type: 'audio/wav' });
|
||||
}
|
||||
|
||||
function writeAscii(view: DataView, offset: number, text: string): void {
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
view.setUint8(offset + i, text.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
@ -28,7 +28,7 @@
|
||||
<select v-model="settings.sttProvider" class="form-input" :disabled="!settings.sttEnabled">
|
||||
<option value="auto">{{ t('settings.sttProviderOptions.auto') }}</option>
|
||||
<option value="openai">OpenAI Whisper</option>
|
||||
<option value="dashscope">DashScope (Paraformer)</option>
|
||||
<option value="dashscope">DashScope (Paraformer Realtime)</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
@ -57,7 +57,7 @@
|
||||
</div>
|
||||
<div class="provider-section">
|
||||
<div class="provider-header">
|
||||
<span class="provider-name">DashScope (Paraformer)</span>
|
||||
<span class="provider-name">DashScope (Paraformer Realtime)</span>
|
||||
<span class="provider-tag">{{ t('settings.sttProviderTags.reuseLlmKey') }}</span>
|
||||
</div>
|
||||
<div class="settings-card"><div class="setting-item"><div class="setting-info"><div class="setting-hint">{{ t('settings.hints.dashscopeSttInfo') }}</div></div></div></div>
|
||||
|
||||
@ -19,6 +19,13 @@ export default defineConfig({
|
||||
'/api': {
|
||||
target: 'http://localhost:18088',
|
||||
changeOrigin: true,
|
||||
// ws:true forwards WebSocket Upgrade requests through to the backend.
|
||||
// Without it Vite serves the GET /api/v1/talk/ws as a regular HTTP
|
||||
// proxy, the Upgrade header gets dropped, and the WS handshake
|
||||
// silently fails — frontend stuck on "Connecting", backend never
|
||||
// sees the connection. TalkMode (STT) lives on this WS so the
|
||||
// whole feature is dead in dev mode without it.
|
||||
ws: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
Loading…
Reference in New Issue
Block a user