import type { Ref } from 'react' import type { VoiceRecorder } from './recorder' import type { SpeechToTextTarget } from './types' import { cn } from '@langgenius/dify-ui/cn' import { useCallback, useEffect, useEffectEvent, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { transcribeAudio } from './api' import s from './index.module.css' import { startVoiceRecorder } from './recorder' const MAX_RECORDING_DURATION = 600 type VoiceInputStatus = 'starting' | 'recording' | 'converting' type VoiceInputProps = { ref?: Ref onConverted: (text: string) => void onCancel: () => void onBeforeTranscribe?: () => Promise onError?: () => void onStartError?: (error: unknown) => void target: SpeechToTextTarget } function VoiceInput({ ref, onCancel, onBeforeTranscribe, onConverted, onError, onStartError, target, }: VoiceInputProps) { const { t } = useTranslation() const recorderRef = useRef(null) const canvasRef = useRef(null) const canvasContextRef = useRef(null) const canvasSizeRef = useRef({ height: 0, width: 0 }) const drawRecordIdRef = useRef(null) const mountedRef = useRef(true) const stopRequestedRef = useRef(false) const cancelledRef = useRef(false) const setupAbortControllerRef = useRef(null) const transcriptionAbortControllerRef = useRef(null) const [duration, setDuration] = useState(0) const [status, setStatus] = useState('starting') const handleRecorderStartError = useEffectEvent((error: unknown) => { onStartError?.(error) onCancel() }) const stopDrawing = useCallback(() => { if (drawRecordIdRef.current !== null) cancelAnimationFrame(drawRecordIdRef.current) drawRecordIdRef.current = null const { height, width } = canvasSizeRef.current canvasContextRef.current?.clearRect(0, 0, width, height) }, []) const drawRecord = useCallback(() => { const recorder = recorderRef.current const context = canvasContextRef.current const { height, width } = canvasSizeRef.current if (!recorder || !context || !height || !width) return const frequencyData = new Uint8Array(recorder.analyser.frequencyBinCount) recorder.analyser.getByteFrequencyData(frequencyData) const lineCount = Math.max(1, Math.floor(width / 3)) const sampleStep = Math.max(1, Math.floor(frequencyData.length / lineCount)) context.clearRect(0, 0, width, height) context.beginPath() for (let index = 0; index < lineCount; index++) { const amplitude = frequencyData[index * sampleStep] ?? 0 const lineHeight = Math.max(1, (amplitude / 255) * height) const x = index * 3 if (context.roundRect) context.roundRect(x, height - lineHeight, 2, lineHeight, [1, 1, 0, 0]) else context.rect(x, height - lineHeight, 2, lineHeight) } context.fill() context.closePath() drawRecordIdRef.current = requestAnimationFrame(drawRecord) }, []) const handleStopRecorder = useCallback(async () => { const recorder = recorderRef.current if (!recorder || status !== 'recording' || stopRequestedRef.current) return stopRequestedRef.current = true setStatus('converting') stopDrawing() let mp3Blob: Blob try { mp3Blob = await recorder.stop() } catch { if (mountedRef.current && !cancelledRef.current) { onError?.() onCancel() } return } if (cancelledRef.current) return try { await onBeforeTranscribe?.() } catch { if (mountedRef.current && !cancelledRef.current) onCancel() return } if (cancelledRef.current) return const file = new File([mp3Blob], 'temp.mp3', { type: 'audio/mp3' }) const abortController = new AbortController() transcriptionAbortControllerRef.current = abortController try { const audioResponse = await transcribeAudio(target, file, abortController.signal) if (mountedRef.current && !cancelledRef.current) onConverted(audioResponse.text) } catch { if (mountedRef.current && !cancelledRef.current) onError?.() } finally { transcriptionAbortControllerRef.current = null if (mountedRef.current && !cancelledRef.current) onCancel() } }, [onBeforeTranscribe, onCancel, onConverted, onError, status, stopDrawing, target]) const handleCancel = useCallback(() => { cancelledRef.current = true setupAbortControllerRef.current?.abort() transcriptionAbortControllerRef.current?.abort() void recorderRef.current?.cancel() onCancel() }, [onCancel]) useEffect(() => { const canvas = canvasRef.current if (!canvas) return const devicePixelRatio = window.devicePixelRatio || 1 const { height, width } = canvas.getBoundingClientRect() canvas.width = devicePixelRatio * width canvas.height = devicePixelRatio * height canvasSizeRef.current = { height, width } const context = canvas.getContext('2d') if (!context) return context.scale(devicePixelRatio, devicePixelRatio) context.fillStyle = 'rgba(209, 224, 255, 1)' canvasContextRef.current = context }, []) useEffect(() => { mountedRef.current = true cancelledRef.current = false let effectCancelled = false const abortController = new AbortController() setupAbortControllerRef.current = abortController void startVoiceRecorder(abortController.signal) .then((recorder) => { if (setupAbortControllerRef.current === abortController) setupAbortControllerRef.current = null if (effectCancelled) { void recorder.cancel() return } recorderRef.current = recorder setStatus('recording') drawRecord() }) .catch((error: unknown) => { if (setupAbortControllerRef.current === abortController) setupAbortControllerRef.current = null if (effectCancelled || abortController.signal.aborted) return handleRecorderStartError(error) }) return () => { effectCancelled = true mountedRef.current = false cancelledRef.current = true abortController.abort() transcriptionAbortControllerRef.current?.abort() stopDrawing() void recorderRef.current?.cancel() } }, [drawRecord, stopDrawing]) useEffect(() => { if (status !== 'recording') return const intervalId = window.setInterval(() => { setDuration((currentDuration) => currentDuration + 1) }, 1000) return () => window.clearInterval(intervalId) }, [status]) useEffect(() => { if (duration >= MAX_RECORDING_DURATION && status === 'recording') void handleStopRecorder() }, [duration, handleStopRecorder, status]) const minutes = Math.floor(duration / 60) .toString() .padStart(2, '0') const seconds = (duration % 60).toString().padStart(2, '0') const isStarting = status === 'starting' const isRecording = status === 'recording' const isConverting = status === 'converting' return (
{(isStarting || isConverting) && (
) } export default VoiceInput