opensource/dify
2
0
Fork 0
mirror of https://github.com/langgenius/dify.git synced 2026-07-24 04:58:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(web): support TTS playback in Safari and Firefox (#39444)

Browse Source 
Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
This commit is contained in:
eux 2026-07-23 14:36:52 +08:00 committed by GitHub
parent a3c18c561e
commit 00fe96325d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 667 additions and 172 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
oxlint-suppressions.json
View File

@ -835,14 +835,6 @@
"count": 1
}
},
"web/app/components/base/audio-btn/audio.ts": {
"node-js/prefer-global/buffer": {
"count": 1
},
"typescript/no-explicit-any": {
"count": 3
}
},
"web/app/components/base/audio-gallery/AudioPlayer.tsx": {
"jsx_a11y/media-has-caption": {
"count": 1

20
web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts
View File

@ -12,14 +12,8 @@ type AudioPlayerCtorArgs = [
type MockAudioPlayerInstance = {
setCallback: ReturnType<typeof vi.fn>
pauseAudio: ReturnType<typeof vi.fn>
destroy: ReturnType<typeof vi.fn>
resetMsgId: ReturnType<typeof vi.fn>
cacheBuffers: Array<ArrayBuffer>
sourceBuffer:
| {
abort: ReturnType<typeof vi.fn>
}
| undefined
}
const mockState = vi.hoisted(() => ({
@ -31,10 +25,8 @@ const mockAudioPlayerConstructor = vi.hoisted(() => vi.fn())
const MockAudioPlayer = vi.hoisted(() => {
return class MockAudioPlayerClass {
setCallback = vi.fn()
pauseAudio = vi.fn()
destroy = vi.fn()
resetMsgId = vi.fn()
cacheBuffers = [new ArrayBuffer(1)]
sourceBuffer = { abort: vi.fn() }
constructor(...args: AudioPlayerCtorArgs) {
mockAudioPlayerConstructor(...args)
@ -132,9 +124,7 @@ describe('AudioPlayerManager', () => {
callback,
)
expect(previous!.pauseAudio).toHaveBeenCalledTimes(1)
expect(previous!.cacheBuffers).toEqual([])
expect(previous!.sourceBuffer?.abort).toHaveBeenCalledTimes(1)
expect(previous!.destroy).toHaveBeenCalledTimes(1)
expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2)
expect(next).toBe(mockState.instances[1])
})
@ -144,7 +134,7 @@ describe('AudioPlayerManager', () => {
const callback = vi.fn()
manager.getAudioPlayer('/text-to-audio', false, 'msg-1', 'hello', 'en-US', callback)
const previous = mockState.instances[0]
previous!.pauseAudio.mockImplementation(() => {
previous!.destroy.mockImplementation(() => {
throw new Error('cleanup failure')
})
@ -152,7 +142,7 @@ describe('AudioPlayerManager', () => {
manager.getAudioPlayer('/apps/1/text-to-audio', false, 'msg-2', 'world', 'en-US', callback)
}).not.toThrow()
expect(previous!.pauseAudio).toHaveBeenCalledTimes(1)
expect(previous!.destroy).toHaveBeenCalledTimes(1)
expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2)
})
})

547
web/app/components/base/audio-btn/__tests__/audio.spec.ts
View File

@ -3,15 +3,8 @@ import { waitFor } from '@testing-library/react'
import { AppSourceType } from '@/service/share'
import AudioPlayer from '../audio'
const mockToastNotify = vi.hoisted(() => vi.fn())
const mockTextToAudioStream = vi.hoisted(() => vi.fn())
vi.mock('@langgenius/dify-ui/toast', () => ({
toast: {
error: (message: string) => mockToastNotify({ type: 'error', message }),
},
}))
vi.mock('@/service/share', () => ({
AppSourceType: {
webApp: 'webApp',
@ -22,7 +15,7 @@ vi.mock('@/service/share', () => ({
type AudioEventName =
| 'ended'
| 'paused'
| 'pause'
| 'loaded'
| 'play'
| 'timeupdate'
@ -30,6 +23,7 @@ type AudioEventName =
| 'canplay'
| 'error'
| 'sourceopen'
| 'updateend'
type AudioEventListener = () => void
@ -51,12 +45,31 @@ type AudioResponse = {
class MockSourceBuffer {
updating = false
private listeners: Partial<Record<AudioEventName, AudioEventListener[]>> = {}
addEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {
const listeners = this.listeners[event] || []
listeners.push(listener)
this.listeners[event] = listeners
})
removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {
this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener)
})
appendBuffer = vi.fn((_buffer: ArrayBuffer) => undefined)
abort = vi.fn(() => undefined)
emit(event: AudioEventName) {
const listeners = this.listeners[event] || []
listeners.forEach((listener) => {
listener()
})
}
}
class MockMediaSource {
readyState: 'open' | 'closed' = 'open'
readyState: 'open' | 'closed' | 'ended' = 'closed'
sourceBuffer = new MockSourceBuffer()
private listeners: Partial<Record<AudioEventName, AudioEventListener[]>> = {}
@ -66,10 +79,15 @@ class MockMediaSource {
this.listeners[event] = listeners
})
removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => {
this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener)
})
addSourceBuffer = vi.fn((_contentType: string) => this.sourceBuffer)
endOfStream = vi.fn(() => undefined)
emit(event: AudioEventName) {
if (event === 'sourceopen') this.readyState = 'open'
const listeners = this.listeners[event] || []
listeners.forEach((listener) => {
listener()
@ -110,7 +128,7 @@ class MockAudio {
}
class MockAudioContext {
state: 'running' | 'suspended' = 'running'
state: 'interrupted' | 'running' | 'suspended' = 'running'
destination = {}
connect = vi.fn(() => undefined)
createMediaElementSource = vi.fn((_audio: MockAudio) => ({
@ -121,9 +139,11 @@ class MockAudioContext {
this.state = 'running'
})
suspend = vi.fn(() => {
suspend = vi.fn(async () => {
this.state = 'suspended'
})
close = vi.fn(async () => undefined)
}
const testState = {
@ -133,6 +153,8 @@ const testState = {
}
class MockMediaSourceCtor extends MockMediaSource {
static isTypeSupported = vi.fn(() => true)
constructor() {
super()
testState.mediaSources.push(this)
@ -156,6 +178,7 @@ class MockAudioContextCtor extends MockAudioContext {
const originalAudio = globalThis.Audio
const originalAudioContext = globalThis.AudioContext
const originalCreateObjectURL = globalThis.URL.createObjectURL
const originalRevokeObjectURL = globalThis.URL.revokeObjectURL
const originalMediaSource = window.MediaSource
const originalManagedMediaSource = window.ManagedMediaSource
@ -192,6 +215,7 @@ describe('AudioPlayer', () => {
testState.mediaSources = []
testState.audios = []
testState.audioContexts = []
MockMediaSourceCtor.isTypeSupported.mockReturnValue(true)
Object.defineProperty(globalThis, 'Audio', {
configurable: true,
@ -208,6 +232,11 @@ describe('AudioPlayer', () => {
writable: true,
value: vi.fn(() => 'blob:mock-url'),
})
Object.defineProperty(globalThis.URL, 'revokeObjectURL', {
configurable: true,
writable: true,
value: vi.fn(),
})
setMediaSourceSupport({ mediaSource: true, managedMediaSource: false })
})
@ -228,6 +257,11 @@ describe('AudioPlayer', () => {
writable: true,
value: originalCreateObjectURL,
})
Object.defineProperty(globalThis.URL, 'revokeObjectURL', {
configurable: true,
writable: true,
value: originalRevokeObjectURL,
})
Object.defineProperty(window, 'MediaSource', {
configurable: true,
writable: true,
@ -256,7 +290,7 @@ describe('AudioPlayer', () => {
expect(audioContext!.connect).toHaveBeenCalledTimes(1)
})
it('should notify unsupported browser when no MediaSource implementation exists', () => {
it('should use complete-audio fallback when no MediaSource implementation exists', () => {
setMediaSourceSupport({ mediaSource: false, managedMediaSource: false })
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
@ -264,12 +298,22 @@ describe('AudioPlayer', () => {
expect(player.mediaSource).toBeNull()
expect(audio!.src).toBe('')
expect(mockToastNotify).toHaveBeenCalledTimes(1)
expect(mockToastNotify).toHaveBeenCalledWith(
expect.objectContaining({
type: 'error',
}),
)
expect(audio!.autoplay).toBe(false)
expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()
})
it('should use complete-audio fallback when MP3 MediaSource is unsupported', () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const audio = testState.audios[0]
expect(MockMediaSourceCtor.isTypeSupported).toHaveBeenCalledWith('audio/mpeg')
expect(player.mediaSource).toBeNull()
expect(testState.mediaSources).toHaveLength(0)
expect(audio!.src).toBe('')
expect(audio!.autoplay).toBe(false)
expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()
})
it('should configure fallback audio controls when ManagedMediaSource is used', () => {
@ -283,6 +327,17 @@ describe('AudioPlayer', () => {
expect(audio!.disableRemotePlayback).toBe(true)
expect(audio!.controls).toBe(true)
})
it('should configure ManagedMediaSource when both media source implementations exist', () => {
setMediaSourceSupport({ mediaSource: true, managedMediaSource: true })
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn())
const audio = testState.audios[0]
expect(player.mediaSource).not.toBeNull()
expect(audio!.disableRemotePlayback).toBe(true)
expect(audio!.controls).toBe(true)
})
})
describe('event wiring', () => {
@ -294,7 +349,7 @@ describe('AudioPlayer', () => {
audio!.emit('play')
audio!.emit('ended')
audio!.emit('error')
audio!.emit('paused')
audio!.emit('pause')
audio!.emit('loaded')
audio!.emit('timeupdate')
audio!.emit('loadeddate')
@ -354,6 +409,7 @@ describe('AudioPlayer', () => {
})
it('should emit error callback and reset load flag when stream response status is not 200', async () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
const callback = vi.fn()
mockTextToAudioStream.mockResolvedValue(
makeAudioResponse(500, [{ value: new Uint8Array([1]), done: true }]),
@ -366,25 +422,171 @@ describe('AudioPlayer', () => {
expect(callback).toHaveBeenCalledWith('error')
})
expect(player.isLoadData).toBe(false)
expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled()
expect(testState.audios[0]!.play).not.toHaveBeenCalled()
})
it('should resume and play immediately when playAudio is called in suspended loaded state', async () => {
it('should play a complete MP3 blob when MediaSource does not support audio/mpeg', async () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
const callback = vi.fn()
mockTextToAudioStream.mockResolvedValue(
makeAudioResponse(200, [
{ value: new Uint8Array([1, 2]), done: false },
{ value: new Uint8Array([3, 4]), done: true },
]),
)
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)
const audio = testState.audios[0]
player.playAudio()
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))
expect(player.mediaSource).toBeNull()
expect(player.cacheBuffers).toHaveLength(0)
expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1)
const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob
expect(audioBlob).toBeInstanceOf(Blob)
expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 })
expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4]))
expect(audio!.src).toBe('blob:mock-url')
expect(callback).toHaveBeenCalledWith('play')
})
it('should wait for the complete MP3 before retrying playback without MediaSource', async () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
let resolveResponse: ((response: AudioResponse) => void) | undefined
mockTextToAudioStream.mockImplementationOnce(
() =>
new Promise<AudioResponse>((resolve) => {
resolveResponse = resolve
}),
)
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn())
const audio = testState.audios[0]
player.playAudio()
player.playAudio()
expect(audio!.play).not.toHaveBeenCalled()
resolveResponse?.(makeAudioResponse(200, [{ value: new Uint8Array([1, 2]), done: true }]))
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))
})
it.each(['suspended', 'interrupted'] as const)(
'should resume and play immediately when playAudio is called in %s loaded state',
async (audioContextState) => {
const callback = vi.fn()
const player = new AudioPlayer(
'/text-to-audio',
false,
'msg-1',
'hello',
undefined,
callback,
)
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
player.isLoadData = true
audioContext!.state = audioContextState
player.playAudio()
await waitFor(() => {
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
})
},
)
it('should request media playback before a suspended audio context finishes resuming', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
let resolveResume: (() => void) | undefined
player.isLoadData = true
audioContext!.state = 'suspended'
audioContext!.resume.mockImplementationOnce(
() =>
new Promise<void>((resolve) => {
resolveResume = () => {
audioContext!.state = 'running'
resolve()
}
}),
)
player.playAudio()
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).not.toHaveBeenCalledWith('play')
resolveResume?.()
await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))
})
it.each(['suspended', 'interrupted'] as const)(
'should resume a %s audio context when the media element is still playing',
async (audioContextState) => {
const callback = vi.fn()
const player = new AudioPlayer(
'/text-to-audio',
false,
'msg-1',
'hello',
undefined,
callback,
)
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
player.isLoadData = true
audio!.paused = false
audioContext!.state = audioContextState
player.playAudio()
await waitFor(() => {
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
})
expect(audio!.play).not.toHaveBeenCalled()
},
)
it('should report an error when the audio context remains interrupted and allow retry', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
player.isLoadData = true
audio!.paused = false
audioContext!.state = 'suspended'
audioContext!.resume.mockImplementationOnce(async () => {
audioContext!.state = 'interrupted'
})
player.playAudio()
await Promise.resolve()
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))
expect(callback).not.toHaveBeenCalledWith('play')
audioContext!.resume.mockImplementationOnce(async () => {
audioContext!.state = 'running'
})
player.playAudio()
await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))
expect(audioContext!.resume).toHaveBeenCalledTimes(2)
expect(audio!.play).not.toHaveBeenCalled()
})
it('should play ended audio when data is already loaded', () => {
it('should play ended audio when data is already loaded', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)
const audio = testState.audios[0]
@ -395,11 +597,13 @@ describe('AudioPlayer', () => {
audio!.ended = true
player.playAudio()
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
await waitFor(() => {
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
})
})
it('should only emit play callback without replaying when loaded audio is already playing', () => {
it('should report loaded audio that is already playing without replaying it', () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback)
const audio = testState.audios[0]
@ -407,6 +611,7 @@ describe('AudioPlayer', () => {
player.isLoadData = true
audioContext!.state = 'running'
audio!.paused = false
audio!.ended = false
player.playAudio()
@ -451,22 +656,20 @@ describe('AudioPlayer', () => {
})
it('should end stream without playback when playAudioWithAudio receives empty content', async () => {
vi.useFakeTimers()
try {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const mediaSource = testState.mediaSources[0]
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const mediaSource = testState.mediaSources[0]
await player.playAudioWithAudio('', true)
await vi.advanceTimersByTimeAsync(40)
await player.playAudioWithAudio('', true)
expect(player.isLoadData).toBe(false)
expect(player.cacheBuffers).toHaveLength(0)
expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)
expect(callback).not.toHaveBeenCalledWith('play')
} finally {
vi.useRealTimers()
}
expect(player.isLoadData).toBe(false)
expect(player.cacheBuffers).toHaveLength(0)
expect(mediaSource!.endOfStream).not.toHaveBeenCalled()
mediaSource!.emit('sourceopen')
expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)
expect(callback).not.toHaveBeenCalledWith('play')
})
it('should decode base64 and start playback when playAudioWithAudio is called with playable content', async () => {
@ -479,8 +682,8 @@ describe('AudioPlayer', () => {
mediaSource!.emit('sourceopen')
audio!.paused = true
audioContext!.state = 'suspended'
await player.playAudioWithAudio(audioBase64, true)
await Promise.resolve()
expect(player.isLoadData).toBe(true)
expect(player.cacheBuffers).toHaveLength(0)
@ -488,9 +691,11 @@ describe('AudioPlayer', () => {
const appendedAudioData = mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0]
expect(appendedAudioData).toBeInstanceOf(ArrayBuffer)
expect(appendedAudioData.byteLength).toBeGreaterThan(0)
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
await waitFor(() => {
expect(audioContext!.resume).toHaveBeenCalledTimes(1)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
})
})
it('should skip playback when playAudioWithAudio is called with play=false', async () => {
@ -507,6 +712,88 @@ describe('AudioPlayer', () => {
expect(callback).not.toHaveBeenCalledWith('play')
})
it('should combine automatic TTS chunks into a playable MP3 blob without MediaSource', async () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const audio = testState.audios[0]
await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)
await player.playAudioWithAudio(Buffer.from([3, 4]).toString('base64'), true)
expect(audio!.play).not.toHaveBeenCalled()
expect(player.cacheBuffers).toHaveLength(2)
await player.playAudioWithAudio('', false)
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))
expect(player.cacheBuffers).toHaveLength(0)
expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1)
const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob
expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 })
expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4]))
expect(audio!.src).toBe('blob:mock-url')
expect(callback).toHaveBeenCalledWith('play')
})
it('should not start fallback playback after it is paused while buffering', async () => {
MockMediaSourceCtor.isTypeSupported.mockReturnValue(false)
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn())
const audio = testState.audios[0]
await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)
player.pauseAudio()
await player.playAudioWithAudio('', false)
expect(audio!.autoplay).toBe(false)
expect(audio!.play).not.toHaveBeenCalled()
expect(audio!.src).toBe('blob:mock-url')
})
it('should fall back to a complete MP3 when addSourceBuffer throws', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const mediaSource = testState.mediaSources[0]
const audio = testState.audios[0]
mediaSource!.addSourceBuffer.mockImplementationOnce(() => {
throw new DOMException('Unsupported type', 'NotSupportedError')
})
mediaSource!.emit('sourceopen')
await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)
await player.playAudioWithAudio('', false)
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))
expect(player.mediaSource).toBeNull()
expect(audio!.autoplay).toBe(false)
expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url')
expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2)
})
it('should complete buffered fallback when addSourceBuffer throws after stream end', async () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn())
const mediaSource = testState.mediaSources[0]
const audio = testState.audios[0]
mediaSource!.addSourceBuffer.mockImplementationOnce(() => {
throw new DOMException('Unsupported type', 'NotSupportedError')
})
await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true)
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1))
await player.playAudioWithAudio('', false)
audio!.paused = true
mediaSource!.emit('sourceopen')
await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(2))
expect(player.mediaSource).toBeNull()
expect(player.cacheBuffers).toHaveLength(0)
expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2)
const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[1]![0] as Blob
expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 2 })
expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2]))
})
it('should play immediately for ended audio in playAudioWithAudio', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
@ -517,7 +804,7 @@ describe('AudioPlayer', () => {
await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))
})
it('should not replay when played list exists in playAudioWithAudio', async () => {
@ -534,18 +821,63 @@ describe('AudioPlayer', () => {
expect(callback).not.toHaveBeenCalledWith('play')
})
it('should replay when paused is false and played list is empty in playAudioWithAudio', async () => {
it('should report a play failure and retry without requesting audio again', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const audio = testState.audios[0]
const mediaSource = testState.mediaSources[0]
mockTextToAudioStream.mockResolvedValue(
makeAudioResponse(200, [{ value: undefined, done: true }]),
)
audio!.play.mockRejectedValueOnce(new DOMException('Playback aborted', 'AbortError'))
audio!.paused = false
audio!.ended = false
audio!.played = null
await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true)
mediaSource!.emit('sourceopen')
player.playAudio()
await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))
expect(callback).not.toHaveBeenCalledWith('play')
expect(player.isLoadData).toBe(true)
audio!.play.mockImplementationOnce(async () => {
audio!.paused = false
})
player.playAudio()
await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))
expect(audio!.play).toHaveBeenCalledTimes(2)
expect(mockTextToAudioStream).toHaveBeenCalledTimes(1)
})
it('should report a resume failure and allow playback to be retried', async () => {
const callback = vi.fn()
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback)
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
const mediaSource = testState.mediaSources[0]
mockTextToAudioStream.mockResolvedValue(
makeAudioResponse(200, [{ value: undefined, done: true }]),
)
audioContext!.state = 'suspended'
audioContext!.resume.mockRejectedValueOnce(new DOMException('Not allowed', 'NotAllowedError'))
mediaSource!.emit('sourceopen')
player.playAudio()
await waitFor(() => expect(callback).toHaveBeenCalledWith('error'))
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(callback).toHaveBeenCalledWith('play')
expect(player.isLoadData).toBe(true)
audioContext!.resume.mockImplementationOnce(async () => {
audioContext!.state = 'running'
})
player.playAudio()
await waitFor(() => expect(callback).toHaveBeenCalledWith('play'))
expect(audioContext!.resume).toHaveBeenCalledTimes(2)
expect(audio!.play).toHaveBeenCalledTimes(1)
expect(mockTextToAudioStream).toHaveBeenCalledTimes(1)
})
})
@ -562,7 +894,7 @@ describe('AudioPlayer', () => {
expect(finishStream).toHaveBeenCalledTimes(1)
})
it('should finish stream when receiveAudioData gets empty bytes while source is open', () => {
it('should finish stream when receiveAudioData gets empty bytes', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const finishStream = vi
.spyOn(player as unknown as { finishStream: () => void }, 'finishStream')
@ -586,6 +918,52 @@ describe('AudioPlayer', () => {
expect(player.cacheBuffers.length).toBe(1)
})
it('should preserve audio received before sourceopen and append it once ready', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(
new Uint8Array([1, 2, 3]),
)
expect(player.cacheBuffers).toHaveLength(1)
expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled()
mediaSource!.emit('sourceopen')
expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)
expect(player.cacheBuffers).toHaveLength(0)
})
it('should append queued buffers in order after updateend', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
mediaSource!.emit('sourceopen')
mediaSource!.sourceBuffer.updating = true
const first = new Uint8Array([1])
const second = new Uint8Array([2])
;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(
first,
)
;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(
second,
)
mediaSource!.sourceBuffer.updating = false
mediaSource!.sourceBuffer.emit('updateend')
expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)
expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0])).toEqual(
first,
)
mediaSource!.sourceBuffer.emit('updateend')
expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(2)
expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[1]![0])).toEqual(
second,
)
})
it('should append previously queued buffer before new one when source buffer is idle', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
@ -603,19 +981,68 @@ describe('AudioPlayer', () => {
expect(player.cacheBuffers.length).toBe(1)
})
it('should append cache chunks and end stream when finishStream drains buffers', () => {
vi.useFakeTimers()
it('should end the stream only after the final queued buffer is appended', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
mediaSource!.emit('sourceopen')
mediaSource!.sourceBuffer.updating = false
mediaSource!.sourceBuffer.updating = true
player.cacheBuffers = [new ArrayBuffer(3)]
;(player as unknown as { finishStream: () => void }).finishStream()
vi.advanceTimersByTime(50)
;(player as unknown as { finishStream: () => void }).finishStream()
expect(mediaSource!.endOfStream).not.toHaveBeenCalled()
mediaSource!.sourceBuffer.updating = false
mediaSource!.sourceBuffer.emit('updateend')
expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1)
expect(mediaSource!.endOfStream).not.toHaveBeenCalled()
mediaSource!.sourceBuffer.emit('updateend')
expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)
vi.useRealTimers()
})
it('should end an open stream at most once', () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
mediaSource!.emit('sourceopen')
;(player as unknown as { finishStream: () => void }).finishStream()
;(player as unknown as { finishStream: () => void }).finishStream()
expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1)
})
it.each(['closed', 'ended'] as const)('should not end a %s media source', (readyState) => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
mediaSource!.emit('sourceopen')
mediaSource!.readyState = readyState
;(player as unknown as { finishStream: () => void }).finishStream()
expect(mediaSource!.endOfStream).not.toHaveBeenCalled()
})
it('should stop buffering and release browser resources after destroy', async () => {
const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null)
const mediaSource = testState.mediaSources[0]
const audio = testState.audios[0]
const audioContext = testState.audioContexts[0]
mediaSource!.emit('sourceopen')
player.destroy()
;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData(
new Uint8Array([1]),
)
;(player as unknown as { finishStream: () => void }).finishStream()
mediaSource!.sourceBuffer.emit('updateend')
await Promise.resolve()
expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled()
expect(mediaSource!.endOfStream).not.toHaveBeenCalled()
expect(audio!.pause).toHaveBeenCalledTimes(1)
expect(audioContext!.close).toHaveBeenCalledTimes(1)
expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url')
})
})
})

4
web/app/components/base/audio-btn/audio.player.manager.ts
View File

@ -35,9 +35,7 @@ export class AudioPlayerManager {
} else {
if (this.audioPlayers) {
try {
this.audioPlayers.pauseAudio()
this.audioPlayers.cacheBuffers = []
this.audioPlayers.sourceBuffer?.abort()
this.audioPlayers.destroy()
} catch {}
}

260
web/app/components/base/audio-btn/audio.ts
View File

@ -1,19 +1,19 @@
import { toast } from '@langgenius/dify-ui/toast'
import { AppSourceType, textToAudioStream } from '@/service/share'
const AUDIO_CONTENT_TYPE = 'audio/mpeg'
declare global {
// oxlint-disable-next-line typescript/consistent-type-definitions
interface Window {
ManagedMediaSource: any
ManagedMediaSource?: typeof MediaSource
}
}
export default class AudioPlayer {
mediaSource: MediaSource | null
audio: HTMLAudioElement
audioContext: AudioContext
sourceBuffer?: any
sourceBuffer?: SourceBuffer
cacheBuffers: ArrayBuffer[] = []
pauseTimer: number | null = null
msgId: string | undefined
msgContent: string | null | undefined = null
voice: string | undefined = undefined
@ -21,6 +21,13 @@ export default class AudioPlayer {
url: string
isPublic: boolean
callback: ((event: string) => void) | null
private objectUrl = ''
private streamEnded = false
private endOfStreamCalled = false
private destroyed = false
private playbackPending = false
private playWhenReady = false
private sourceOpenListener?: () => void
constructor(
streamUrl: string,
isPublic: boolean,
@ -37,25 +44,26 @@ export default class AudioPlayer {
this.voice = voice
this.callback = callback
// Compatible with iphone ios17 ManagedMediaSource
const MediaSource = window.ManagedMediaSource || window.MediaSource
if (!MediaSource) {
toast.error(
'Your browser does not support audio streaming, if you are using an iPhone, please update to iOS 17.1 or later.',
)
}
this.mediaSource = MediaSource ? new MediaSource() : null
const MediaSourceConstructor = window.ManagedMediaSource || window.MediaSource
const isManagedMediaSource = Boolean(
window.ManagedMediaSource && MediaSourceConstructor === window.ManagedMediaSource,
)
const supportsStreaming = Boolean(MediaSourceConstructor?.isTypeSupported?.(AUDIO_CONTENT_TYPE))
this.mediaSource =
supportsStreaming && MediaSourceConstructor ? new MediaSourceConstructor() : null
this.audio = new Audio()
this.setCallback(callback)
if (!window.MediaSource) {
if (this.mediaSource && isManagedMediaSource) {
// if use ManagedMediaSource
this.audio.disableRemotePlayback = true
this.audio.controls = true
}
this.audio.src = this.mediaSource ? URL.createObjectURL(this.mediaSource) : ''
this.audio.autoplay = true
this.listenMediaSource(AUDIO_CONTENT_TYPE)
this.objectUrl = this.mediaSource ? URL.createObjectURL(this.mediaSource) : ''
this.audio.src = this.objectUrl
this.audio.autoplay = Boolean(this.mediaSource)
const source = this.audioContext.createMediaElementSource(this.audio)
source.connect(this.audioContext.destination)
this.listenMediaSource('audio/mpeg')
}
public resetMsgId(msgId: string) {
@ -63,10 +71,77 @@ export default class AudioPlayer {
}
private listenMediaSource(contentType: string) {
this.mediaSource?.addEventListener('sourceopen', () => {
if (this.sourceBuffer) return
this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType)
})
this.sourceOpenListener = () => {
if (this.destroyed || this.sourceBuffer) return
try {
this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType)
this.sourceBuffer?.addEventListener('updateend', this.flushBuffers)
this.flushBuffers()
} catch {
this.mediaSource = null
this.audio.autoplay = false
this.releaseObjectUrl()
if (this.streamEnded) this.finishBlobAudio()
}
}
this.mediaSource?.addEventListener('sourceopen', this.sourceOpenListener)
}
private flushBuffers = () => {
if (
this.destroyed ||
!this.sourceBuffer ||
this.sourceBuffer.updating ||
this.mediaSource?.readyState !== 'open'
)
return
const nextBuffer = this.cacheBuffers.shift()
if (nextBuffer) {
this.sourceBuffer.appendBuffer(nextBuffer)
return
}
if (this.streamEnded && !this.endOfStreamCalled) {
this.endOfStreamCalled = true
this.mediaSource.endOfStream()
}
}
private requestPlayback(reportIfPlaying = false) {
if (this.destroyed || this.playbackPending) return
if (!this.isAudioContextPaused() && !this.audio.paused && !this.audio.ended) {
if (reportIfPlaying) this.callback?.('play')
return
}
this.playbackPending = true
void this.resumeAndPlay()
}
private isAudioContextPaused() {
return this.audioContext.state === 'suspended' || this.audioContext.state === 'interrupted'
}
private async resumeAndPlay() {
try {
const pendingOperations: Promise<unknown>[] = []
if (this.isAudioContextPaused()) pendingOperations.push(this.audioContext.resume())
if (this.audio.paused || this.audio.ended) pendingOperations.push(this.audio.play())
await Promise.all(pendingOperations)
if (this.destroyed) return
if (this.isAudioContextPaused()) {
this.callback?.('error')
return
}
if (!this.destroyed) this.callback?.('play')
} catch {
if (!this.destroyed) this.callback?.('error')
} finally {
this.playbackPending = false
}
}
public setCallback(callback: ((event: string) => void) | null) {
@ -80,7 +155,7 @@ export default class AudioPlayer {
false,
)
this.audio.addEventListener(
'paused',
'pause',
() => {
callback('paused')
},
@ -133,7 +208,7 @@ export default class AudioPlayer {
private async loadAudio() {
try {
const audioResponse: any = await textToAudioStream(
const audioResponse = (await textToAudioStream(
this.url,
this.isPublic ? AppSourceType.webApp : AppSourceType.installedApp,
{ content_type: 'audio/mpeg' },
@ -143,19 +218,21 @@ export default class AudioPlayer {
voice: this.voice,
text: this.msgContent,
},
)
)) as Response
if (audioResponse.status !== 200) {
this.isLoadData = false
if (this.callback) this.callback('error')
this.callback?.('error')
return
}
if (!audioResponse.body) throw new Error('Audio response body is missing')
const reader = audioResponse.body.getReader()
while (true) {
const { value, done } = await reader.read()
if (value?.byteLength) this.receiveAudioData(value)
if (done) {
this.receiveAudioData(value)
this.finishStream()
break
}
this.receiveAudioData(value)
}
} catch {
this.isLoadData = false
@ -166,46 +243,29 @@ export default class AudioPlayer {
// play audio
public playAudio() {
if (this.isLoadData) {
if (this.audioContext.state === 'suspended') {
this.audioContext.resume().then((_) => {
this.audio.play()
this.callback?.('play')
})
} else if (this.audio.ended) {
this.audio.play()
this.callback?.('play')
if (!this.mediaSource && !this.objectUrl) {
this.playWhenReady = true
return
}
this.callback?.('play')
this.requestPlayback(true)
} else {
this.isLoadData = true
this.audioContext.resume().then((_) => {
this.audio.play()
this.callback?.('play')
})
this.playWhenReady = true
if (this.mediaSource) this.requestPlayback(true)
else if (this.isAudioContextPaused()) void this.audioContext.resume().catch(() => {})
this.loadAudio()
}
}
private theEndOfStream() {
const endTimer = setInterval(() => {
if (!this.sourceBuffer?.updating) {
this.mediaSource?.endOfStream()
clearInterval(endTimer)
}
}, 10)
}
private finishStream() {
const timer = setInterval(() => {
if (!this.cacheBuffers.length) {
this.theEndOfStream()
clearInterval(timer)
}
if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
const arrayBuffer = this.cacheBuffers.shift()!
this.sourceBuffer?.appendBuffer(arrayBuffer)
}
}, 10)
if (this.destroyed) return
this.streamEnded = true
if (this.mediaSource) {
this.flushBuffers()
return
}
this.finishBlobAudio()
}
public async playAudioWithAudio(audio: string, play = true) {
@ -213,54 +273,82 @@ export default class AudioPlayer {
this.finishStream()
return
}
const audioContent = Buffer.from(audio, 'base64')
this.receiveAudioData(new Uint8Array(audioContent))
const audioContent = Uint8Array.from(atob(audio), (char) => char.charCodeAt(0))
this.receiveAudioData(audioContent)
if (play) {
this.isLoadData = true
if (this.audio.paused) {
this.audioContext.resume().then((_) => {
this.audio.play()
this.callback?.('play')
})
} else if (this.audio.ended) {
this.audio.play()
this.callback?.('play')
} else if (this.audio.played) {
/* empty */
} else {
this.audio.play()
this.callback?.('play')
}
this.playWhenReady = true
if (this.mediaSource) this.requestPlayback()
}
}
public pauseAudio() {
this.playWhenReady = false
this.callback?.('paused')
this.audio.pause()
this.audioContext.suspend()
void this.audioContext.suspend().catch(() => {})
}
private receiveAudioData(unit8Array: Uint8Array) {
public destroy() {
if (this.destroyed) return
this.destroyed = true
this.cacheBuffers = []
this.callback?.('paused')
this.audio.pause()
if (this.sourceOpenListener)
this.mediaSource?.removeEventListener('sourceopen', this.sourceOpenListener)
if (this.sourceBuffer) {
this.sourceBuffer.removeEventListener('updateend', this.flushBuffers)
if (this.mediaSource?.readyState === 'open') {
try {
this.sourceBuffer.abort()
} catch {}
}
}
void this.audioContext.close().catch(() => {})
this.releaseObjectUrl()
}
private receiveAudioData(unit8Array: Uint8Array | undefined) {
if (this.destroyed || this.streamEnded) return
if (!unit8Array) {
this.finishStream()
return
}
const audioData = this.byteArrayToArrayBuffer(unit8Array)
if (!audioData.byteLength) {
if (this.mediaSource?.readyState === 'open') this.finishStream()
this.finishStream()
return
}
if (this.sourceBuffer?.updating) {
this.cacheBuffers.push(audioData)
} else {
if (this.cacheBuffers.length && !this.sourceBuffer?.updating) {
this.cacheBuffers.push(audioData)
const cacheBuffer = this.cacheBuffers.shift()!
this.sourceBuffer?.appendBuffer(cacheBuffer)
} else {
this.sourceBuffer?.appendBuffer(audioData)
}
this.cacheBuffers.push(audioData)
this.flushBuffers()
}
private finishBlobAudio() {
if (!this.cacheBuffers.length) {
if (!this.objectUrl) this.isLoadData = false
return
}
const audioBlob = new Blob(this.cacheBuffers, { type: AUDIO_CONTENT_TYPE })
this.cacheBuffers = []
this.releaseObjectUrl()
this.objectUrl = URL.createObjectURL(audioBlob)
this.audio.src = this.objectUrl
this.isLoadData = true
if (this.playWhenReady) this.requestPlayback()
}
private releaseObjectUrl() {
if (!this.objectUrl) return
URL.revokeObjectURL(this.objectUrl)
this.objectUrl = ''
this.audio.src = ''
}
private byteArrayToArrayBuffer(byteArray: Uint8Array): ArrayBuffer {