diff --git a/oxlint-suppressions.json b/oxlint-suppressions.json index fc02eaf6753..fb2c46efaf5 100644 --- a/oxlint-suppressions.json +++ b/oxlint-suppressions.json @@ -835,14 +835,6 @@ "count": 1 } }, - "web/app/components/base/audio-btn/audio.ts": { - "node-js/prefer-global/buffer": { - "count": 1 - }, - "typescript/no-explicit-any": { - "count": 3 - } - }, "web/app/components/base/audio-gallery/AudioPlayer.tsx": { "jsx_a11y/media-has-caption": { "count": 1 diff --git a/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts b/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts index a93e7e50383..60023038339 100644 --- a/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts +++ b/web/app/components/base/audio-btn/__tests__/audio.player.manager.spec.ts @@ -12,14 +12,8 @@ type AudioPlayerCtorArgs = [ type MockAudioPlayerInstance = { setCallback: ReturnType - pauseAudio: ReturnType + destroy: ReturnType resetMsgId: ReturnType - cacheBuffers: Array - sourceBuffer: - | { - abort: ReturnType - } - | undefined } const mockState = vi.hoisted(() => ({ @@ -31,10 +25,8 @@ const mockAudioPlayerConstructor = vi.hoisted(() => vi.fn()) const MockAudioPlayer = vi.hoisted(() => { return class MockAudioPlayerClass { setCallback = vi.fn() - pauseAudio = vi.fn() + destroy = vi.fn() resetMsgId = vi.fn() - cacheBuffers = [new ArrayBuffer(1)] - sourceBuffer = { abort: vi.fn() } constructor(...args: AudioPlayerCtorArgs) { mockAudioPlayerConstructor(...args) @@ -132,9 +124,7 @@ describe('AudioPlayerManager', () => { callback, ) - expect(previous!.pauseAudio).toHaveBeenCalledTimes(1) - expect(previous!.cacheBuffers).toEqual([]) - expect(previous!.sourceBuffer?.abort).toHaveBeenCalledTimes(1) + expect(previous!.destroy).toHaveBeenCalledTimes(1) expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2) expect(next).toBe(mockState.instances[1]) }) @@ -144,7 +134,7 @@ describe('AudioPlayerManager', () => { const callback = vi.fn() manager.getAudioPlayer('/text-to-audio', false, 'msg-1', 'hello', 'en-US', callback) const previous = mockState.instances[0] - previous!.pauseAudio.mockImplementation(() => { + previous!.destroy.mockImplementation(() => { throw new Error('cleanup failure') }) @@ -152,7 +142,7 @@ describe('AudioPlayerManager', () => { manager.getAudioPlayer('/apps/1/text-to-audio', false, 'msg-2', 'world', 'en-US', callback) }).not.toThrow() - expect(previous!.pauseAudio).toHaveBeenCalledTimes(1) + expect(previous!.destroy).toHaveBeenCalledTimes(1) expect(mockAudioPlayerConstructor).toHaveBeenCalledTimes(2) }) }) diff --git a/web/app/components/base/audio-btn/__tests__/audio.spec.ts b/web/app/components/base/audio-btn/__tests__/audio.spec.ts index 95cd0ef97c9..ac69433dd5b 100644 --- a/web/app/components/base/audio-btn/__tests__/audio.spec.ts +++ b/web/app/components/base/audio-btn/__tests__/audio.spec.ts @@ -3,15 +3,8 @@ import { waitFor } from '@testing-library/react' import { AppSourceType } from '@/service/share' import AudioPlayer from '../audio' -const mockToastNotify = vi.hoisted(() => vi.fn()) const mockTextToAudioStream = vi.hoisted(() => vi.fn()) -vi.mock('@langgenius/dify-ui/toast', () => ({ - toast: { - error: (message: string) => mockToastNotify({ type: 'error', message }), - }, -})) - vi.mock('@/service/share', () => ({ AppSourceType: { webApp: 'webApp', @@ -22,7 +15,7 @@ vi.mock('@/service/share', () => ({ type AudioEventName = | 'ended' - | 'paused' + | 'pause' | 'loaded' | 'play' | 'timeupdate' @@ -30,6 +23,7 @@ type AudioEventName = | 'canplay' | 'error' | 'sourceopen' + | 'updateend' type AudioEventListener = () => void @@ -51,12 +45,31 @@ type AudioResponse = { class MockSourceBuffer { updating = false + private listeners: Partial> = {} + + addEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => { + const listeners = this.listeners[event] || [] + listeners.push(listener) + this.listeners[event] = listeners + }) + + removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => { + this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener) + }) + appendBuffer = vi.fn((_buffer: ArrayBuffer) => undefined) abort = vi.fn(() => undefined) + + emit(event: AudioEventName) { + const listeners = this.listeners[event] || [] + listeners.forEach((listener) => { + listener() + }) + } } class MockMediaSource { - readyState: 'open' | 'closed' = 'open' + readyState: 'open' | 'closed' | 'ended' = 'closed' sourceBuffer = new MockSourceBuffer() private listeners: Partial> = {} @@ -66,10 +79,15 @@ class MockMediaSource { this.listeners[event] = listeners }) + removeEventListener = vi.fn((event: AudioEventName, listener: AudioEventListener) => { + this.listeners[event] = (this.listeners[event] || []).filter((item) => item !== listener) + }) + addSourceBuffer = vi.fn((_contentType: string) => this.sourceBuffer) endOfStream = vi.fn(() => undefined) emit(event: AudioEventName) { + if (event === 'sourceopen') this.readyState = 'open' const listeners = this.listeners[event] || [] listeners.forEach((listener) => { listener() @@ -110,7 +128,7 @@ class MockAudio { } class MockAudioContext { - state: 'running' | 'suspended' = 'running' + state: 'interrupted' | 'running' | 'suspended' = 'running' destination = {} connect = vi.fn(() => undefined) createMediaElementSource = vi.fn((_audio: MockAudio) => ({ @@ -121,9 +139,11 @@ class MockAudioContext { this.state = 'running' }) - suspend = vi.fn(() => { + suspend = vi.fn(async () => { this.state = 'suspended' }) + + close = vi.fn(async () => undefined) } const testState = { @@ -133,6 +153,8 @@ const testState = { } class MockMediaSourceCtor extends MockMediaSource { + static isTypeSupported = vi.fn(() => true) + constructor() { super() testState.mediaSources.push(this) @@ -156,6 +178,7 @@ class MockAudioContextCtor extends MockAudioContext { const originalAudio = globalThis.Audio const originalAudioContext = globalThis.AudioContext const originalCreateObjectURL = globalThis.URL.createObjectURL +const originalRevokeObjectURL = globalThis.URL.revokeObjectURL const originalMediaSource = window.MediaSource const originalManagedMediaSource = window.ManagedMediaSource @@ -192,6 +215,7 @@ describe('AudioPlayer', () => { testState.mediaSources = [] testState.audios = [] testState.audioContexts = [] + MockMediaSourceCtor.isTypeSupported.mockReturnValue(true) Object.defineProperty(globalThis, 'Audio', { configurable: true, @@ -208,6 +232,11 @@ describe('AudioPlayer', () => { writable: true, value: vi.fn(() => 'blob:mock-url'), }) + Object.defineProperty(globalThis.URL, 'revokeObjectURL', { + configurable: true, + writable: true, + value: vi.fn(), + }) setMediaSourceSupport({ mediaSource: true, managedMediaSource: false }) }) @@ -228,6 +257,11 @@ describe('AudioPlayer', () => { writable: true, value: originalCreateObjectURL, }) + Object.defineProperty(globalThis.URL, 'revokeObjectURL', { + configurable: true, + writable: true, + value: originalRevokeObjectURL, + }) Object.defineProperty(window, 'MediaSource', { configurable: true, writable: true, @@ -256,7 +290,7 @@ describe('AudioPlayer', () => { expect(audioContext!.connect).toHaveBeenCalledTimes(1) }) - it('should notify unsupported browser when no MediaSource implementation exists', () => { + it('should use complete-audio fallback when no MediaSource implementation exists', () => { setMediaSourceSupport({ mediaSource: false, managedMediaSource: false }) const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) @@ -264,12 +298,22 @@ describe('AudioPlayer', () => { expect(player.mediaSource).toBeNull() expect(audio!.src).toBe('') - expect(mockToastNotify).toHaveBeenCalledTimes(1) - expect(mockToastNotify).toHaveBeenCalledWith( - expect.objectContaining({ - type: 'error', - }), - ) + expect(audio!.autoplay).toBe(false) + expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled() + }) + + it('should use complete-audio fallback when MP3 MediaSource is unsupported', () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) + + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const audio = testState.audios[0] + + expect(MockMediaSourceCtor.isTypeSupported).toHaveBeenCalledWith('audio/mpeg') + expect(player.mediaSource).toBeNull() + expect(testState.mediaSources).toHaveLength(0) + expect(audio!.src).toBe('') + expect(audio!.autoplay).toBe(false) + expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled() }) it('should configure fallback audio controls when ManagedMediaSource is used', () => { @@ -283,6 +327,17 @@ describe('AudioPlayer', () => { expect(audio!.disableRemotePlayback).toBe(true) expect(audio!.controls).toBe(true) }) + + it('should configure ManagedMediaSource when both media source implementations exist', () => { + setMediaSourceSupport({ mediaSource: true, managedMediaSource: true }) + + const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn()) + const audio = testState.audios[0] + + expect(player.mediaSource).not.toBeNull() + expect(audio!.disableRemotePlayback).toBe(true) + expect(audio!.controls).toBe(true) + }) }) describe('event wiring', () => { @@ -294,7 +349,7 @@ describe('AudioPlayer', () => { audio!.emit('play') audio!.emit('ended') audio!.emit('error') - audio!.emit('paused') + audio!.emit('pause') audio!.emit('loaded') audio!.emit('timeupdate') audio!.emit('loadeddate') @@ -354,6 +409,7 @@ describe('AudioPlayer', () => { }) it('should emit error callback and reset load flag when stream response status is not 200', async () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) const callback = vi.fn() mockTextToAudioStream.mockResolvedValue( makeAudioResponse(500, [{ value: new Uint8Array([1]), done: true }]), @@ -366,25 +422,171 @@ describe('AudioPlayer', () => { expect(callback).toHaveBeenCalledWith('error') }) expect(player.isLoadData).toBe(false) + expect(globalThis.URL.createObjectURL).not.toHaveBeenCalled() + expect(testState.audios[0]!.play).not.toHaveBeenCalled() }) - it('should resume and play immediately when playAudio is called in suspended loaded state', async () => { + it('should play a complete MP3 blob when MediaSource does not support audio/mpeg', async () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) + const callback = vi.fn() + mockTextToAudioStream.mockResolvedValue( + makeAudioResponse(200, [ + { value: new Uint8Array([1, 2]), done: false }, + { value: new Uint8Array([3, 4]), done: true }, + ]), + ) + + const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback) + const audio = testState.audios[0] + + player.playAudio() + + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1)) + expect(player.mediaSource).toBeNull() + expect(player.cacheBuffers).toHaveLength(0) + expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1) + const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob + expect(audioBlob).toBeInstanceOf(Blob) + expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 }) + expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4])) + expect(audio!.src).toBe('blob:mock-url') + expect(callback).toHaveBeenCalledWith('play') + }) + + it('should wait for the complete MP3 before retrying playback without MediaSource', async () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) + let resolveResponse: ((response: AudioResponse) => void) | undefined + mockTextToAudioStream.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveResponse = resolve + }), + ) + + const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, vi.fn()) + const audio = testState.audios[0] + + player.playAudio() + player.playAudio() + + expect(audio!.play).not.toHaveBeenCalled() + + resolveResponse?.(makeAudioResponse(200, [{ value: new Uint8Array([1, 2]), done: true }])) + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1)) + }) + + it.each(['suspended', 'interrupted'] as const)( + 'should resume and play immediately when playAudio is called in %s loaded state', + async (audioContextState) => { + const callback = vi.fn() + const player = new AudioPlayer( + '/text-to-audio', + false, + 'msg-1', + 'hello', + undefined, + callback, + ) + const audio = testState.audios[0] + const audioContext = testState.audioContexts[0] + + player.isLoadData = true + audioContext!.state = audioContextState + player.playAudio() + + await waitFor(() => { + expect(audioContext!.resume).toHaveBeenCalledTimes(1) + expect(audio!.play).toHaveBeenCalledTimes(1) + expect(callback).toHaveBeenCalledWith('play') + }) + }, + ) + + it('should request media playback before a suspended audio context finishes resuming', async () => { + const callback = vi.fn() + const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback) + const audio = testState.audios[0] + const audioContext = testState.audioContexts[0] + let resolveResume: (() => void) | undefined + + player.isLoadData = true + audioContext!.state = 'suspended' + audioContext!.resume.mockImplementationOnce( + () => + new Promise((resolve) => { + resolveResume = () => { + audioContext!.state = 'running' + resolve() + } + }), + ) + + player.playAudio() + + expect(audioContext!.resume).toHaveBeenCalledTimes(1) + expect(audio!.play).toHaveBeenCalledTimes(1) + expect(callback).not.toHaveBeenCalledWith('play') + + resolveResume?.() + await waitFor(() => expect(callback).toHaveBeenCalledWith('play')) + }) + + it.each(['suspended', 'interrupted'] as const)( + 'should resume a %s audio context when the media element is still playing', + async (audioContextState) => { + const callback = vi.fn() + const player = new AudioPlayer( + '/text-to-audio', + false, + 'msg-1', + 'hello', + undefined, + callback, + ) + const audio = testState.audios[0] + const audioContext = testState.audioContexts[0] + + player.isLoadData = true + audio!.paused = false + audioContext!.state = audioContextState + player.playAudio() + + await waitFor(() => { + expect(audioContext!.resume).toHaveBeenCalledTimes(1) + expect(callback).toHaveBeenCalledWith('play') + }) + expect(audio!.play).not.toHaveBeenCalled() + }, + ) + + it('should report an error when the audio context remains interrupted and allow retry', async () => { const callback = vi.fn() const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback) const audio = testState.audios[0] const audioContext = testState.audioContexts[0] player.isLoadData = true + audio!.paused = false audioContext!.state = 'suspended' + audioContext!.resume.mockImplementationOnce(async () => { + audioContext!.state = 'interrupted' + }) player.playAudio() - await Promise.resolve() - expect(audioContext!.resume).toHaveBeenCalledTimes(1) - expect(audio!.play).toHaveBeenCalledTimes(1) - expect(callback).toHaveBeenCalledWith('play') + await waitFor(() => expect(callback).toHaveBeenCalledWith('error')) + expect(callback).not.toHaveBeenCalledWith('play') + + audioContext!.resume.mockImplementationOnce(async () => { + audioContext!.state = 'running' + }) + player.playAudio() + + await waitFor(() => expect(callback).toHaveBeenCalledWith('play')) + expect(audioContext!.resume).toHaveBeenCalledTimes(2) + expect(audio!.play).not.toHaveBeenCalled() }) - it('should play ended audio when data is already loaded', () => { + it('should play ended audio when data is already loaded', async () => { const callback = vi.fn() const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback) const audio = testState.audios[0] @@ -395,11 +597,13 @@ describe('AudioPlayer', () => { audio!.ended = true player.playAudio() - expect(audio!.play).toHaveBeenCalledTimes(1) - expect(callback).toHaveBeenCalledWith('play') + await waitFor(() => { + expect(audio!.play).toHaveBeenCalledTimes(1) + expect(callback).toHaveBeenCalledWith('play') + }) }) - it('should only emit play callback without replaying when loaded audio is already playing', () => { + it('should report loaded audio that is already playing without replaying it', () => { const callback = vi.fn() const player = new AudioPlayer('/text-to-audio', false, 'msg-1', 'hello', undefined, callback) const audio = testState.audios[0] @@ -407,6 +611,7 @@ describe('AudioPlayer', () => { player.isLoadData = true audioContext!.state = 'running' + audio!.paused = false audio!.ended = false player.playAudio() @@ -451,22 +656,20 @@ describe('AudioPlayer', () => { }) it('should end stream without playback when playAudioWithAudio receives empty content', async () => { - vi.useFakeTimers() - try { - const callback = vi.fn() - const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) - const mediaSource = testState.mediaSources[0] + const callback = vi.fn() + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) + const mediaSource = testState.mediaSources[0] - await player.playAudioWithAudio('', true) - await vi.advanceTimersByTimeAsync(40) + await player.playAudioWithAudio('', true) - expect(player.isLoadData).toBe(false) - expect(player.cacheBuffers).toHaveLength(0) - expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1) - expect(callback).not.toHaveBeenCalledWith('play') - } finally { - vi.useRealTimers() - } + expect(player.isLoadData).toBe(false) + expect(player.cacheBuffers).toHaveLength(0) + expect(mediaSource!.endOfStream).not.toHaveBeenCalled() + + mediaSource!.emit('sourceopen') + + expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1) + expect(callback).not.toHaveBeenCalledWith('play') }) it('should decode base64 and start playback when playAudioWithAudio is called with playable content', async () => { @@ -479,8 +682,8 @@ describe('AudioPlayer', () => { mediaSource!.emit('sourceopen') audio!.paused = true + audioContext!.state = 'suspended' await player.playAudioWithAudio(audioBase64, true) - await Promise.resolve() expect(player.isLoadData).toBe(true) expect(player.cacheBuffers).toHaveLength(0) @@ -488,9 +691,11 @@ describe('AudioPlayer', () => { const appendedAudioData = mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0] expect(appendedAudioData).toBeInstanceOf(ArrayBuffer) expect(appendedAudioData.byteLength).toBeGreaterThan(0) - expect(audioContext!.resume).toHaveBeenCalledTimes(1) - expect(audio!.play).toHaveBeenCalledTimes(1) - expect(callback).toHaveBeenCalledWith('play') + await waitFor(() => { + expect(audioContext!.resume).toHaveBeenCalledTimes(1) + expect(audio!.play).toHaveBeenCalledTimes(1) + expect(callback).toHaveBeenCalledWith('play') + }) }) it('should skip playback when playAudioWithAudio is called with play=false', async () => { @@ -507,6 +712,88 @@ describe('AudioPlayer', () => { expect(callback).not.toHaveBeenCalledWith('play') }) + it('should combine automatic TTS chunks into a playable MP3 blob without MediaSource', async () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) + const callback = vi.fn() + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) + const audio = testState.audios[0] + + await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true) + await player.playAudioWithAudio(Buffer.from([3, 4]).toString('base64'), true) + + expect(audio!.play).not.toHaveBeenCalled() + expect(player.cacheBuffers).toHaveLength(2) + + await player.playAudioWithAudio('', false) + + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1)) + expect(player.cacheBuffers).toHaveLength(0) + expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(1) + const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[0]![0] as Blob + expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 4 }) + expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2, 3, 4])) + expect(audio!.src).toBe('blob:mock-url') + expect(callback).toHaveBeenCalledWith('play') + }) + + it('should not start fallback playback after it is paused while buffering', async () => { + MockMediaSourceCtor.isTypeSupported.mockReturnValue(false) + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn()) + const audio = testState.audios[0] + + await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true) + player.pauseAudio() + await player.playAudioWithAudio('', false) + + expect(audio!.autoplay).toBe(false) + expect(audio!.play).not.toHaveBeenCalled() + expect(audio!.src).toBe('blob:mock-url') + }) + + it('should fall back to a complete MP3 when addSourceBuffer throws', async () => { + const callback = vi.fn() + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) + const mediaSource = testState.mediaSources[0] + const audio = testState.audios[0] + mediaSource!.addSourceBuffer.mockImplementationOnce(() => { + throw new DOMException('Unsupported type', 'NotSupportedError') + }) + + mediaSource!.emit('sourceopen') + await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true) + await player.playAudioWithAudio('', false) + + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1)) + expect(player.mediaSource).toBeNull() + expect(audio!.autoplay).toBe(false) + expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url') + expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2) + }) + + it('should complete buffered fallback when addSourceBuffer throws after stream end', async () => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', vi.fn()) + const mediaSource = testState.mediaSources[0] + const audio = testState.audios[0] + mediaSource!.addSourceBuffer.mockImplementationOnce(() => { + throw new DOMException('Unsupported type', 'NotSupportedError') + }) + + await player.playAudioWithAudio(Buffer.from([1, 2]).toString('base64'), true) + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(1)) + await player.playAudioWithAudio('', false) + audio!.paused = true + + mediaSource!.emit('sourceopen') + + await waitFor(() => expect(audio!.play).toHaveBeenCalledTimes(2)) + expect(player.mediaSource).toBeNull() + expect(player.cacheBuffers).toHaveLength(0) + expect(globalThis.URL.createObjectURL).toHaveBeenCalledTimes(2) + const audioBlob = vi.mocked(globalThis.URL.createObjectURL).mock.calls[1]![0] as Blob + expect(audioBlob).toMatchObject({ type: 'audio/mpeg', size: 2 }) + expect(new Uint8Array(await audioBlob.arrayBuffer())).toEqual(new Uint8Array([1, 2])) + }) + it('should play immediately for ended audio in playAudioWithAudio', async () => { const callback = vi.fn() const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) @@ -517,7 +804,7 @@ describe('AudioPlayer', () => { await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true) expect(audio!.play).toHaveBeenCalledTimes(1) - expect(callback).toHaveBeenCalledWith('play') + await waitFor(() => expect(callback).toHaveBeenCalledWith('play')) }) it('should not replay when played list exists in playAudioWithAudio', async () => { @@ -534,18 +821,63 @@ describe('AudioPlayer', () => { expect(callback).not.toHaveBeenCalledWith('play') }) - it('should replay when paused is false and played list is empty in playAudioWithAudio', async () => { + it('should report a play failure and retry without requesting audio again', async () => { const callback = vi.fn() const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) const audio = testState.audios[0] + const mediaSource = testState.mediaSources[0] + mockTextToAudioStream.mockResolvedValue( + makeAudioResponse(200, [{ value: undefined, done: true }]), + ) + audio!.play.mockRejectedValueOnce(new DOMException('Playback aborted', 'AbortError')) - audio!.paused = false - audio!.ended = false - audio!.played = null - await player.playAudioWithAudio(Buffer.from('hello').toString('base64'), true) + mediaSource!.emit('sourceopen') + player.playAudio() + await waitFor(() => expect(callback).toHaveBeenCalledWith('error')) + expect(callback).not.toHaveBeenCalledWith('play') + expect(player.isLoadData).toBe(true) + + audio!.play.mockImplementationOnce(async () => { + audio!.paused = false + }) + player.playAudio() + + await waitFor(() => expect(callback).toHaveBeenCalledWith('play')) + + expect(audio!.play).toHaveBeenCalledTimes(2) + expect(mockTextToAudioStream).toHaveBeenCalledTimes(1) + }) + + it('should report a resume failure and allow playback to be retried', async () => { + const callback = vi.fn() + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', callback) + const audio = testState.audios[0] + const audioContext = testState.audioContexts[0] + const mediaSource = testState.mediaSources[0] + mockTextToAudioStream.mockResolvedValue( + makeAudioResponse(200, [{ value: undefined, done: true }]), + ) + audioContext!.state = 'suspended' + audioContext!.resume.mockRejectedValueOnce(new DOMException('Not allowed', 'NotAllowedError')) + + mediaSource!.emit('sourceopen') + player.playAudio() + + await waitFor(() => expect(callback).toHaveBeenCalledWith('error')) expect(audio!.play).toHaveBeenCalledTimes(1) - expect(callback).toHaveBeenCalledWith('play') + expect(player.isLoadData).toBe(true) + + audioContext!.resume.mockImplementationOnce(async () => { + audioContext!.state = 'running' + }) + player.playAudio() + + await waitFor(() => expect(callback).toHaveBeenCalledWith('play')) + + expect(audioContext!.resume).toHaveBeenCalledTimes(2) + expect(audio!.play).toHaveBeenCalledTimes(1) + expect(mockTextToAudioStream).toHaveBeenCalledTimes(1) }) }) @@ -562,7 +894,7 @@ describe('AudioPlayer', () => { expect(finishStream).toHaveBeenCalledTimes(1) }) - it('should finish stream when receiveAudioData gets empty bytes while source is open', () => { + it('should finish stream when receiveAudioData gets empty bytes', () => { const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) const finishStream = vi .spyOn(player as unknown as { finishStream: () => void }, 'finishStream') @@ -586,6 +918,52 @@ describe('AudioPlayer', () => { expect(player.cacheBuffers.length).toBe(1) }) + it('should preserve audio received before sourceopen and append it once ready', () => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const mediaSource = testState.mediaSources[0] + + ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData( + new Uint8Array([1, 2, 3]), + ) + + expect(player.cacheBuffers).toHaveLength(1) + expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled() + + mediaSource!.emit('sourceopen') + + expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1) + expect(player.cacheBuffers).toHaveLength(0) + }) + + it('should append queued buffers in order after updateend', () => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const mediaSource = testState.mediaSources[0] + mediaSource!.emit('sourceopen') + mediaSource!.sourceBuffer.updating = true + + const first = new Uint8Array([1]) + const second = new Uint8Array([2]) + ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData( + first, + ) + ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData( + second, + ) + + mediaSource!.sourceBuffer.updating = false + mediaSource!.sourceBuffer.emit('updateend') + expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1) + expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[0]![0])).toEqual( + first, + ) + + mediaSource!.sourceBuffer.emit('updateend') + expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(2) + expect(new Uint8Array(mediaSource!.sourceBuffer.appendBuffer.mock.calls[1]![0])).toEqual( + second, + ) + }) + it('should append previously queued buffer before new one when source buffer is idle', () => { const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) const mediaSource = testState.mediaSources[0] @@ -603,19 +981,68 @@ describe('AudioPlayer', () => { expect(player.cacheBuffers.length).toBe(1) }) - it('should append cache chunks and end stream when finishStream drains buffers', () => { - vi.useFakeTimers() + it('should end the stream only after the final queued buffer is appended', () => { const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) const mediaSource = testState.mediaSources[0] mediaSource!.emit('sourceopen') - mediaSource!.sourceBuffer.updating = false + mediaSource!.sourceBuffer.updating = true player.cacheBuffers = [new ArrayBuffer(3)] - ;(player as unknown as { finishStream: () => void }).finishStream() - vi.advanceTimersByTime(50) + ;(player as unknown as { finishStream: () => void }).finishStream() + + expect(mediaSource!.endOfStream).not.toHaveBeenCalled() + + mediaSource!.sourceBuffer.updating = false + mediaSource!.sourceBuffer.emit('updateend') expect(mediaSource!.sourceBuffer.appendBuffer).toHaveBeenCalledTimes(1) + expect(mediaSource!.endOfStream).not.toHaveBeenCalled() + + mediaSource!.sourceBuffer.emit('updateend') expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1) - vi.useRealTimers() + }) + + it('should end an open stream at most once', () => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const mediaSource = testState.mediaSources[0] + mediaSource!.emit('sourceopen') + + ;(player as unknown as { finishStream: () => void }).finishStream() + ;(player as unknown as { finishStream: () => void }).finishStream() + + expect(mediaSource!.endOfStream).toHaveBeenCalledTimes(1) + }) + + it.each(['closed', 'ended'] as const)('should not end a %s media source', (readyState) => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const mediaSource = testState.mediaSources[0] + mediaSource!.emit('sourceopen') + mediaSource!.readyState = readyState + + ;(player as unknown as { finishStream: () => void }).finishStream() + + expect(mediaSource!.endOfStream).not.toHaveBeenCalled() + }) + + it('should stop buffering and release browser resources after destroy', async () => { + const player = new AudioPlayer('/text-to-audio', true, 'msg-1', 'hello', 'en-US', null) + const mediaSource = testState.mediaSources[0] + const audio = testState.audios[0] + const audioContext = testState.audioContexts[0] + mediaSource!.emit('sourceopen') + + player.destroy() + ;(player as unknown as { receiveAudioData: (data: Uint8Array) => void }).receiveAudioData( + new Uint8Array([1]), + ) + ;(player as unknown as { finishStream: () => void }).finishStream() + mediaSource!.sourceBuffer.emit('updateend') + await Promise.resolve() + + expect(mediaSource!.sourceBuffer.appendBuffer).not.toHaveBeenCalled() + expect(mediaSource!.endOfStream).not.toHaveBeenCalled() + expect(audio!.pause).toHaveBeenCalledTimes(1) + expect(audioContext!.close).toHaveBeenCalledTimes(1) + expect(globalThis.URL.revokeObjectURL).toHaveBeenCalledWith('blob:mock-url') }) }) }) diff --git a/web/app/components/base/audio-btn/audio.player.manager.ts b/web/app/components/base/audio-btn/audio.player.manager.ts index 59fb1faee34..fefeac7ca82 100644 --- a/web/app/components/base/audio-btn/audio.player.manager.ts +++ b/web/app/components/base/audio-btn/audio.player.manager.ts @@ -35,9 +35,7 @@ export class AudioPlayerManager { } else { if (this.audioPlayers) { try { - this.audioPlayers.pauseAudio() - this.audioPlayers.cacheBuffers = [] - this.audioPlayers.sourceBuffer?.abort() + this.audioPlayers.destroy() } catch {} } diff --git a/web/app/components/base/audio-btn/audio.ts b/web/app/components/base/audio-btn/audio.ts index be115023d9f..123c768f1f1 100644 --- a/web/app/components/base/audio-btn/audio.ts +++ b/web/app/components/base/audio-btn/audio.ts @@ -1,19 +1,19 @@ -import { toast } from '@langgenius/dify-ui/toast' import { AppSourceType, textToAudioStream } from '@/service/share' +const AUDIO_CONTENT_TYPE = 'audio/mpeg' + declare global { // oxlint-disable-next-line typescript/consistent-type-definitions interface Window { - ManagedMediaSource: any + ManagedMediaSource?: typeof MediaSource } } export default class AudioPlayer { mediaSource: MediaSource | null audio: HTMLAudioElement audioContext: AudioContext - sourceBuffer?: any + sourceBuffer?: SourceBuffer cacheBuffers: ArrayBuffer[] = [] - pauseTimer: number | null = null msgId: string | undefined msgContent: string | null | undefined = null voice: string | undefined = undefined @@ -21,6 +21,13 @@ export default class AudioPlayer { url: string isPublic: boolean callback: ((event: string) => void) | null + private objectUrl = '' + private streamEnded = false + private endOfStreamCalled = false + private destroyed = false + private playbackPending = false + private playWhenReady = false + private sourceOpenListener?: () => void constructor( streamUrl: string, isPublic: boolean, @@ -37,25 +44,26 @@ export default class AudioPlayer { this.voice = voice this.callback = callback // Compatible with iphone ios17 ManagedMediaSource - const MediaSource = window.ManagedMediaSource || window.MediaSource - if (!MediaSource) { - toast.error( - 'Your browser does not support audio streaming, if you are using an iPhone, please update to iOS 17.1 or later.', - ) - } - this.mediaSource = MediaSource ? new MediaSource() : null + const MediaSourceConstructor = window.ManagedMediaSource || window.MediaSource + const isManagedMediaSource = Boolean( + window.ManagedMediaSource && MediaSourceConstructor === window.ManagedMediaSource, + ) + const supportsStreaming = Boolean(MediaSourceConstructor?.isTypeSupported?.(AUDIO_CONTENT_TYPE)) + this.mediaSource = + supportsStreaming && MediaSourceConstructor ? new MediaSourceConstructor() : null this.audio = new Audio() this.setCallback(callback) - if (!window.MediaSource) { + if (this.mediaSource && isManagedMediaSource) { // if use ManagedMediaSource this.audio.disableRemotePlayback = true this.audio.controls = true } - this.audio.src = this.mediaSource ? URL.createObjectURL(this.mediaSource) : '' - this.audio.autoplay = true + this.listenMediaSource(AUDIO_CONTENT_TYPE) + this.objectUrl = this.mediaSource ? URL.createObjectURL(this.mediaSource) : '' + this.audio.src = this.objectUrl + this.audio.autoplay = Boolean(this.mediaSource) const source = this.audioContext.createMediaElementSource(this.audio) source.connect(this.audioContext.destination) - this.listenMediaSource('audio/mpeg') } public resetMsgId(msgId: string) { @@ -63,10 +71,77 @@ export default class AudioPlayer { } private listenMediaSource(contentType: string) { - this.mediaSource?.addEventListener('sourceopen', () => { - if (this.sourceBuffer) return - this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType) - }) + this.sourceOpenListener = () => { + if (this.destroyed || this.sourceBuffer) return + try { + this.sourceBuffer = this.mediaSource?.addSourceBuffer(contentType) + this.sourceBuffer?.addEventListener('updateend', this.flushBuffers) + this.flushBuffers() + } catch { + this.mediaSource = null + this.audio.autoplay = false + this.releaseObjectUrl() + if (this.streamEnded) this.finishBlobAudio() + } + } + this.mediaSource?.addEventListener('sourceopen', this.sourceOpenListener) + } + + private flushBuffers = () => { + if ( + this.destroyed || + !this.sourceBuffer || + this.sourceBuffer.updating || + this.mediaSource?.readyState !== 'open' + ) + return + + const nextBuffer = this.cacheBuffers.shift() + if (nextBuffer) { + this.sourceBuffer.appendBuffer(nextBuffer) + return + } + + if (this.streamEnded && !this.endOfStreamCalled) { + this.endOfStreamCalled = true + this.mediaSource.endOfStream() + } + } + + private requestPlayback(reportIfPlaying = false) { + if (this.destroyed || this.playbackPending) return + if (!this.isAudioContextPaused() && !this.audio.paused && !this.audio.ended) { + if (reportIfPlaying) this.callback?.('play') + return + } + + this.playbackPending = true + void this.resumeAndPlay() + } + + private isAudioContextPaused() { + return this.audioContext.state === 'suspended' || this.audioContext.state === 'interrupted' + } + + private async resumeAndPlay() { + try { + const pendingOperations: Promise[] = [] + if (this.isAudioContextPaused()) pendingOperations.push(this.audioContext.resume()) + if (this.audio.paused || this.audio.ended) pendingOperations.push(this.audio.play()) + + await Promise.all(pendingOperations) + if (this.destroyed) return + if (this.isAudioContextPaused()) { + this.callback?.('error') + return + } + + if (!this.destroyed) this.callback?.('play') + } catch { + if (!this.destroyed) this.callback?.('error') + } finally { + this.playbackPending = false + } } public setCallback(callback: ((event: string) => void) | null) { @@ -80,7 +155,7 @@ export default class AudioPlayer { false, ) this.audio.addEventListener( - 'paused', + 'pause', () => { callback('paused') }, @@ -133,7 +208,7 @@ export default class AudioPlayer { private async loadAudio() { try { - const audioResponse: any = await textToAudioStream( + const audioResponse = (await textToAudioStream( this.url, this.isPublic ? AppSourceType.webApp : AppSourceType.installedApp, { content_type: 'audio/mpeg' }, @@ -143,19 +218,21 @@ export default class AudioPlayer { voice: this.voice, text: this.msgContent, }, - ) + )) as Response if (audioResponse.status !== 200) { this.isLoadData = false - if (this.callback) this.callback('error') + this.callback?.('error') + return } + if (!audioResponse.body) throw new Error('Audio response body is missing') const reader = audioResponse.body.getReader() while (true) { const { value, done } = await reader.read() + if (value?.byteLength) this.receiveAudioData(value) if (done) { - this.receiveAudioData(value) + this.finishStream() break } - this.receiveAudioData(value) } } catch { this.isLoadData = false @@ -166,46 +243,29 @@ export default class AudioPlayer { // play audio public playAudio() { if (this.isLoadData) { - if (this.audioContext.state === 'suspended') { - this.audioContext.resume().then((_) => { - this.audio.play() - this.callback?.('play') - }) - } else if (this.audio.ended) { - this.audio.play() - this.callback?.('play') + if (!this.mediaSource && !this.objectUrl) { + this.playWhenReady = true + return } - this.callback?.('play') + this.requestPlayback(true) } else { this.isLoadData = true - this.audioContext.resume().then((_) => { - this.audio.play() - this.callback?.('play') - }) + this.playWhenReady = true + if (this.mediaSource) this.requestPlayback(true) + else if (this.isAudioContextPaused()) void this.audioContext.resume().catch(() => {}) this.loadAudio() } } - private theEndOfStream() { - const endTimer = setInterval(() => { - if (!this.sourceBuffer?.updating) { - this.mediaSource?.endOfStream() - clearInterval(endTimer) - } - }, 10) - } - private finishStream() { - const timer = setInterval(() => { - if (!this.cacheBuffers.length) { - this.theEndOfStream() - clearInterval(timer) - } - if (this.cacheBuffers.length && !this.sourceBuffer?.updating) { - const arrayBuffer = this.cacheBuffers.shift()! - this.sourceBuffer?.appendBuffer(arrayBuffer) - } - }, 10) + if (this.destroyed) return + this.streamEnded = true + if (this.mediaSource) { + this.flushBuffers() + return + } + + this.finishBlobAudio() } public async playAudioWithAudio(audio: string, play = true) { @@ -213,54 +273,82 @@ export default class AudioPlayer { this.finishStream() return } - const audioContent = Buffer.from(audio, 'base64') - this.receiveAudioData(new Uint8Array(audioContent)) + const audioContent = Uint8Array.from(atob(audio), (char) => char.charCodeAt(0)) + this.receiveAudioData(audioContent) if (play) { this.isLoadData = true - if (this.audio.paused) { - this.audioContext.resume().then((_) => { - this.audio.play() - this.callback?.('play') - }) - } else if (this.audio.ended) { - this.audio.play() - this.callback?.('play') - } else if (this.audio.played) { - /* empty */ - } else { - this.audio.play() - this.callback?.('play') - } + this.playWhenReady = true + if (this.mediaSource) this.requestPlayback() } } public pauseAudio() { + this.playWhenReady = false this.callback?.('paused') this.audio.pause() - this.audioContext.suspend() + void this.audioContext.suspend().catch(() => {}) } - private receiveAudioData(unit8Array: Uint8Array) { + public destroy() { + if (this.destroyed) return + + this.destroyed = true + this.cacheBuffers = [] + this.callback?.('paused') + this.audio.pause() + + if (this.sourceOpenListener) + this.mediaSource?.removeEventListener('sourceopen', this.sourceOpenListener) + + if (this.sourceBuffer) { + this.sourceBuffer.removeEventListener('updateend', this.flushBuffers) + if (this.mediaSource?.readyState === 'open') { + try { + this.sourceBuffer.abort() + } catch {} + } + } + + void this.audioContext.close().catch(() => {}) + this.releaseObjectUrl() + } + + private receiveAudioData(unit8Array: Uint8Array | undefined) { + if (this.destroyed || this.streamEnded) return if (!unit8Array) { this.finishStream() return } const audioData = this.byteArrayToArrayBuffer(unit8Array) if (!audioData.byteLength) { - if (this.mediaSource?.readyState === 'open') this.finishStream() + this.finishStream() return } - if (this.sourceBuffer?.updating) { - this.cacheBuffers.push(audioData) - } else { - if (this.cacheBuffers.length && !this.sourceBuffer?.updating) { - this.cacheBuffers.push(audioData) - const cacheBuffer = this.cacheBuffers.shift()! - this.sourceBuffer?.appendBuffer(cacheBuffer) - } else { - this.sourceBuffer?.appendBuffer(audioData) - } + this.cacheBuffers.push(audioData) + this.flushBuffers() + } + + private finishBlobAudio() { + if (!this.cacheBuffers.length) { + if (!this.objectUrl) this.isLoadData = false + return } + + const audioBlob = new Blob(this.cacheBuffers, { type: AUDIO_CONTENT_TYPE }) + this.cacheBuffers = [] + this.releaseObjectUrl() + this.objectUrl = URL.createObjectURL(audioBlob) + this.audio.src = this.objectUrl + this.isLoadData = true + if (this.playWhenReady) this.requestPlayback() + } + + private releaseObjectUrl() { + if (!this.objectUrl) return + + URL.revokeObjectURL(this.objectUrl) + this.objectUrl = '' + this.audio.src = '' } private byteArrayToArrayBuffer(byteArray: Uint8Array): ArrayBuffer {