diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml index 22bd37e871f..491f65c28ee 100644 --- a/.github/workflows/post-merge.yml +++ b/.github/workflows/post-merge.yml @@ -25,8 +25,15 @@ jobs: filters: | external_e2e: - 'e2e/features/agent-v2/**' + - 'e2e/features/step-definitions/agent-v2/**' + - 'e2e/features/support/**' + - 'e2e/fixtures/test-materials/**' - 'e2e/scripts/**' - 'e2e/support/**' + - 'e2e/cucumber.config.ts' + - 'e2e/package.json' + - 'e2e/test-env.ts' + - 'e2e/tsx-register.js' - '.github/workflows/post-merge.yml' - '.github/workflows/web-e2e.yml' - '.github/actions/setup-web/**' @@ -40,9 +47,25 @@ jobs: - 'api/services/plugin/**' - 'api/core/tools/**' - 'api/services/tools/**' + - 'packages/contracts/generated/api/console/agent/**' + - 'packages/contracts/generated/api/console/orpc.gen.ts' - 'web/features/agent-v2/**' + - 'web/app/(commonLayout)/agents/**' + - 'web/app/(commonLayout)/@detailSidebar/agents/**' - 'web/app/(commonLayout)/roster/**' + - 'web/app/components/base/chat/chat/**' + - 'web/app/components/base/voice-input/**' - 'web/app/components/workflow/nodes/agent-v2/**' + - 'web/i18n/en-US/agent-v-2.json' + - 'web/i18n/en-US/common.json' + - 'web/service/base.ts' + - 'web/service/client.ts' + - 'web/service/console-link.ts' + - 'web/service/console-openapi-url.ts' + - 'web/service/console-router-loader.ts' + - 'web/service/share.ts' + - 'web/package.json' + - 'pnpm-workspace.yaml' external-e2e: name: External Runtime E2E diff --git a/.github/workflows/web-e2e.yml b/.github/workflows/web-e2e.yml index a43937fcd2f..80798668189 100644 --- a/.github/workflows/web-e2e.yml +++ b/.github/workflows/web-e2e.yml @@ -80,6 +80,8 @@ jobs: E2E_MARKETPLACE_PLUGIN_IDS: ${{ vars.E2E_MARKETPLACE_PLUGIN_IDS }} E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS: ${{ vars.E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS }} E2E_MODEL_PROVIDER_CREDENTIALS_JSON: ${{ secrets.E2E_MODEL_PROVIDER_CREDENTIALS_JSON }} + E2E_SPEECH_TO_TEXT_MODEL_NAME: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_NAME || 'gpt-4o-mini-transcribe' }} + E2E_SPEECH_TO_TEXT_MODEL_PROVIDER: ${{ vars.E2E_SPEECH_TO_TEXT_MODEL_PROVIDER || 'openai' }} E2E_START_AGENT_BACKEND: ${{ vars.E2E_START_AGENT_BACKEND || '1' }} E2E_STABLE_MODEL_NAME: ${{ vars.E2E_STABLE_MODEL_NAME || 'gpt-5-nano' }} E2E_STABLE_MODEL_PROVIDER: ${{ vars.E2E_STABLE_MODEL_PROVIDER || 'openai' }} diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 0c9ed786a1e..06ef0dc7acb 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -1,13 +1,18 @@ import logging +from uuid import UUID from flask import request from flask_restx import Resource -from pydantic import BaseModel, Field, RootModel -from werkzeug.exceptions import InternalServerError +from pydantic import BaseModel, ConfigDict, Field, RootModel +from sqlalchemy.orm import Session +from werkzeug.datastructures import FileStorage +from werkzeug.exceptions import HTTPException, InternalServerError import services from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models +from controllers.common.wraps import enforce_rbac_access from controllers.console import console_ns +from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model from controllers.console.app.error import ( AppUnavailableError, AudioTooLargeError, @@ -17,15 +22,19 @@ from controllers.console.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) -from controllers.console.app.wraps import get_app_model +from controllers.console.app.wraps import get_app_model, with_session from controllers.console.wraps import ( RBACPermission, RBACResourceScope, account_initialization_required, + edit_permission_required, rbac_permission_required, setup_required, + with_current_tenant_id, + with_current_user, ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from extensions.ext_database import db @@ -33,13 +42,17 @@ from fields.base import ResponseModel from graphon.model_runtime.errors.invoke import InvokeError from libs.helper import dump_response from libs.login import current_user, login_required -from models import App, AppMode +from models import Account, App, AppMode +from models.agent import AgentConfigDraftType +from models.agent_config_entities import AgentSoulConfig +from services.agent.composer_service import AgentComposerService from services.app_ref_service import AppRefService from services.audio_service import AudioService from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -61,6 +74,12 @@ class AudioTranscriptResponse(ResponseModel): text: str = Field(description="Transcribed text from audio") +class AgentAudioTranscriptFormPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + draft_type: AgentConfigDraftType = AgentConfigDraftType.DRAFT + + class TextToSpeechVoiceResponse(ResponseModel): # see api/core/plugin/impl/model.py name: str = Field(description="Voice display name") @@ -71,7 +90,7 @@ class TextToSpeechVoiceListResponse(RootModel[list[TextToSpeechVoiceResponse]]): root: list[TextToSpeechVoiceResponse] = Field(description="Available voices") -register_schema_models(console_ns, TextToSpeechPayload, TextToSpeechVoiceQuery) +register_schema_models(console_ns, AgentAudioTranscriptFormPayload, TextToSpeechPayload, TextToSpeechVoiceQuery) register_response_schema_models( console_ns, AudioTranscriptResponse, @@ -79,12 +98,90 @@ register_response_schema_models( TextToSpeechVoiceListResponse, ) +_AUDIO_TRANSCRIPT_FILE_PARAM = { + "description": "MP3 audio to transcribe", + "in": "formData", + "type": "file", + "required": True, +} +_AGENT_AUDIO_TRANSCRIPT_PARAMS = { + "file": _AUDIO_TRANSCRIPT_FILE_PARAM, + "draft_type": { + "description": "Agent debug config source", + "in": "formData", + "type": "string", + "enum": [draft_type.value for draft_type in AgentConfigDraftType], + "default": AgentConfigDraftType.DRAFT.value, + "required": False, + }, +} +_CONSOLE_AUDIO_TRANSCRIPT_APP_MODES = [ + AppMode.CHAT, + AppMode.AGENT_CHAT, + AppMode.ADVANCED_CHAT, + AppMode.AGENT, +] + + +def _transcribe_audio_to_text( + *, + app_model: App, + file: FileStorage | None, + agent_soul: AgentSoulConfig | None = None, +) -> dict[str, str]: + try: + if agent_soul is None: + response = AudioService.transcript_asr( + app_model=app_model, + file=file, + end_user=None, + ) + else: + response = AudioService.transcript_agent_asr( + app_model=app_model, + agent_soul=agent_soul, + file=file, + end_user=None, + ) + return dump_response(AudioTranscriptResponse, response) + except services.errors.app_model_config.AppModelConfigBrokenError: + logger.exception("App model config broken.") + raise AppUnavailableError() + except NoAudioUploadedServiceError: + raise NoAudioUploadedError() + except AudioTooLargeServiceError as e: + raise AudioTooLargeError(str(e)) + except UnsupportedAudioTypeServiceError: + raise UnsupportedAudioTypeError() + except ProviderNotSupportSpeechToTextServiceError: + raise ProviderNotSupportSpeechToTextError() + except SpeechToTextDisabledServiceError: + raise SpeechToTextDisabledError() + except ProviderTokenNotInitError as ex: + raise ProviderNotInitializeError(ex.description) + except QuotaExceededError: + raise ProviderQuotaExceededError() + except ModelCurrentlyNotSupportError: + raise ProviderModelCurrentlyNotSupportError() + except InvokeError as e: + raise CompletionRequestError(e.description) + except HTTPException: + raise + except ValueError: + raise + except Exception as e: + logger.exception("Failed to transcribe audio to text") + raise InternalServerError() from e + @console_ns.route("/apps//audio-to-text") class ChatMessageAudioApi(Resource): @console_ns.doc("chat_message_audio_transcript") @console_ns.doc(description="Transcript audio to text for chat messages") - @console_ns.doc(params={"app_id": "App ID"}) + @console_ns.doc( + consumes=["multipart/form-data"], + params={"app_id": "App ID", "file": _AUDIO_TRANSCRIPT_FILE_PARAM}, + ) @console_ns.response( 200, "Audio transcription successful", @@ -95,42 +192,63 @@ class ChatMessageAudioApi(Resource): @setup_required @login_required @account_initialization_required - @get_app_model(mode=[AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT]) + @get_app_model(mode=_CONSOLE_AUDIO_TRANSCRIPT_APP_MODES) def post(self, app_model: App): - file = request.files["file"] + return _transcribe_audio_to_text(app_model=app_model, file=request.files.get("file")) - try: - response = AudioService.transcript_asr( - app_model=app_model, - file=file, - end_user=None, - ) - return dump_response(AudioTranscriptResponse, response) - except services.errors.app_model_config.AppModelConfigBrokenError: - logger.exception("App model config broken.") - raise AppUnavailableError() - except NoAudioUploadedServiceError: - raise NoAudioUploadedError() - except AudioTooLargeServiceError as e: - raise AudioTooLargeError(str(e)) - except UnsupportedAudioTypeServiceError: - raise UnsupportedAudioTypeError() - except ProviderNotSupportSpeechToTextServiceError: - raise ProviderNotSupportSpeechToTextError() - except ProviderTokenNotInitError as ex: - raise ProviderNotInitializeError(ex.description) - except QuotaExceededError: - raise ProviderQuotaExceededError() - except ModelCurrentlyNotSupportError: - raise ProviderModelCurrentlyNotSupportError() - except InvokeError as e: - raise CompletionRequestError(e.description) - except ValueError as e: - raise e - except Exception as e: - logger.exception("Failed to handle post request to ChatMessageAudioApi") - raise InternalServerError() +@console_ns.route("/agent//audio-to-text") +class AgentChatMessageAudioApi(Resource): + @console_ns.doc("agent_chat_message_audio_transcript") + @console_ns.doc(description="Transcribe audio using the current Agent debug configuration") + @console_ns.doc( + consumes=["multipart/form-data"], + params={"agent_id": "Agent ID", **_AGENT_AUDIO_TRANSCRIPT_PARAMS}, + ) + @console_ns.response( + 200, + "Audio transcription successful", + console_ns.models[AudioTranscriptResponse.__name__], + ) + @console_ns.response(400, "Bad request - Speech to text disabled or unsupported audio") + @console_ns.response(404, "Agent or build draft not found") + @console_ns.response(413, "Audio file too large") + @setup_required + @login_required + @account_initialization_required + @edit_permission_required + @with_current_user + @with_current_tenant_id + @with_session + def post( + self, + session: Session, + current_tenant_id: str, + current_user: Account, + agent_id: UUID, + ): + payload = AgentAudioTranscriptFormPayload.model_validate(request.form.to_dict(flat=True)) + app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id) + # Agent routes expose Agent ids, while APP RBAC is keyed by the resolved runtime App id. + enforce_rbac_access( + tenant_id=current_tenant_id, + account_id=current_user.id, + resource_type=RBACResourceScope.APP, + scene=RBACPermission.APP_TEST_AND_RUN, + path_args={"app_id": app_model.id}, + ) + agent_soul = AgentComposerService.load_agent_soul_for_debug( + tenant_id=current_tenant_id, + agent_id=str(agent_id), + account_id=current_user.id, + draft_type=payload.draft_type, + session=session, + ) + return _transcribe_audio_to_text( + app_model=app_model, + agent_soul=agent_soul, + file=request.files.get("file"), + ) @console_ns.route("/apps//text-to-audio") diff --git a/api/controllers/console/app/error.py b/api/controllers/console/app/error.py index 3fa15d6d6d7..1bb6fafb224 100644 --- a/api/controllers/console/app/error.py +++ b/api/controllers/console/app/error.py @@ -79,6 +79,12 @@ class ProviderNotSupportSpeechToTextError(BaseHTTPException): code = 400 +class SpeechToTextDisabledError(BaseHTTPException): + error_code = "speech_to_text_disabled" + description = "Speech to text is disabled." + code = 400 + + class DraftWorkflowNotExist(BaseHTTPException): error_code = "draft_workflow_not_exist" description = "Draft workflow need to be initialized." diff --git a/api/controllers/console/explore/audio.py b/api/controllers/console/explore/audio.py index e5f98f0f655..3ad170ebccf 100644 --- a/api/controllers/console/explore/audio.py +++ b/api/controllers/console/explore/audio.py @@ -16,6 +16,7 @@ from controllers.console.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from controllers.console.explore.wraps import InstalledAppResource @@ -30,6 +31,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -69,6 +71,8 @@ class ChatAudioApi(InstalledAppResource): raise UnsupportedAudioTypeError() except ProviderNotSupportSpeechToTextServiceError: raise ProviderNotSupportSpeechToTextError() + except SpeechToTextDisabledServiceError: + raise SpeechToTextDisabledError() except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/console/explore/trial.py b/api/controllers/console/explore/trial.py index d01eb9c38b1..47c4b625512 100644 --- a/api/controllers/console/explore/trial.py +++ b/api/controllers/console/explore/trial.py @@ -34,6 +34,7 @@ from controllers.console.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from controllers.console.app.wraps import get_app_model_with_trial, with_session @@ -75,6 +76,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) from services.errors.conversation import ConversationNotExistsError @@ -606,6 +608,8 @@ class TrialChatAudioApi(TrialAppResource): raise UnsupportedAudioTypeError() except ProviderNotSupportSpeechToTextServiceError: raise ProviderNotSupportSpeechToTextError() + except SpeechToTextDisabledServiceError: + raise SpeechToTextDisabledError() except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/service_api/app/audio.py b/api/controllers/service_api/app/audio.py index 56d7e28b95a..28ac24ee7ac 100644 --- a/api/controllers/service_api/app/audio.py +++ b/api/controllers/service_api/app/audio.py @@ -18,6 +18,7 @@ from controllers.service_api.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from controllers.service_api.schema import binary_response, expect_with_user, multipart_file_params @@ -33,6 +34,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -54,6 +56,7 @@ class AudioApi(Resource): 200: "Successfully converted audio to text.", 400: ( "- `app_unavailable` : App unavailable or misconfigured.\n" + "- `speech_to_text_disabled` : Speech-to-text is disabled for this app.\n" "- `provider_not_support_speech_to_text` : Model provider does not support speech-to-text.\n" "- `provider_not_initialize` : No valid model provider credentials found.\n" "- `provider_quota_exceeded` : Model provider quota exhausted.\n" @@ -115,6 +118,8 @@ class AudioApi(Resource): raise UnsupportedAudioTypeError() except ProviderNotSupportSpeechToTextServiceError: raise ProviderNotSupportSpeechToTextError() + except SpeechToTextDisabledServiceError: + raise SpeechToTextDisabledError() except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/service_api/app/error.py b/api/controllers/service_api/app/error.py index 970e1deb3e4..e6f97e98249 100644 --- a/api/controllers/service_api/app/error.py +++ b/api/controllers/service_api/app/error.py @@ -97,6 +97,12 @@ class ProviderNotSupportSpeechToTextError(BaseHTTPException): code = 400 +class SpeechToTextDisabledError(BaseHTTPException): + error_code = "speech_to_text_disabled" + description = "Speech to text is disabled." + code = 400 + + class FileNotFoundError(BaseHTTPException): error_code = "file_not_found" description = "The requested file was not found." diff --git a/api/controllers/web/audio.py b/api/controllers/web/audio.py index 52e9492dbf8..defda5ab0d9 100644 --- a/api/controllers/web/audio.py +++ b/api/controllers/web/audio.py @@ -16,6 +16,7 @@ from controllers.web.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from controllers.web.wraps import WebApiResource @@ -31,6 +32,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -91,6 +93,8 @@ class AudioApi(WebApiResource): raise UnsupportedAudioTypeError() except ProviderNotSupportSpeechToTextServiceError: raise ProviderNotSupportSpeechToTextError() + except SpeechToTextDisabledServiceError: + raise SpeechToTextDisabledError() except ProviderTokenNotInitError as ex: raise ProviderNotInitializeError(ex.description) except QuotaExceededError: diff --git a/api/controllers/web/error.py b/api/controllers/web/error.py index 077b4726e47..c6ea2aea192 100644 --- a/api/controllers/web/error.py +++ b/api/controllers/web/error.py @@ -103,6 +103,12 @@ class ProviderNotSupportSpeechToTextError(BaseHTTPException): code = 400 +class SpeechToTextDisabledError(BaseHTTPException): + error_code = "speech_to_text_disabled" + description = "Speech to text is disabled." + code = 400 + + class WebAppAuthRequiredError(BaseHTTPException): error_code = "web_sso_auth_required" description = "Web app authentication required." diff --git a/api/openapi/markdown/console-openapi.md b/api/openapi/markdown/console-openapi.md index 12a02b6d3ad..00c2aafe263 100644 --- a/api/openapi/markdown/console-openapi.md +++ b/api/openapi/markdown/console-openapi.md @@ -465,6 +465,30 @@ Check if activation token is valid | ---- | ----------- | | 204 | Agent service API key deleted | +### [POST] /agent/{agent_id}/audio-to-text +Transcribe audio using the current Agent debug configuration + +#### Parameters + +| Name | Located in | Description | Required | Schema | +| ---- | ---------- | ----------- | -------- | ------ | +| agent_id | path | Agent ID | Yes | string (uuid) | + +#### Request Body + +| Required | Schema | +| -------- | ------ | +| Yes | **multipart/form-data**: { **"draft_type"**: string,
**Available values:** "debug_build", "draft",
**Default:** draft, **"file"**: binary }
| + +#### Responses + +| Code | Description | Schema | +| ---- | ----------- | ------ | +| 200 | Audio transcription successful | **application/json**: [AudioTranscriptResponse](#audiotranscriptresponse)
| +| 400 | Bad request - Speech to text disabled or unsupported audio | | +| 404 | Agent or build draft not found | | +| 413 | Audio file too large | | + ### [POST] /agent/{agent_id}/build-chat/finalize Run a build-draft Agent App turn that asks the agent to push config updates @@ -2638,6 +2662,12 @@ Transcript audio to text for chat messages | ---- | ---------- | ----------- | -------- | ------ | | app_id | path | App ID | Yes | string (uuid) | +#### Request Body + +| Required | Schema | +| -------- | ------ | +| Yes | **multipart/form-data**: { **"file"**: binary }
| + #### Responses | Code | Description | Schema | @@ -13096,6 +13126,12 @@ default (the config form sends the full desired feature state on save). | role | string | Agent role | No | | use_icon_as_answer_icon | boolean | Use icon as answer icon | No | +#### AgentAudioTranscriptFormPayload + +| Name | Type | Description | Required | +| ---- | ---- | ----------- | -------- | +| draft_type | [AgentConfigDraftType](#agentconfigdrafttype) | | No | + #### AgentAverageResponseTimeStatisticResponse | Name | Type | Description | Required | diff --git a/api/openapi/markdown/service-openapi.md b/api/openapi/markdown/service-openapi.md index c442b7a0ff7..6cd962146d8 100644 --- a/api/openapi/markdown/service-openapi.md +++ b/api/openapi/markdown/service-openapi.md @@ -278,7 +278,7 @@ Convert audio file to text. Supported MIME types: `audio/mp3`, `audio/mpga`, `au | Code | Description | Schema | | ---- | ----------- | ------ | | 200 | Successfully converted audio to text. | **application/json**: [AudioTranscriptResponse](#audiotranscriptresponse)
| -| 400 | - `app_unavailable` : App unavailable or misconfigured. - `provider_not_support_speech_to_text` : Model provider does not support speech-to-text. - `provider_not_initialize` : No valid model provider credentials found. - `provider_quota_exceeded` : Model provider quota exhausted. - `model_currently_not_support` : Current model does not support this operation. - `completion_request_error` : Speech recognition request failed. | | +| 400 | - `app_unavailable` : App unavailable or misconfigured. - `speech_to_text_disabled` : Speech-to-text is disabled for this app. - `provider_not_support_speech_to_text` : Model provider does not support speech-to-text. - `provider_not_initialize` : No valid model provider credentials found. - `provider_quota_exceeded` : Model provider quota exhausted. - `model_currently_not_support` : Current model does not support this operation. - `completion_request_error` : Speech recognition request failed. | | | 401 | Unauthorized - invalid API token | | | 403 | Forbidden - token scope, app, dataset, or workspace access denied | | | 413 | `audio_too_large` : Audio file size exceeded the limit. | | diff --git a/api/services/agent/composer_service.py b/api/services/agent/composer_service.py index 4839012dd3a..3f4c50a794d 100644 --- a/api/services/agent/composer_service.py +++ b/api/services/agent/composer_service.py @@ -358,6 +358,32 @@ class AgentComposerService: agent = cls._require_agent(tenant_id=tenant_id, agent_id=agent_id, session=session) return cls._load_agent_composer_for_agent(tenant_id=tenant_id, agent=agent, session=session) + @classmethod + def load_agent_soul_for_debug( + cls, + *, + tenant_id: str, + agent_id: str, + account_id: str, + draft_type: AgentConfigDraftType, + session: Session, + ) -> AgentSoulConfig: + """Load the same normal or account-owned build draft used by Agent debug chat.""" + if draft_type == AgentConfigDraftType.DEBUG_BUILD: + state = cls.load_agent_app_build_draft( + tenant_id=tenant_id, + agent_id=agent_id, + account_id=account_id, + session=session, + ) + else: + state = cls.load_agent_composer( + tenant_id=tenant_id, + agent_id=agent_id, + session=session, + ) + return AgentSoulConfig.model_validate(state["agent_soul"]) + @classmethod def _load_agent_composer_for_agent(cls, *, tenant_id: str, agent: Agent, session: Session) -> dict[str, Any]: draft = cls._get_or_create_agent_draft( diff --git a/api/services/agent/roster_service.py b/api/services/agent/roster_service.py index 33fd372398b..7758b81e59a 100644 --- a/api/services/agent/roster_service.py +++ b/api/services/agent/roster_service.py @@ -798,6 +798,18 @@ class AgentRosterService: ) ) + def get_published_agent_soul_for_app(self, *, tenant_id: str, app_id: str) -> AgentSoulConfig | None: + """Return the active Agent Soul used by a published Agent App runtime.""" + agent = self.get_app_backing_agent(tenant_id=tenant_id, app_id=app_id) + if agent is None: + return None + version = self._get_version( + tenant_id=tenant_id, + agent_id=agent.id, + version_id=agent.active_config_snapshot_id, + ) + return AgentSoulConfig.model_validate(version.config_snapshot_dict) + def get_agent_app_model(self, *, tenant_id: str, agent_id: str) -> App: """Resolve the Agent App hidden behind an app-backed Agent id. diff --git a/api/services/audio_service.py b/api/services/audio_service.py index 52c71edd576..597b23d412a 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -10,16 +10,21 @@ from sqlalchemy.orm import Session from werkzeug.datastructures import FileStorage from constants import AUDIO_EXTENSIONS +from core.app.apps.agent_app.app_feature_projection import merge_agent_app_features from core.model_manager import ModelManager +from extensions.ext_database import db from graphon.model_runtime.entities.model_entities import ModelType +from models.agent_config_entities import AgentSoulConfig from models.enums import MessageStatus from models.model import App, AppMode, Message +from services.agent.roster_service import AgentRosterService from services.app_ref_service import MessageRef from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, ProviderNotSupportTextToSpeechServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) from services.workflow_service import WorkflowService @@ -41,23 +46,72 @@ class AudioService: return session.scalar(stmt.limit(1)) @classmethod - def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None): + def transcript_asr(cls, app_model: App, file: FileStorage | None, end_user: str | None = None) -> dict[str, str]: + """Transcribe audio after enforcing the effective feature configuration. + + Published Agent Apps use their active Agent Soul. Historical Agent Apps + without a backing roster Agent retain the legacy AppModelConfig fallback. + + Raises: + SpeechToTextDisabledServiceError: If the effective feature configuration disables STT. + """ + if app_model.mode == AppMode.AGENT: + agent_soul = AgentRosterService(db.session).get_published_agent_soul_for_app( + tenant_id=app_model.tenant_id, + app_id=app_model.id, + ) + if agent_soul is not None: + return cls.transcript_agent_asr( + app_model=app_model, + agent_soul=agent_soul, + file=file, + end_user=end_user, + ) + if app_model.mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}: workflow = app_model.workflow if workflow is None: - raise ValueError("Speech to text is not enabled") + raise SpeechToTextDisabledServiceError() features_dict = workflow.features_dict if "speech_to_text" not in features_dict or not features_dict["speech_to_text"].get("enabled"): - raise ValueError("Speech to text is not enabled") + raise SpeechToTextDisabledServiceError() else: app_model_config = app_model.app_model_config if not app_model_config: - raise ValueError("Speech to text is not enabled") + raise SpeechToTextDisabledServiceError() if not app_model_config.speech_to_text_dict["enabled"]: - raise ValueError("Speech to text is not enabled") + raise SpeechToTextDisabledServiceError() + return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user) + + @classmethod + def transcript_agent_asr( + cls, + app_model: App, + agent_soul: AgentSoulConfig, + file: FileStorage | None, + end_user: str | None = None, + ) -> dict[str, str]: + """Transcribe Agent audio after applying Soul-first runtime feature projection. + + Raises: + SpeechToTextDisabledServiceError: If the merged Agent feature configuration disables STT. + """ + features = merge_agent_app_features( + agent_soul=agent_soul, + app_model_config=app_model.app_model_config, + ) + if not features.get("speech_to_text", {}).get("enabled"): + raise SpeechToTextDisabledServiceError() + + return cls._invoke_speech_to_text(app_model=app_model, file=file, end_user=end_user) + + @classmethod + def _invoke_speech_to_text( + cls, app_model: App, file: FileStorage | None, end_user: str | None = None + ) -> dict[str, str]: if file is None: raise NoAudioUploadedServiceError() diff --git a/api/services/errors/audio.py b/api/services/errors/audio.py index 4005cbfcd7a..b610d088c24 100644 --- a/api/services/errors/audio.py +++ b/api/services/errors/audio.py @@ -14,6 +14,10 @@ class ProviderNotSupportSpeechToTextServiceError(Exception): pass +class SpeechToTextDisabledServiceError(Exception): + """Raised when the effective app configuration disables speech-to-text.""" + + class ProviderNotSupportTextToSpeechServiceError(Exception): pass diff --git a/api/tests/unit_tests/controllers/console/app/test_audio.py b/api/tests/unit_tests/controllers/console/app/test_audio.py index 138683169c7..8084e7c883e 100644 --- a/api/tests/unit_tests/controllers/console/app/test_audio.py +++ b/api/tests/unit_tests/controllers/console/app/test_audio.py @@ -4,13 +4,20 @@ import io from inspect import unwrap from types import SimpleNamespace from unittest.mock import patch +from uuid import UUID import pytest from flask import Flask from werkzeug.datastructures import FileStorage -from werkzeug.exceptions import InternalServerError +from werkzeug.exceptions import Forbidden, InternalServerError -from controllers.console.app.audio import ChatMessageAudioApi, ChatMessageTextApi, TextModesApi +from controllers.console.app import audio as audio_module +from controllers.console.app.audio import ( + AgentChatMessageAudioApi, + ChatMessageAudioApi, + ChatMessageTextApi, + TextModesApi, +) from controllers.console.app.error import ( AppUnavailableError, AudioTooLargeError, @@ -20,10 +27,16 @@ from controllers.console.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from graphon.model_runtime.errors.invoke import InvokeError +from models import AppMode +from models.agent import AgentConfigDraftType +from models.agent_config_entities import AgentSoulConfig +from services.agent.composer_service import AgentComposerService +from services.agent.errors import AgentVersionNotFoundError from services.app_ref_service import MessageRef from services.audio_service import AudioService from services.errors.app_model_config import AppModelConfigBrokenError @@ -32,6 +45,7 @@ from services.errors.audio import ( NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, ProviderNotSupportTextToSpeechLanageServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -52,6 +66,180 @@ def test_console_audio_api_success(app: Flask, monkeypatch: pytest.MonkeyPatch) assert response == {"text": "ok"} +def test_console_audio_api_accepts_published_agent_apps() -> None: + assert AppMode.AGENT in audio_module._CONSOLE_AUDIO_TRANSCRIPT_APP_MODES + + +def test_agent_console_audio_api_uses_agent_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + app_model = SimpleNamespace(id="backing-app-1") + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + calls: dict[str, object] = {} + + def resolve_agent_runtime_app_model(**kwargs): + calls["resolver"] = kwargs + return app_model + + def load_agent_soul_for_debug(**kwargs): + calls["draft"] = kwargs + return agent_soul + + def transcript_agent_asr(**kwargs): + calls["asr"] = kwargs + return {"text": "agent transcript"} + + def enforce_rbac_access(**kwargs): + calls["rbac"] = kwargs + + monkeypatch.setattr(audio_module, "resolve_agent_runtime_app_model", resolve_agent_runtime_app_model) + monkeypatch.setattr(audio_module, "enforce_rbac_access", enforce_rbac_access) + monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug) + monkeypatch.setattr(AudioService, "transcript_agent_asr", transcript_agent_asr) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + session = SimpleNamespace() + current_user = SimpleNamespace(id="account-1") + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data(), "draft_type": "debug_build"}, + ): + response = handler( + api, + session=session, + current_tenant_id="tenant-1", + current_user=current_user, + agent_id=agent_id, + ) + + assert response == {"text": "agent transcript"} + assert calls["resolver"] == {"tenant_id": "tenant-1", "agent_id": agent_id} + assert calls["rbac"] == { + "tenant_id": "tenant-1", + "account_id": "account-1", + "resource_type": audio_module.RBACResourceScope.APP, + "scene": audio_module.RBACPermission.APP_TEST_AND_RUN, + "path_args": {"app_id": "backing-app-1"}, + } + assert calls["draft"] == { + "tenant_id": "tenant-1", + "agent_id": str(agent_id), + "account_id": "account-1", + "draft_type": AgentConfigDraftType.DEBUG_BUILD, + "session": session, + } + assert calls["asr"] == { + "app_model": app_model, + "agent_soul": agent_soul, + "file": calls["asr"]["file"], + "end_user": None, + } + + +def test_agent_console_audio_api_defaults_to_normal_draft(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + captured: dict[str, object] = {} + monkeypatch.setattr( + audio_module, + "resolve_agent_runtime_app_model", + lambda **_kwargs: SimpleNamespace(id="backing-app-1"), + ) + + def load_agent_soul_for_debug(**kwargs): + captured.update(kwargs) + return AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + + monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug) + monkeypatch.setattr(AudioService, "transcript_agent_asr", lambda **_kwargs: {"text": "ok"}) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data()}, + ): + response = handler( + api, + session=SimpleNamespace(), + current_tenant_id="tenant-1", + current_user=SimpleNamespace(id="account-1"), + agent_id=agent_id, + ) + + assert response == {"text": "ok"} + assert captured["draft_type"] == AgentConfigDraftType.DRAFT + + +def test_agent_console_audio_api_checks_rbac_with_backing_app_id(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + app_model = SimpleNamespace(id="backing-app-1") + soul_loaded = False + + monkeypatch.setattr(audio_module, "resolve_agent_runtime_app_model", lambda **_kwargs: app_model) + + def deny_access(**kwargs): + assert kwargs["path_args"] == {"app_id": "backing-app-1"} + raise Forbidden() + + def load_agent_soul_for_debug(**_kwargs): + nonlocal soul_loaded + soul_loaded = True + return AgentSoulConfig() + + monkeypatch.setattr(audio_module, "enforce_rbac_access", deny_access) + monkeypatch.setattr(AgentComposerService, "load_agent_soul_for_debug", load_agent_soul_for_debug) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data()}, + ): + with pytest.raises(Forbidden): + handler( + api, + session=SimpleNamespace(), + current_tenant_id="tenant-1", + current_user=SimpleNamespace(id="account-1"), + agent_id=agent_id, + ) + + assert soul_loaded is False + + +def test_agent_console_audio_api_preserves_missing_build_draft_404(app: Flask, monkeypatch: pytest.MonkeyPatch) -> None: + agent_id = UUID("019ef3d2-b24c-7803-b428-18b5ee8fb853") + monkeypatch.setattr( + audio_module, + "resolve_agent_runtime_app_model", + lambda **_kwargs: SimpleNamespace(id="backing-app-1"), + ) + monkeypatch.setattr( + AgentComposerService, + "load_agent_soul_for_debug", + lambda **_kwargs: (_ for _ in ()).throw(AgentVersionNotFoundError()), + ) + + api = AgentChatMessageAudioApi() + handler = unwrap(api.post) + with app.test_request_context( + f"/console/api/agent/{agent_id}/audio-to-text", + method="POST", + data={"file": _file_data(), "draft_type": "debug_build"}, + ): + with pytest.raises(AgentVersionNotFoundError): + handler( + api, + session=SimpleNamespace(), + current_tenant_id="tenant-1", + current_user=SimpleNamespace(id="account-1"), + agent_id=agent_id, + ) + + @pytest.mark.parametrize( ("exc", "expected"), [ @@ -60,6 +248,7 @@ def test_console_audio_api_success(app: Flask, monkeypatch: pytest.MonkeyPatch) (AudioTooLargeServiceError("too big"), AudioTooLargeError), (UnsupportedAudioTypeServiceError(), UnsupportedAudioTypeError), (ProviderNotSupportSpeechToTextServiceError(), ProviderNotSupportSpeechToTextError), + (SpeechToTextDisabledServiceError(), SpeechToTextDisabledError), (ProviderTokenNotInitError("token"), ProviderNotInitializeError), (QuotaExceededError(), ProviderQuotaExceededError), (ModelCurrentlyNotSupportError(), ProviderModelCurrentlyNotSupportError), diff --git a/api/tests/unit_tests/controllers/console/explore/test_audio.py b/api/tests/unit_tests/controllers/console/explore/test_audio.py index c1930cfbcf5..387cb97f61b 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_audio.py +++ b/api/tests/unit_tests/controllers/console/explore/test_audio.py @@ -14,6 +14,7 @@ from controllers.console.app.error import ( ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError, ProviderQuotaExceededError, + SpeechToTextDisabledError, ) from core.errors.error import ( ModelCurrentlyNotSupportError, @@ -25,6 +26,7 @@ from services.app_ref_service import MessageRef from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, + SpeechToTextDisabledServiceError, ) @@ -185,6 +187,22 @@ class TestChatAudioApi: with pytest.raises(audio_module.ProviderNotSupportSpeechToTextError): self.method(installed_app) + def test_speech_to_text_disabled(self, app: Flask, installed_app, audio_file): + with ( + app.test_request_context( + "/", + data={"file": audio_file}, + content_type="multipart/form-data", + ), + patch.object( + audio_module.AudioService, + "transcript_asr", + side_effect=SpeechToTextDisabledServiceError(), + ), + ): + with pytest.raises(SpeechToTextDisabledError): + self.method(installed_app) + def test_provider_not_initialized(self, app: Flask, installed_app, audio_file): with ( app.test_request_context( diff --git a/api/tests/unit_tests/controllers/console/explore/test_trial.py b/api/tests/unit_tests/controllers/console/explore/test_trial.py index 60cd09ebac2..d810bf4da70 100644 --- a/api/tests/unit_tests/controllers/console/explore/test_trial.py +++ b/api/tests/unit_tests/controllers/console/explore/test_trial.py @@ -18,6 +18,7 @@ from controllers.console.app.error import ( ProviderModelCurrentlyNotSupportError, ProviderNotInitializeError, ProviderQuotaExceededError, + SpeechToTextDisabledError, ) from controllers.console.explore.error import ( NotChatAppError, @@ -35,6 +36,7 @@ from models import Account from models.account import TenantStatus from models.model import AppMode from services.app_ref_service import MessageRef +from services.errors.audio import SpeechToTextDisabledServiceError from services.errors.conversation import ConversationNotExistsError from services.errors.llm import InvokeRateLimitError @@ -784,6 +786,24 @@ class TestTrialChatAudioApi: with pytest.raises(module.ProviderNotSupportSpeechToTextError): method(api, account, trial_app_chat) + def test_speech_to_text_disabled(self, app: Flask, trial_app_chat: MagicMock, account: Account) -> None: + api = module.TrialChatAudioApi() + method = unwrap(api.post) + file_data = _file_data() + + with ( + app.test_request_context( + "/", method="POST", data={"file": (file_data, "test.wav")}, content_type="multipart/form-data" + ), + patch.object( + module.AudioService, + "transcript_asr", + side_effect=SpeechToTextDisabledServiceError(), + ), + ): + with pytest.raises(SpeechToTextDisabledError): + method(api, account, trial_app_chat) + def test_provider_not_init(self, app: Flask, trial_app_chat: MagicMock, account: Account) -> None: api = module.TrialChatAudioApi() method = unwrap(api.post) diff --git a/api/tests/unit_tests/controllers/service_api/app/test_audio.py b/api/tests/unit_tests/controllers/service_api/app/test_audio.py index 3615eb6d209..1dd7175c086 100644 --- a/api/tests/unit_tests/controllers/service_api/app/test_audio.py +++ b/api/tests/unit_tests/controllers/service_api/app/test_audio.py @@ -28,6 +28,7 @@ from controllers.service_api.app.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError @@ -39,6 +40,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -207,6 +209,7 @@ class TestAudioApi: (AudioTooLargeServiceError("too big"), AudioTooLargeError), (UnsupportedAudioTypeServiceError(), UnsupportedAudioTypeError), (ProviderNotSupportSpeechToTextServiceError(), ProviderNotSupportSpeechToTextError), + (SpeechToTextDisabledServiceError(), SpeechToTextDisabledError), (ProviderTokenNotInitError("token"), ProviderNotInitializeError), (QuotaExceededError(), ProviderQuotaExceededError), (ModelCurrentlyNotSupportError(), ProviderModelCurrentlyNotSupportError), diff --git a/api/tests/unit_tests/controllers/web/test_audio.py b/api/tests/unit_tests/controllers/web/test_audio.py index a3f773f10ff..4764a95940b 100644 --- a/api/tests/unit_tests/controllers/web/test_audio.py +++ b/api/tests/unit_tests/controllers/web/test_audio.py @@ -18,6 +18,7 @@ from controllers.web.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, ) from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError @@ -27,6 +28,7 @@ from services.errors.audio import ( AudioTooLargeServiceError, NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -83,6 +85,16 @@ class TestAudioApi: with pytest.raises(ProviderNotSupportSpeechToTextError): AudioApi().post(_app_model(), _end_user()) + @patch( + "controllers.web.audio.AudioService.transcript_asr", + side_effect=SpeechToTextDisabledServiceError(), + ) + def test_speech_to_text_disabled(self, mock_asr: MagicMock, app: Flask) -> None: + data = {"file": (BytesIO(b"x"), "x.mp3")} + with app.test_request_context("/audio-to-text", method="POST", data=data, content_type="multipart/form-data"): + with pytest.raises(SpeechToTextDisabledError): + AudioApi().post(_app_model(), _end_user()) + @patch( "controllers.web.audio.AudioService.transcript_asr", side_effect=ProviderTokenNotInitError(description="no token"), diff --git a/api/tests/unit_tests/controllers/web/test_error.py b/api/tests/unit_tests/controllers/web/test_error.py index 9bf58ee0c5f..78a72fc624f 100644 --- a/api/tests/unit_tests/controllers/web/test_error.py +++ b/api/tests/unit_tests/controllers/web/test_error.py @@ -22,6 +22,7 @@ from controllers.web.error import ( ProviderNotInitializeError, ProviderNotSupportSpeechToTextError, ProviderQuotaExceededError, + SpeechToTextDisabledError, UnsupportedAudioTypeError, WebAppAuthAccessDeniedError, WebAppAuthRequiredError, @@ -45,6 +46,7 @@ _ERROR_SPECS: list[tuple[type, str, int]] = [ (AudioTooLargeError, "audio_too_large", 413), (UnsupportedAudioTypeError, "unsupported_audio_type", 415), (ProviderNotSupportSpeechToTextError, "provider_not_support_speech_to_text", 400), + (SpeechToTextDisabledError, "speech_to_text_disabled", 400), (WebAppAuthRequiredError, "web_sso_auth_required", 401), (WebAppAuthAccessDeniedError, "web_app_access_denied", 401), (InvokeRateLimitError, "rate_limit_error", 429), diff --git a/api/tests/unit_tests/services/agent/test_agent_services.py b/api/tests/unit_tests/services/agent/test_agent_services.py index 22c3f4c5500..2f80599689d 100644 --- a/api/tests/unit_tests/services/agent/test_agent_services.py +++ b/api/tests/unit_tests/services/agent/test_agent_services.py @@ -116,6 +116,25 @@ def test_agent_soul_has_model(): assert agent_soul_has_model(AgentSoulConfig()) is False +def test_get_published_agent_soul_for_app_uses_active_snapshot(): + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + agent = SimpleNamespace(id="agent-1", active_config_snapshot_id="version-1") + version = SimpleNamespace(config_snapshot_dict=agent_soul.model_dump(mode="json")) + service = AgentRosterService(FakeSession(scalar=[agent, version])) + + result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="app-1") + + assert result == agent_soul + + +def test_get_published_agent_soul_for_app_returns_none_without_backing_agent(): + service = AgentRosterService(FakeSession(scalar=[None])) + + result = service.get_published_agent_soul_for_app(tenant_id="tenant-1", app_id="legacy-app-1") + + assert result is None + + def test_load_workflow_composer_returns_empty_state(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(AgentComposerService, "_get_draft_workflow", lambda **kwargs: SimpleNamespace(id="workflow-1")) monkeypatch.setattr(AgentComposerService, "_get_workflow_binding", lambda **kwargs: None) @@ -632,7 +651,7 @@ def test_publish_agent_app_draft_rejects_missing_model(monkeypatch: pytest.Monke base_snapshot_id="version-1", config_snapshot=AgentSoulConfig(), ) - fake_session = FakeSession(scalar=[agent, draft]) + fake_session = FakeSession(scalar=[agent, draft, None]) def fail_create_config_version(**_kwargs): raise AssertionError("config version must not be created when Agent Soul has no model") @@ -777,6 +796,74 @@ def test_agent_app_build_draft_checkout_and_apply_use_user_isolated_draft(monkey assert fake_session.commits == 1 +@pytest.mark.parametrize( + ("draft_type", "account_id"), + [ + (AgentConfigDraftType.DRAFT, None), + (AgentConfigDraftType.DEBUG_BUILD, "account-1"), + ], +) +def test_load_agent_soul_for_debug_selects_requested_draft( + draft_type: AgentConfigDraftType, + account_id: str | None, +): + agent = Agent( + id="agent-1", + tenant_id="tenant-1", + name="Iris", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + status=AgentStatus.ACTIVE, + ) + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + draft = AgentConfigDraft( + tenant_id="tenant-1", + agent_id="agent-1", + draft_type=draft_type, + account_id=account_id, + draft_owner_key=account_id or "", + config_snapshot=agent_soul, + ) + fake_session = FakeSession( + scalar=[draft] if draft_type == AgentConfigDraftType.DEBUG_BUILD else [agent, draft, None] + ) + + result = AgentComposerService.load_agent_soul_for_debug( + tenant_id="tenant-1", + agent_id="agent-1", + account_id="account-1", + draft_type=draft_type, + session=fake_session, + ) + + assert result == agent_soul + + +def test_load_agent_soul_for_debug_requires_existing_build_draft(): + agent = Agent( + id="agent-1", + tenant_id="tenant-1", + name="Iris", + description="", + agent_kind=AgentKind.DIFY_AGENT, + scope=AgentScope.ROSTER, + source=AgentSource.AGENT_APP, + status=AgentStatus.ACTIVE, + ) + fake_session = FakeSession(scalar=[None]) + + with pytest.raises(AgentVersionNotFoundError): + AgentComposerService.load_agent_soul_for_debug( + tenant_id="tenant-1", + agent_id="agent-1", + account_id="account-1", + draft_type=AgentConfigDraftType.DEBUG_BUILD, + session=fake_session, + ) + + def test_agent_app_build_draft_apply_marks_unpublished_when_build_draft_differs(monkeypatch: pytest.MonkeyPatch): agent = Agent( id="agent-1", diff --git a/api/tests/unit_tests/services/test_audio_service.py b/api/tests/unit_tests/services/test_audio_service.py index bd865dd5dd3..a6ccae11e8d 100644 --- a/api/tests/unit_tests/services/test_audio_service.py +++ b/api/tests/unit_tests/services/test_audio_service.py @@ -59,6 +59,7 @@ from unittest.mock import MagicMock, Mock, create_autospec, patch import pytest from werkzeug.datastructures import FileStorage +from models.agent_config_entities import AgentSoulConfig from models.enums import MessageStatus from models.model import App, AppMode, AppModelConfig, Message from models.workflow import Workflow @@ -69,6 +70,7 @@ from services.errors.audio import ( NoAudioUploadedServiceError, ProviderNotSupportSpeechToTextServiceError, ProviderNotSupportTextToSpeechServiceError, + SpeechToTextDisabledServiceError, UnsupportedAudioTypeServiceError, ) @@ -267,6 +269,117 @@ class TestAudioServiceASR: # Assert assert result == {"text": "Workflow transcribed text"} + @patch("services.audio_service.AgentRosterService", autospec=True) + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_asr_success_published_agent_mode( + self, + mock_model_manager_class, + mock_roster_service_class, + factory: AudioServiceTestDataFactory, + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = agent_soul + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Published Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_asr(app_model=app, file=file, end_user="end-user-1") + + assert result == {"text": "Published Agent transcript"} + mock_roster_service_class.return_value.get_published_agent_soul_for_app.assert_called_once_with( + tenant_id=app.tenant_id, + app_id=app.id, + ) + + @patch("services.audio_service.AgentRosterService", autospec=True) + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_asr_legacy_agent_falls_back_to_app_model_config( + self, + mock_model_manager_class, + mock_roster_service_class, + factory: AudioServiceTestDataFactory, + ): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + mock_roster_service_class.return_value.get_published_agent_soul_for_app.return_value = None + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Legacy Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_asr(app_model=app, file=file) + + assert result == {"text": "Legacy Agent transcript"} + + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_agent_asr_uses_agent_soul_feature( + self, mock_model_manager_class, factory: AudioServiceTestDataFactory + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": True}}}) + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Agent transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_agent_asr( + app_model=app, + agent_soul=agent_soul, + file=file, + end_user="account-1", + ) + + assert result == {"text": "Agent transcript"} + mock_model_manager_class.assert_called_once_with(tenant_id=app.tenant_id, user_id="account-1") + + @pytest.mark.parametrize( + "agent_soul", + [ + AgentSoulConfig(), + AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}}), + ], + ) + def test_transcript_agent_asr_rejects_disabled_feature( + self, factory: AudioServiceTestDataFactory, agent_soul: AgentSoulConfig + ): + app = factory.create_app_mock(mode=AppMode.AGENT) + file = factory.create_file_storage_mock() + + with pytest.raises(SpeechToTextDisabledServiceError): + AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file) + + @patch("services.audio_service.ModelManager.for_tenant", autospec=True) + def test_transcript_agent_asr_preserves_legacy_feature_fallback( + self, mock_model_manager_class, factory: AudioServiceTestDataFactory + ): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}} + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + mock_model_instance = MagicMock() + mock_model_instance.invoke_speech2text.return_value = "Legacy feature transcript" + mock_model_manager_class.return_value.get_default_model_instance.return_value = mock_model_instance + + result = AudioService.transcript_agent_asr( + app_model=app, + agent_soul=AgentSoulConfig(), + file=file, + ) + + assert result == {"text": "Legacy feature transcript"} + + def test_transcript_agent_asr_soul_disabled_overrides_legacy_feature(self, factory: AudioServiceTestDataFactory): + app_model_config = factory.create_app_model_config_mock(speech_to_text_dict={"enabled": True}) + app_model_config.to_dict.return_value = {"speech_to_text": {"enabled": True}} + app = factory.create_app_mock(mode=AppMode.AGENT, app_model_config=app_model_config) + file = factory.create_file_storage_mock() + agent_soul = AgentSoulConfig.model_validate({"app_features": {"speech_to_text": {"enabled": False}}}) + + with pytest.raises(SpeechToTextDisabledServiceError): + AudioService.transcript_agent_asr(app_model=app, agent_soul=agent_soul, file=file) + def test_transcript_asr_raises_error_when_feature_disabled_chat_mode(self, factory: AudioServiceTestDataFactory): """Test that ASR raises error when speech-to-text is disabled in CHAT mode.""" # Arrange @@ -278,7 +391,7 @@ class TestAudioServiceASR: file = factory.create_file_storage_mock() # Act & Assert - with pytest.raises(ValueError, match="Speech to text is not enabled"): + with pytest.raises(SpeechToTextDisabledServiceError): AudioService.transcript_asr(app_model=app, file=file) def test_transcript_asr_raises_error_when_feature_disabled_workflow_mode( @@ -294,7 +407,7 @@ class TestAudioServiceASR: file = factory.create_file_storage_mock() # Act & Assert - with pytest.raises(ValueError, match="Speech to text is not enabled"): + with pytest.raises(SpeechToTextDisabledServiceError): AudioService.transcript_asr(app_model=app, file=file) def test_transcript_asr_raises_error_when_workflow_missing(self, factory: AudioServiceTestDataFactory): @@ -307,7 +420,7 @@ class TestAudioServiceASR: file = factory.create_file_storage_mock() # Act & Assert - with pytest.raises(ValueError, match="Speech to text is not enabled"): + with pytest.raises(SpeechToTextDisabledServiceError): AudioService.transcript_asr(app_model=app, file=file) def test_transcript_asr_raises_error_when_no_file_uploaded(self, factory: AudioServiceTestDataFactory): diff --git a/e2e/.env.example b/e2e/.env.example index 530736a3546..6af88b6e4b9 100644 --- a/e2e/.env.example +++ b/e2e/.env.example @@ -12,6 +12,9 @@ E2E_AGENT_DECISION_MODEL_PROVIDER=openai E2E_AGENT_DECISION_MODEL_NAME=gpt-5.5 E2E_AGENT_DECISION_MODEL_TYPE=llm +E2E_SPEECH_TO_TEXT_MODEL_PROVIDER=openai +E2E_SPEECH_TO_TEXT_MODEL_NAME=gpt-4o-mini-transcribe + E2E_MODEL_PROVIDER_CREDENTIALS_JSON='{"openai_api_key":"replace-with-real-key"}' # Optional: external runtime CI/local profile. Defaults prepare Agent V2 runtime diff --git a/e2e/AGENTS.md b/e2e/AGENTS.md index 6f41c0e921e..d5bcb0bc868 100644 --- a/e2e/AGENTS.md +++ b/e2e/AGENTS.md @@ -209,10 +209,13 @@ Feature: Create dataset - `@fresh` — only runs in `e2e:full` mode (requires uninitialized instance) - `@external-model` — scenario execution can call a real model provider. Use this only for runtime requests, not for scenarios that only require an active model fixture. - `@external-tool` — scenario execution can call a real third-party tool provider. Use this only for runtime tool execution, not for plugin installation, discovery, or local deterministic tools. +- `@microphone` — runs the scenario in an isolated Chromium instance backed by the checked-in fake audio fixture and grants microphone permission only to that scenario context. - `@skip` — excluded from all runs External runtime commands are opt-in. `pnpm -C e2e e2e:external:prepare` reads `E2E_EXTERNAL_RUNTIME_SEED_SPECS`, defaulting to `agent-v2:external-runtime`, and runs the matching seed packs before the external suite. `pnpm -C e2e e2e:external` reads `E2E_EXTERNAL_RUNTIME_TAGS`, defaulting to `(@external-model or @external-tool) and not @feature-gated and not @skip and not @preview`. +The Agent v2 external runtime seed also prepares the workspace default Speech-to-Text model. `E2E_SPEECH_TO_TEXT_MODEL_PROVIDER` and `E2E_SPEECH_TO_TEXT_MODEL_NAME` select an existing model or the model configured through `E2E_MODEL_PROVIDER_CREDENTIALS_JSON`; they default to `openai` and `gpt-4o-mini-transcribe`. + Some external runtime scenarios need feature-owned services in addition to a real model or tool provider. Do not overload `@external-model` or `@external-tool` to mean those services are available. For Agent v2, scenarios that require the standalone `dify-agent` run server use the feature tag `@agent-backend-runtime` plus the explicit step `the Agent v2 runtime backend is available`. Run them with `E2E_START_AGENT_BACKEND=1` to let E2E start `dify-agent` and the shellctl local sandbox required by its `dify.config`/`dify.shell` runtime layers, or set `E2E_AGENT_BACKEND_URL`/`AGENT_BACKEND_BASE_URL` when an existing server should be reused. Keep scenarios short and declarative. Each step should describe **what** the user does, not **how** the UI works. diff --git a/e2e/features/agent-v2/AGENTS.md b/e2e/features/agent-v2/AGENTS.md index 9f8081970fc..5c45561fafb 100644 --- a/e2e/features/agent-v2/AGENTS.md +++ b/e2e/features/agent-v2/AGENTS.md @@ -36,6 +36,8 @@ Use tags in three layers: - `@publish` — publish and publish-bar state. - `@access-point` — Web app, Backend service API, and Workflow access surfaces. - `@stable-model` — active model fixture dependency. Apply this to every scenario that includes `the Agent Builder stable chat model is available` or otherwise requires an active model configured in the workspace. +- `@speech-to-text-model` — active workspace default Speech-to-Text model dependency. Apply this to scenarios that include `the workspace default speech-to-text model is active`. +- `@microphone` — deterministic Chromium fake microphone dependency. The scenario hook launches a fake-audio browser and grants microphone permission only to tagged scenarios. - `@agent-decision-model` — stronger active model fixture dependency for scenarios that specifically validate Agent autonomous planning or resource-selection behavior, such as generated-query Knowledge Retrieval. Do not use it for scenarios that only need a model to exist or answer a deterministic prompt. - `@tool-fixture` — preseeded Tool dependency such as `JSON Process / JSON Replace` or `Tavily / Tavily Search`. - `@skill-fixture` — checked-in or preseeded Skill dependency such as `e2e-summary-skill`. @@ -75,6 +77,7 @@ Keep Agent v2 step definitions grouped by user capability, not by DOM component - `access-point-service-api.steps.ts` — Backend service API entrypoints, keys, API reference, and service requests. - `access-point-workflow.steps.ts` — Workflow access references. - `preflight.steps.ts` — explicit `Given` entrypoints for Agent Builder preflight resources. +- `speech-to-text.steps.ts` — Agent Build voice input, multipart request, and transcribed input behavior. Cucumber step definitions are globally registered. Do not duplicate the same step text across files, even if one is written as `Given` and another as `Then`. @@ -87,6 +90,7 @@ Agent v2 business state belongs under `world.agentBuilder`; do not keep adding A Use the existing namespace shape: - `world.agentBuilder.preflight.stableModel` +- `world.agentBuilder.preflight.speechToTextModel` - `world.agentBuilder.preflight.brokenModel` - `world.agentBuilder.preflight.preseededResources` - `world.agentBuilder.accessPoint.serviceApiBaseURL` @@ -98,6 +102,7 @@ Use the existing namespace shape: - `world.agentBuilder.accessPoint.workflowReferencePage` - `world.agentBuilder.accessPoint.composerDraftSnapshot` - `world.agentBuilder.configure.concurrentPage` +- `world.agentBuilder.speechToText.request` - `world.agentBuilder.workflow.agentConsolePage` - `world.agentBuilder.workflow.outputVariables` @@ -163,6 +168,8 @@ Use `the Agent v2 runtime backend is available` before scenarios tagged `@agent- Use `the Agent Builder stable chat model is available` before scenarios that need a real Agent Soul model configuration. This includes true runtime scenarios, model-backed build-mode assertions, and Workflow Agent v2 node setup because the backend rejects Agent nodes without model config. Do not add the model preflight to pure navigation or identity checks unless the setup API itself requires model config. `E2E_STABLE_MODEL_PROVIDER`, `E2E_STABLE_MODEL_NAME`, and optional `E2E_STABLE_MODEL_TYPE` are selectors for a model already configured in the workspace; they are not provider credentials. The step defaults to `openai` / `gpt-5-nano` / `llm`, verifies the selected model is present and `active` through `/console/api/workspaces/current/models/model-types/{type}`, then stores it on `DifyWorld.agentBuilder.preflight.stableModel`. +Use `the workspace default speech-to-text model is active` before real speech-to-text scenarios. The preflight reads `/console/api/workspaces/current/default-model?model_type=speech2text`, verifies that same provider/model is active in the Speech-to-Text model list, and stores it on `DifyWorld.agentBuilder.preflight.speechToTextModel`. It must not configure a model or provider credential. External runtime preparation selects `E2E_SPEECH_TO_TEXT_MODEL_PROVIDER` / `E2E_SPEECH_TO_TEXT_MODEL_NAME`, defaulting to `openai` / `gpt-4o-mini-transcribe`, reuses `E2E_MODEL_PROVIDER_CREDENTIALS_JSON` when provider setup is required, and selects the active model as the workspace default. + Use `the Agent Builder agent-decision chat model is available` before scenarios that need a stronger model to exercise Agent autonomous planning, generated query selection, or tool/resource choice. `E2E_AGENT_DECISION_MODEL_PROVIDER`, `E2E_AGENT_DECISION_MODEL_NAME`, and optional `E2E_AGENT_DECISION_MODEL_TYPE` are selectors for a second active model fixture, defaulting to `openai` / `gpt-5.5` / `llm`. The step stores the model on `DifyWorld.agentBuilder.preflight.agentDecisionModel`. Do not use this fixture as a broad replacement for `@stable-model`; it is intentionally narrower and costlier. Keep `@stable-model` on Build draft apply scenarios that click `Apply`. The current product path calls `/build-chat/finalize` before applying the draft, and the backend returns `model is required` when the Agent Soul has no model config. Discard-only and pending-draft isolation scenarios can stay model-free when they do not finalize the Build draft. diff --git a/e2e/features/agent-v2/preflight.feature b/e2e/features/agent-v2/preflight.feature index aa130331304..6450f7fbdf9 100644 --- a/e2e/features/agent-v2/preflight.feature +++ b/e2e/features/agent-v2/preflight.feature @@ -14,6 +14,11 @@ Feature: Agent Builder preseeded environment Given I am signed in as the default E2E admin And the Agent Builder stable chat model is available + @speech-to-text-model + Scenario: Default speech-to-text model is available + Given I am signed in as the default E2E admin + And the workspace default speech-to-text model is active + @agent-decision-model Scenario: Agent-decision chat model is available Given I am signed in as the default E2E admin diff --git a/e2e/features/agent-v2/speech-to-text.feature b/e2e/features/agent-v2/speech-to-text.feature new file mode 100644 index 00000000000..8a85f752add --- /dev/null +++ b/e2e/features/agent-v2/speech-to-text.feature @@ -0,0 +1,13 @@ +@agent-v2 @authenticated +Feature: Agent v2 speech-to-text + @build @speech-to-text @microphone @external-model @speech-to-text-model + Scenario: Recorded speech is transcribed into the current Agent input + Given I am signed in as the default E2E admin + And the workspace default speech-to-text model is active + And an Agent v2 test agent with speech-to-text enabled has been created via API + When I open the Agent v2 configure page + And I start Agent v2 voice input + And I stop Agent v2 voice input after the fixture speech has played + Then the Agent v2 speech-to-text request should succeed + And the transcribed fixture phrase "Purple elephant seven" should appear in the Agent v2 input + And the Agent v2 input should regain focus after transcription diff --git a/e2e/features/agent-v2/support/agent-builder-resources.ts b/e2e/features/agent-v2/support/agent-builder-resources.ts index a47cb64a0b2..852c211e625 100644 --- a/e2e/features/agent-v2/support/agent-builder-resources.ts +++ b/e2e/features/agent-v2/support/agent-builder-resources.ts @@ -1,5 +1,6 @@ export const agentBuilderPreseededResources = { stableChatModel: 'E2E Stable Chat Model', + speechToTextModel: 'Workspace default Speech-to-Text model', summarySkill: 'e2e-summary-skill', jsonReplaceTool: 'JSON Process / JSON Replace', tavilySearchTool: 'Tavily / Tavily Search', diff --git a/e2e/features/agent-v2/support/agent-soul.ts b/e2e/features/agent-v2/support/agent-soul.ts index 367bed2530d..7b2b7908409 100644 --- a/e2e/features/agent-v2/support/agent-soul.ts +++ b/e2e/features/agent-v2/support/agent-soul.ts @@ -85,6 +85,18 @@ export function createAgentSoulConfigWithModel( } } +export function createAgentSoulConfigWithSpeechToText(agentSoul: AgentSoulConfig): AgentSoulConfig { + return { + ...agentSoul, + app_features: { + ...agentSoul.app_features, + speech_to_text: { + enabled: true, + }, + }, + } +} + export function createPublishableAgentSoulConfig(agentSoul: AgentSoulConfig): AgentSoulConfig { if (agentSoul.model) return agentSoul diff --git a/e2e/features/agent-v2/support/preflight/models.ts b/e2e/features/agent-v2/support/preflight/models.ts index 598e104108e..23006a18494 100644 --- a/e2e/features/agent-v2/support/preflight/models.ts +++ b/e2e/features/agent-v2/support/preflight/models.ts @@ -1,4 +1,7 @@ -import type { ProviderWithModelsResponse } from '@dify/contracts/api/console/workspaces/types.gen' +import type { + DefaultModelDataResponse, + ProviderWithModelsResponse, +} from '@dify/contracts/api/console/workspaces/types.gen' import type { DifyWorld } from '../../../support/world' import { createApiContext, expectApiResponseOK } from '../../../../support/api' import { agentBuilderPreseededResources } from '../agent-builder-resources' @@ -151,6 +154,49 @@ export async function skipMissingAgentBuilderStableChatModel( }) } +export async function skipMissingAgentBuilderSpeechToTextModel( + world: DifyWorld, +): Promise<'skipped' | NonNullable> { + const ctx = await createApiContext() + let defaultModel: NonNullable + + try { + const response = await ctx.get( + '/console/api/workspaces/current/default-model?model_type=speech2text', + ) + await expectApiResponseOK(response, `Check ${agentBuilderPreseededResources.speechToTextModel}`) + const body = (await response.json()) as DefaultModelDataResponse + if (!body.data) { + return skipBlockedPrecondition( + world, + `${agentBuilderPreseededResources.speechToTextModel} is not configured.`, + { + owner: 'model-provider/seed', + remediation: + 'Configure an active workspace default Speech-to-Text model before running the external scenario.', + }, + ) + } + defaultModel = body.data + } finally { + await ctx.dispose() + } + + return skipMissingAgentBuilderModel( + world, + { + ok: true, + provider: defaultModel.provider.provider, + resourceName: agentBuilderPreseededResources.speechToTextModel, + type: 'speech2text', + value: defaultModel.model, + }, + { + requireActive: true, + }, + ) +} + export async function skipMissingAgentBuilderAgentDecisionChatModel( world: DifyWorld, ): Promise<'skipped' | NonNullable> { diff --git a/e2e/features/agent-v2/support/seed.ts b/e2e/features/agent-v2/support/seed.ts index 167ef544e35..cee59cefa9d 100644 --- a/e2e/features/agent-v2/support/seed.ts +++ b/e2e/features/agent-v2/support/seed.ts @@ -11,6 +11,7 @@ import type { } from '@dify/contracts/api/console/datasets/types.gen' import type { AvailableModelListResponse, + DefaultModelDataResponse, ModelProviderListResponse, } from '@dify/contracts/api/console/workspaces/types.gen' import type { SeedContext, SeedResource, SeedTask } from '../../../support/seed' @@ -65,6 +66,8 @@ type ToolResource = SeedResource & { } const modelCredentialEnv = 'E2E_MODEL_PROVIDER_CREDENTIALS_JSON' +const speechToTextModelProviderEnv = 'E2E_SPEECH_TO_TEXT_MODEL_PROVIDER' +const speechToTextModelNameEnv = 'E2E_SPEECH_TO_TEXT_MODEL_NAME' const marketplacePluginIdsEnv = 'E2E_MARKETPLACE_PLUGIN_IDS' const marketplacePluginUniqueIdentifiersEnv = 'E2E_MARKETPLACE_PLUGIN_UNIQUE_IDENTIFIERS' const oauthToolCredentialIdEnv = 'E2E_OAUTH_TOOL_CREDENTIAL_ID' @@ -104,6 +107,12 @@ const agentDecisionModelConfig = (): StableModel => ({ type: process.env.E2E_AGENT_DECISION_MODEL_TYPE?.trim() || 'llm', }) +const speechToTextModelConfig = (): StableModel => ({ + name: process.env[speechToTextModelNameEnv]?.trim() || 'gpt-4o-mini-transcribe', + provider: process.env[speechToTextModelProviderEnv]?.trim() || 'openai', + type: 'speech2text', +}) + const parseJsonEnv = (envName: string) => { const raw = process.env[envName]?.trim() if (!raw) return { ok: false as const, reason: `${envName} is required.` } @@ -121,7 +130,7 @@ const parseJsonEnv = (envName: string) => { } } -const findChatModel = async (config: StableModel, title: string) => { +const findModel = async (config: StableModel, title: string) => { const ctx = await createApiContext() try { const response = await ctx.get( @@ -237,7 +246,7 @@ const upsertStableProviderCredential = async ( } } -const seedChatModel = async ( +const seedModel = async ( context: SeedContext, { config, @@ -247,7 +256,7 @@ const seedChatModel = async ( title: string }, ) => { - const existing = await findChatModel(config, title) + const existing = await findModel(config, title) const resource = { id: `${existing?.provider ?? config.provider}/${existing?.name ?? config.name}`, kind: 'model', @@ -303,7 +312,7 @@ const seedChatModel = async ( } } - const seeded = await findChatModel(config, title) + const seeded = await findModel(config, title) if (seeded?.status !== activeModelStatus) { return blocked( title, @@ -319,17 +328,100 @@ const seedChatModel = async ( } const seedStableModel = async (context: SeedContext) => - seedChatModel(context, { + seedModel(context, { config: stableModelConfig(), title: agentBuilderPreseededResources.stableChatModel, }) const seedAgentDecisionModel = async (context: SeedContext) => - seedChatModel(context, { + seedModel(context, { config: agentDecisionModelConfig(), title: agentBuilderPreseededResources.agentDecisionChatModel, }) +const getDefaultModel = async (modelType: string) => { + const ctx = await createApiContext() + try { + const response = await ctx.get( + `/console/api/workspaces/current/default-model?${buildQuery({ model_type: modelType })}`, + ) + await expectApiResponseOK(response, `Get default ${modelType} model`) + const body = (await response.json()) as DefaultModelDataResponse + return body.data + } finally { + await ctx.dispose() + } +} + +const setDefaultModel = async (model: StableModel) => { + const ctx = await createApiContext() + try { + const response = await ctx.post('/console/api/workspaces/current/default-model', { + data: { + model_settings: [ + { + model: model.name, + model_type: model.type, + provider: model.provider, + }, + ], + }, + }) + await expectApiResponseOK(response, `Set default ${model.type} model`) + } finally { + await ctx.dispose() + } +} + +const seedSpeechToTextModel = async (context: SeedContext) => { + const config = speechToTextModelConfig() + const title = agentBuilderPreseededResources.speechToTextModel + const modelResult = await seedModel(context, { config, title }) + if (modelResult.status === 'blocked' || modelResult.status === 'skipped') return modelResult + + const model = await findModel(config, title) + if (!model || model.status !== activeModelStatus) + return blocked(title, `${config.provider}/${config.name} is not active after model setup.`) + + const resource = { + id: `${model.provider}/${model.name}`, + kind: 'model', + name: title, + } + const defaultModel = await getDefaultModel(config.type) + const isExpectedDefault = + defaultModel?.model === model.name && + matchesProvider(defaultModel.provider.provider, model.provider) + + if (isExpectedDefault) + return modelResult.status === 'updated' ? modelResult : verified(title, resource) + + if (context.dryRun) + return skipped( + title, + `Would set ${model.provider}/${model.name} as the workspace default Speech-to-Text model.`, + ) + + await setDefaultModel({ + name: model.name, + provider: model.provider, + type: config.type, + }) + + const updatedDefaultModel = await getDefaultModel(config.type) + if ( + updatedDefaultModel?.model !== model.name || + !matchesProvider(updatedDefaultModel.provider.provider, model.provider) + ) { + return blocked( + title, + `${model.provider}/${model.name} was not selected as the workspace default Speech-to-Text model.`, + ) + } + + return updated(title, resource) +} + type BuiltinToolProvider = { label?: { en_US?: string; zh_Hans?: string } name: string @@ -1003,10 +1095,19 @@ const agentV2FullSeedTasks = (): SeedTask[] => [ }, ] +const agentV2ExternalRuntimeSeedTasks = (): SeedTask[] => [ + ...agentV2BaseSeedTasks(), + { + id: 'speech-to-text-model', + title: agentBuilderPreseededResources.speechToTextModel, + run: seedSpeechToTextModel, + }, +] + export const createAgentV2SeedTasks = (profile: string = 'full'): SeedTask[] => { if (profile === 'full') return agentV2FullSeedTasks() - if (profile === 'external-runtime') return agentV2BaseSeedTasks() + if (profile === 'external-runtime') return agentV2ExternalRuntimeSeedTasks() throw new Error(`Unknown Agent V2 seed profile "${profile}".`) } diff --git a/e2e/features/step-definitions/agent-v2/preflight.steps.ts b/e2e/features/step-definitions/agent-v2/preflight.steps.ts index 55c24d69931..89e3d2515d2 100644 --- a/e2e/features/step-definitions/agent-v2/preflight.steps.ts +++ b/e2e/features/step-definitions/agent-v2/preflight.steps.ts @@ -23,6 +23,7 @@ import { import { skipMissingAgentBuilderAgentDecisionChatModel, skipMissingAgentBuilderBrokenChatModel, + skipMissingAgentBuilderSpeechToTextModel, skipMissingAgentBuilderStableChatModel, } from '../../agent-v2/support/preflight/models' import { skipMissingPreseededTool } from '../../agent-v2/support/preflight/tools' @@ -34,6 +35,13 @@ Given('the Agent Builder stable chat model is available', async function (this: this.agentBuilder.preflight.stableModel = stableModel }) +Given('the workspace default speech-to-text model is active', async function (this: DifyWorld) { + const speechToTextModel = await skipMissingAgentBuilderSpeechToTextModel(this) + if (speechToTextModel === 'skipped') return speechToTextModel + + this.agentBuilder.preflight.speechToTextModel = speechToTextModel +}) + Given('the Agent Builder agent-decision chat model is available', async function (this: DifyWorld) { const agentDecisionModel = await skipMissingAgentBuilderAgentDecisionChatModel(this) if (agentDecisionModel === 'skipped') return agentDecisionModel diff --git a/e2e/features/step-definitions/agent-v2/speech-to-text.steps.ts b/e2e/features/step-definitions/agent-v2/speech-to-text.steps.ts new file mode 100644 index 00000000000..f16da20de6b --- /dev/null +++ b/e2e/features/step-definitions/agent-v2/speech-to-text.steps.ts @@ -0,0 +1,118 @@ +import type { DifyWorld } from '../../support/world' +import { Given, Then, When } from '@cucumber/cucumber' +import { expect } from '@playwright/test' +import { voiceInputTestMaterial } from '../../../support/test-materials' +import { createConfiguredTestAgent } from '../../agent-v2/support/agent' +import { + createAgentSoulConfigWithSpeechToText, + normalAgentSoulConfig, +} from '../../agent-v2/support/agent-soul' +import { getCurrentAgentId } from './configure-helpers' + +const getAgentInput = (world: DifyWorld) => + world.getPage().getByPlaceholder('Describe what your agent should do') + +const normalizeFixturePhrase = (value: string) => + value + .toLocaleLowerCase() + .replaceAll('seven', '7') + .replaceAll(/[^\p{L}\p{N}]+/gu, '') + +Given( + 'an Agent v2 test agent with speech-to-text enabled has been created via API', + async function (this: DifyWorld) { + if (!this.agentBuilder.preflight.speechToTextModel) { + throw new Error( + 'Create a speech-to-text Agent v2 test agent after the default Speech-to-Text model preflight.', + ) + } + + const agent = await createConfiguredTestAgent({ + agentSoul: createAgentSoulConfigWithSpeechToText(normalAgentSoulConfig), + }) + this.createdAgentIds.push(agent.id) + this.lastCreatedAgentName = agent.name + this.lastCreatedAgentRole = agent.role ?? undefined + }, +) + +When('I start Agent v2 voice input', async function (this: DifyWorld) { + const page = this.getPage() + + await expect(getAgentInput(this)).toBeVisible({ timeout: 30_000 }) + await page.getByRole('button', { name: 'Voice input' }).click() + await expect(page.getByText('Speak now...')).toBeVisible({ timeout: 30_000 }) +}) + +When( + 'I stop Agent v2 voice input after the fixture speech has played', + async function (this: DifyWorld) { + const page = this.getPage() + const agentId = getCurrentAgentId(this) + + await expect(page.getByTestId('voice-input-timer')).toHaveText( + voiceInputTestMaterial.recordingDuration, + { timeout: 15_000 }, + ) + + const responsePromise = page.waitForResponse( + (response) => + response.request().method() === 'POST' && + new URL(response.url()).pathname === `/console/api/agent/${agentId}/audio-to-text`, + ) + await page.getByRole('button', { name: 'Stop recording' }).click() + const response = await responsePromise + const request = response.request() + + this.agentBuilder.speechToText.request = { + contentType: request.headers()['content-type'] ?? '', + path: new URL(response.url()).pathname, + status: response.status(), + } + }, +) + +Then('the Agent v2 speech-to-text request should succeed', async function (this: DifyWorld) { + const request = this.agentBuilder.speechToText.request + if (!request) throw new Error('No Agent v2 speech-to-text request was captured.') + + expect(request).toEqual( + expect.objectContaining({ + contentType: expect.stringContaining('multipart/form-data'), + path: `/console/api/agent/${getCurrentAgentId(this)}/audio-to-text`, + status: 200, + }), + ) +}) + +Then( + 'the transcribed fixture phrase {string} should appear in the Agent v2 input', + async function (this: DifyWorld, expectedPhrase: string) { + const input = getAgentInput(this) + await expect + .poll(async () => normalizeFixturePhrase(await input.inputValue()), { timeout: 60_000 }) + .toBe(normalizeFixturePhrase(expectedPhrase)) + }, +) + +Then( + 'the Agent v2 input should regain focus after transcription', + async function (this: DifyWorld) { + const input = getAgentInput(this) + await expect(input).toBeFocused() + + const selection = await input.evaluate((element) => { + const textarea = element as HTMLTextAreaElement + return { + end: textarea.selectionEnd, + length: textarea.value.length, + start: textarea.selectionStart, + } + }) + expect(selection).toEqual({ + end: selection.length, + length: selection.length, + start: selection.length, + }) + }, +) diff --git a/e2e/features/support/hooks.ts b/e2e/features/support/hooks.ts index cd3415ae44f..b35668fca86 100644 --- a/e2e/features/support/hooks.ts +++ b/e2e/features/support/hooks.ts @@ -9,6 +9,7 @@ import { chromium } from '@playwright/test' import { AUTH_BOOTSTRAP_TIMEOUT_MS, ensureAuthenticatedState } from '../../fixtures/auth' import { deleteTestApp } from '../../support/api' import { deleteTestDataset } from '../../support/datasets' +import { getVoiceInputTestMaterialPath } from '../../support/test-materials' import { deleteBuiltinToolCredential } from '../../support/tools' import { baseURL, cucumberHeadless, cucumberSlowMo } from '../../test-env' import { deleteTestAgent } from '../agent-v2/support/agent' @@ -22,6 +23,7 @@ const e2eRoot = fileURLToPath(new URL('../..', import.meta.url)) const artifactsDir = path.join(e2eRoot, 'cucumber-report', 'artifacts') let browser: Browser | undefined +let microphoneBrowserPromise: Promise | undefined setDefaultTimeout(60_000) @@ -101,17 +103,43 @@ BeforeAll({ timeout: AUTH_BOOTSTRAP_TIMEOUT_MS }, async () => { await ensureAuthenticatedState(browser, baseURL) }) +const getMicrophoneBrowser = () => { + microphoneBrowserPromise ??= chromium.launch({ + args: [ + '--use-fake-device-for-media-stream', + '--use-fake-ui-for-media-stream', + `--use-file-for-fake-audio-capture=${getVoiceInputTestMaterialPath()}%noloop`, + ], + headless: cucumberHeadless, + slowMo: cucumberSlowMo, + }) + + return microphoneBrowserPromise +} + Before(async function (this: DifyWorld, { pickle }) { if (!browser) throw new Error('Shared Playwright browser is not available.') - const isUnauthenticatedScenario = pickle.tags.some((tag) => tag.name === '@unauthenticated') + const scenarioTags = pickle.tags.map((tag) => tag.name) + const isMicrophoneScenario = scenarioTags.includes('@microphone') + const isUnauthenticatedScenario = scenarioTags.includes('@unauthenticated') + const scenarioBrowser = isMicrophoneScenario ? await getMicrophoneBrowser() : browser - if (isUnauthenticatedScenario) await this.startUnauthenticatedSession(browser) - else await this.startAuthenticatedSession(browser) + if (isUnauthenticatedScenario) await this.startUnauthenticatedSession(scenarioBrowser) + else await this.startAuthenticatedSession(scenarioBrowser) + + if (isMicrophoneScenario) { + if (!this.context) + throw new Error('Playwright context has not been initialized for the microphone scenario.') + + await this.context.grantPermissions(['microphone'], { + origin: new URL(baseURL).origin, + }) + } this.scenarioStartedAt = Date.now() - const tags = pickle.tags.map((tag) => tag.name).join(' ') + const tags = scenarioTags.join(' ') console.warn(`[e2e] start ${pickle.name}${tags ? ` ${tags}` : ''}`) }) @@ -197,6 +225,9 @@ After(async function (this: DifyWorld, { pickle, result }) { }) AfterAll(async () => { + const microphoneBrowser = await microphoneBrowserPromise?.catch(() => undefined) + await microphoneBrowser?.close() await browser?.close() + microphoneBrowserPromise = undefined browser = undefined }) diff --git a/e2e/features/support/world.ts b/e2e/features/support/world.ts index 5b6736a4554..21e4fa8e18e 100644 --- a/e2e/features/support/world.ts +++ b/e2e/features/support/world.ts @@ -36,12 +36,18 @@ export type AgentV2WorkflowOutputVariable = { name: string type: string } +export type AgentBuilderSpeechToTextRequest = { + contentType: string + path: string + status: number +} export const createAgentBuilderWorldState = () => ({ preflight: { agentDecisionModel: undefined as AgentBuilderChatModel | undefined, brokenModel: undefined as AgentBuilderChatModel | undefined, preseededResources: {} as Record, + speechToTextModel: undefined as AgentBuilderChatModel | undefined, stableModel: undefined as AgentBuilderChatModel | undefined, }, accessPoint: { @@ -57,6 +63,9 @@ export const createAgentBuilderWorldState = () => ({ configure: { concurrentPage: undefined as Page | undefined, }, + speechToText: { + request: undefined as AgentBuilderSpeechToTextRequest | undefined, + }, workflow: { agentConsolePage: undefined as Page | undefined, outputVariables: [] as AgentV2WorkflowOutputVariable[], diff --git a/e2e/fixtures/test-materials/voice-input.wav b/e2e/fixtures/test-materials/voice-input.wav new file mode 100644 index 00000000000..128e73182fd Binary files /dev/null and b/e2e/fixtures/test-materials/voice-input.wav differ diff --git a/e2e/scripts/env.ts b/e2e/scripts/env.ts index d8cc3eb7801..5d33516426f 100644 --- a/e2e/scripts/env.ts +++ b/e2e/scripts/env.ts @@ -76,6 +76,9 @@ export const validateE2eEnv = () => E2E_ADMIN_PASSWORD: process.env.E2E_ADMIN_PASSWORD, E2E_AGENT_BACKEND_PORT: process.env.E2E_AGENT_BACKEND_PORT, E2E_AGENT_BACKEND_URL: process.env.E2E_AGENT_BACKEND_URL, + E2E_AGENT_DECISION_MODEL_NAME: process.env.E2E_AGENT_DECISION_MODEL_NAME, + E2E_AGENT_DECISION_MODEL_PROVIDER: process.env.E2E_AGENT_DECISION_MODEL_PROVIDER, + E2E_AGENT_DECISION_MODEL_TYPE: process.env.E2E_AGENT_DECISION_MODEL_TYPE, E2E_API_URL: process.env.E2E_API_URL, E2E_BASE_URL: process.env.E2E_BASE_URL, E2E_BROKEN_MODEL_NAME: process.env.E2E_BROKEN_MODEL_NAME, @@ -101,6 +104,8 @@ export const validateE2eEnv = () => E2E_SHELLCTL_IMAGE: process.env.E2E_SHELLCTL_IMAGE, E2E_SHELLCTL_PORT: process.env.E2E_SHELLCTL_PORT, E2E_SHELLCTL_URL: process.env.E2E_SHELLCTL_URL, + E2E_SPEECH_TO_TEXT_MODEL_NAME: process.env.E2E_SPEECH_TO_TEXT_MODEL_NAME, + E2E_SPEECH_TO_TEXT_MODEL_PROVIDER: process.env.E2E_SPEECH_TO_TEXT_MODEL_PROVIDER, E2E_START_AGENT_BACKEND: process.env.E2E_START_AGENT_BACKEND, E2E_STABLE_MODEL_NAME: process.env.E2E_STABLE_MODEL_NAME, E2E_STABLE_MODEL_PROVIDER: process.env.E2E_STABLE_MODEL_PROVIDER, @@ -113,6 +118,9 @@ export const validateE2eEnv = () => E2E_ADMIN_PASSWORD: z.string().min(1).optional(), E2E_AGENT_BACKEND_PORT: z.coerce.number().int().positive().max(65535).optional(), E2E_AGENT_BACKEND_URL: z.url().optional(), + E2E_AGENT_DECISION_MODEL_NAME: z.string().min(1).optional(), + E2E_AGENT_DECISION_MODEL_PROVIDER: z.string().min(1).optional(), + E2E_AGENT_DECISION_MODEL_TYPE: z.string().min(1).optional(), E2E_API_URL: z.url().optional(), E2E_BASE_URL: z.url().optional(), E2E_BROKEN_MODEL_NAME: z.string().min(1).optional(), @@ -137,6 +145,8 @@ export const validateE2eEnv = () => E2E_SHELLCTL_IMAGE: z.string().min(1).optional(), E2E_SHELLCTL_PORT: z.coerce.number().int().positive().max(65535).optional(), E2E_SHELLCTL_URL: z.url().optional(), + E2E_SPEECH_TO_TEXT_MODEL_NAME: z.string().min(1).optional(), + E2E_SPEECH_TO_TEXT_MODEL_PROVIDER: z.string().min(1).optional(), E2E_START_AGENT_BACKEND: booleanString.optional(), E2E_STABLE_MODEL_NAME: z.string().min(1).optional(), E2E_STABLE_MODEL_PROVIDER: z.string().min(1).optional(), diff --git a/e2e/support/test-materials.ts b/e2e/support/test-materials.ts index 891f17f7d22..a6391f52402 100644 --- a/e2e/support/test-materials.ts +++ b/e2e/support/test-materials.ts @@ -10,7 +10,14 @@ export const generatedTestMaterialsDir = fileURLToPath( new URL('../.generated-test-materials', import.meta.url), ) +export const voiceInputTestMaterial = { + fileName: 'voice-input.wav', + recordingDuration: '00:07', +} as const + export const getTestMaterialPath = (fileName: string) => path.join(testMaterialsDir, fileName) +export const getVoiceInputTestMaterialPath = () => + getTestMaterialPath(voiceInputTestMaterial.fileName) export async function getGeneratedTextMaterialPath({ fileName, diff --git a/eslint-suppressions.json b/eslint-suppressions.json index c39a6b33ef9..57ff3f781ee 100644 --- a/eslint-suppressions.json +++ b/eslint-suppressions.json @@ -2308,11 +2308,6 @@ "count": 2 } }, - "web/app/components/base/voice-input/__tests__/index.spec.tsx": { - "ts/no-explicit-any": { - "count": 3 - } - }, "web/app/components/base/voice-input/index.stories.tsx": { "jsx-a11y/click-events-have-key-events": { "count": 2 @@ -2330,19 +2325,6 @@ "count": 1 } }, - "web/app/components/base/voice-input/index.tsx": { - "jsx-a11y/click-events-have-key-events": { - "count": 2 - }, - "jsx-a11y/no-static-element-interactions": { - "count": 2 - } - }, - "web/app/components/base/voice-input/utils.ts": { - "ts/no-explicit-any": { - "count": 4 - } - }, "web/app/components/billing/plan/assets/index.tsx": { "no-barrel-files/no-barrel-files": { "count": 4 @@ -7078,11 +7060,6 @@ "count": 1 } }, - "web/types/lamejs.d.ts": { - "ts/no-explicit-any": { - "count": 3 - } - }, "web/types/pipeline.tsx": { "ts/no-explicit-any": { "count": 3 diff --git a/packages/contracts/generated/api/console/agent/orpc.gen.ts b/packages/contracts/generated/api/console/agent/orpc.gen.ts index 0b6524dbc15..2cc590c433d 100644 --- a/packages/contracts/generated/api/console/agent/orpc.gen.ts +++ b/packages/contracts/generated/api/console/agent/orpc.gen.ts @@ -117,6 +117,9 @@ import { zPostAgentByAgentIdApiEnableResponse, zPostAgentByAgentIdApiKeysPath, zPostAgentByAgentIdApiKeysResponse, + zPostAgentByAgentIdAudioToTextBody, + zPostAgentByAgentIdAudioToTextPath, + zPostAgentByAgentIdAudioToTextResponse, zPostAgentByAgentIdBuildChatFinalizePath, zPostAgentByAgentIdBuildChatFinalizeResponse, zPostAgentByAgentIdBuildDraftApplyPath, @@ -269,9 +272,33 @@ export const apiKeys = { } /** - * Run a build-draft Agent App turn that asks the agent to push config updates + * Transcribe audio using the current Agent debug configuration */ export const post3 = oc + .route({ + description: 'Transcribe audio using the current Agent debug configuration', + inputStructure: 'detailed', + method: 'POST', + operationId: 'postAgentByAgentIdAudioToText', + path: '/agent/{agent_id}/audio-to-text', + tags: ['console'], + }) + .input( + z.object({ + body: zPostAgentByAgentIdAudioToTextBody, + params: zPostAgentByAgentIdAudioToTextPath, + }), + ) + .output(zPostAgentByAgentIdAudioToTextResponse) + +export const audioToText = { + post: post3, +} + +/** + * Run a build-draft Agent App turn that asks the agent to push config updates + */ +export const post4 = oc .route({ description: 'Run a build-draft Agent App turn that asks the agent to push config updates', inputStructure: 'detailed', @@ -284,14 +311,14 @@ export const post3 = oc .output(zPostAgentByAgentIdBuildChatFinalizeResponse) export const finalize = { - post: post3, + post: post4, } export const buildChat = { finalize, } -export const post4 = oc +export const post5 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -303,10 +330,10 @@ export const post4 = oc .output(zPostAgentByAgentIdBuildDraftApplyResponse) export const apply = { - post: post4, + post: post5, } -export const post5 = oc +export const post6 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -323,7 +350,7 @@ export const post5 = oc .output(zPostAgentByAgentIdBuildDraftCheckoutResponse) export const checkout = { - post: post5, + post: post6, } export const delete2 = oc @@ -395,7 +422,7 @@ export const byMessageId = { /** * Stop a running Agent App chat message generation */ -export const post6 = oc +export const post7 = oc .route({ description: 'Stop a running Agent App chat message generation', inputStructure: 'detailed', @@ -408,7 +435,7 @@ export const post6 = oc .output(zPostAgentByAgentIdChatMessagesByTaskIdStopResponse) export const stop = { - post: post6, + post: post7, } export const byTaskId = { @@ -456,7 +483,7 @@ export const candidates = { get: get7, } -export const post7 = oc +export const post8 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -473,7 +500,7 @@ export const post7 = oc .output(zPostAgentByAgentIdComposerValidateResponse) export const validate = { - post: post7, + post: post8, } export const get8 = oc @@ -583,7 +610,7 @@ export const get11 = oc ) .output(zGetAgentByAgentIdConfigFilesResponse) -export const post8 = oc +export const post9 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -603,7 +630,7 @@ export const post8 = oc export const files = { get: get11, - post: post8, + post: post9, byName, } @@ -627,7 +654,7 @@ export const manifest = { get: get12, } -export const post9 = oc +export const post10 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -646,7 +673,7 @@ export const post9 = oc .output(zPostAgentByAgentIdConfigSkillsUploadResponse) export const upload = { - post: post9, + post: post10, } export const get13 = oc @@ -801,7 +828,7 @@ export const config = { skills, } -export const post10 = oc +export const post11 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -814,10 +841,10 @@ export const post10 = oc .output(zPostAgentByAgentIdCopyResponse) export const copy = { - post: post10, + post: post11, } -export const post11 = oc +export const post12 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -829,7 +856,7 @@ export const post11 = oc .output(zPostAgentByAgentIdDebugConversationRefreshResponse) export const refresh = { - post: post11, + post: post12, } export const debugConversation = { @@ -961,7 +988,7 @@ export const drive = { /** * Update an Agent App's presentation features (opener, follow-up, citations, ...) */ -export const post12 = oc +export const post13 = oc .route({ description: "Update an Agent App's presentation features (opener, follow-up, citations, ...)", inputStructure: 'detailed', @@ -976,13 +1003,13 @@ export const post12 = oc .output(zPostAgentByAgentIdFeaturesResponse) export const features = { - post: post12, + post: post13, } /** * Create or update Agent App message feedback */ -export const post13 = oc +export const post14 = oc .route({ description: 'Create or update Agent App message feedback', inputStructure: 'detailed', @@ -997,7 +1024,7 @@ export const post13 = oc .output(zPostAgentByAgentIdFeedbacksResponse) export const feedbacks = { - post: post13, + post: post14, } /** @@ -1020,7 +1047,7 @@ export const delete5 = oc /** * Commit an uploaded file into the Agent App drive under files/ */ -export const post14 = oc +export const post15 = oc .route({ description: 'Commit an uploaded file into the Agent App drive under files/', inputStructure: 'detailed', @@ -1035,7 +1062,7 @@ export const post14 = oc export const files4 = { delete: delete5, - post: post14, + post: post15, } export const get24 = oc @@ -1118,7 +1145,7 @@ export const messages2 = { byMessageId: byMessageId2, } -export const post15 = oc +export const post16 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1130,7 +1157,7 @@ export const post15 = oc .output(zPostAgentByAgentIdPublishResponse) export const publish = { - post: post15, + post: post16, } /** @@ -1179,7 +1206,7 @@ export const read = { /** * Upload one Agent App sandbox file and return a signed download URL */ -export const post16 = oc +export const post17 = oc .route({ description: 'Upload one Agent App sandbox file and return a signed download URL', inputStructure: 'detailed', @@ -1197,7 +1224,7 @@ export const post16 = oc .output(zPostAgentByAgentIdSandboxFilesUploadResponse) export const upload2 = { - post: post16, + post: post17, } /** @@ -1249,7 +1276,7 @@ export const sandbox = { /** * Upload + standardize a Skill into an Agent App drive */ -export const post17 = oc +export const post18 = oc .route({ description: 'Upload + standardize a Skill into an Agent App drive', inputStructure: 'detailed', @@ -1268,13 +1295,13 @@ export const post17 = oc .output(zPostAgentByAgentIdSkillsUploadResponse) export const upload3 = { - post: post17, + post: post18, } /** * Infer CLI tool + ENV suggestions from a standardized Agent App skill */ -export const post18 = oc +export const post19 = oc .route({ description: 'Infer CLI tool + ENV suggestions from a standardized Agent App skill', inputStructure: 'detailed', @@ -1287,7 +1314,7 @@ export const post18 = oc .output(zPostAgentByAgentIdSkillsBySlugInferToolsResponse) export const inferTools = { - post: post18, + post: post19, } /** @@ -1339,7 +1366,7 @@ export const statistics = { summary, } -export const post19 = oc +export const post20 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1351,7 +1378,7 @@ export const post19 = oc .output(zPostAgentByAgentIdVersionsByVersionIdRestoreResponse) export const restore = { - post: post19, + post: post20, } export const get33 = oc @@ -1427,6 +1454,7 @@ export const byAgentId = { apiAccess, apiEnable, apiKeys, + audioToText, buildChat, buildDraft, chatMessages, @@ -1460,7 +1488,7 @@ export const get36 = oc .input(z.object({ query: zGetAgentQuery.optional() })) .output(zGetAgentResponse) -export const post20 = oc +export const post21 = oc .route({ inputStructure: 'detailed', method: 'POST', @@ -1474,7 +1502,7 @@ export const post20 = oc export const agent = { get: get36, - post: post20, + post: post21, inviteOptions, byAgentId, } diff --git a/packages/contracts/generated/api/console/agent/types.gen.ts b/packages/contracts/generated/api/console/agent/types.gen.ts index f093bbdd0e2..d2a20022984 100644 --- a/packages/contracts/generated/api/console/agent/types.gen.ts +++ b/packages/contracts/generated/api/console/agent/types.gen.ts @@ -111,6 +111,10 @@ export type ApiKeyItem = { type: string } +export type AudioTranscriptResponse = { + text: string +} + export type SimpleResultResponse = { result: string } @@ -2160,6 +2164,31 @@ export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponses = { export type DeleteAgentByAgentIdApiKeysByApiKeyIdResponse = DeleteAgentByAgentIdApiKeysByApiKeyIdResponses[keyof DeleteAgentByAgentIdApiKeysByApiKeyIdResponses] +export type PostAgentByAgentIdAudioToTextData = { + body: { + draft_type?: 'debug_build' | 'draft' + file: Blob | File + } + path: { + agent_id: string + } + query?: never + url: '/agent/{agent_id}/audio-to-text' +} + +export type PostAgentByAgentIdAudioToTextErrors = { + 400: unknown + 404: unknown + 413: unknown +} + +export type PostAgentByAgentIdAudioToTextResponses = { + 200: AudioTranscriptResponse +} + +export type PostAgentByAgentIdAudioToTextResponse = + PostAgentByAgentIdAudioToTextResponses[keyof PostAgentByAgentIdAudioToTextResponses] + export type PostAgentByAgentIdBuildChatFinalizeData = { body?: never path: { diff --git a/packages/contracts/generated/api/console/agent/zod.gen.ts b/packages/contracts/generated/api/console/agent/zod.gen.ts index eb3272bb9e6..bedd050d498 100644 --- a/packages/contracts/generated/api/console/agent/zod.gen.ts +++ b/packages/contracts/generated/api/console/agent/zod.gen.ts @@ -47,6 +47,13 @@ export const zApiKeyList = z.object({ data: z.array(zApiKeyItem), }) +/** + * AudioTranscriptResponse + */ +export const zAudioTranscriptResponse = z.object({ + text: z.string(), +}) + /** * SimpleResultResponse */ @@ -2842,6 +2849,20 @@ export const zDeleteAgentByAgentIdApiKeysByApiKeyIdPath = z.object({ */ export const zDeleteAgentByAgentIdApiKeysByApiKeyIdResponse = z.void() +export const zPostAgentByAgentIdAudioToTextBody = z.object({ + draft_type: z.enum(['debug_build', 'draft']).optional().default('draft'), + file: z.custom(), +}) + +export const zPostAgentByAgentIdAudioToTextPath = z.object({ + agent_id: z.uuid(), +}) + +/** + * Audio transcription successful + */ +export const zPostAgentByAgentIdAudioToTextResponse = zAudioTranscriptResponse + export const zPostAgentByAgentIdBuildChatFinalizePath = z.object({ agent_id: z.uuid(), }) diff --git a/packages/contracts/generated/api/console/apps/orpc.gen.ts b/packages/contracts/generated/api/console/apps/orpc.gen.ts index 7acf0623743..7af9c84a97f 100644 --- a/packages/contracts/generated/api/console/apps/orpc.gen.ts +++ b/packages/contracts/generated/api/console/apps/orpc.gen.ts @@ -341,6 +341,7 @@ import { zPostAppsByAppIdApiEnableBody, zPostAppsByAppIdApiEnablePath, zPostAppsByAppIdApiEnableResponse, + zPostAppsByAppIdAudioToTextBody, zPostAppsByAppIdAudioToTextPath, zPostAppsByAppIdAudioToTextResponse, zPostAppsByAppIdChatMessagesByTaskIdStopPath, @@ -1768,7 +1769,9 @@ export const post20 = oc path: '/apps/{app_id}/audio-to-text', tags: ['console'], }) - .input(z.object({ params: zPostAppsByAppIdAudioToTextPath })) + .input( + z.object({ body: zPostAppsByAppIdAudioToTextBody, params: zPostAppsByAppIdAudioToTextPath }), + ) .output(zPostAppsByAppIdAudioToTextResponse) export const audioToText = { diff --git a/packages/contracts/generated/api/console/apps/types.gen.ts b/packages/contracts/generated/api/console/apps/types.gen.ts index c5175e7f3f2..6dd6568c767 100644 --- a/packages/contracts/generated/api/console/apps/types.gen.ts +++ b/packages/contracts/generated/api/console/apps/types.gen.ts @@ -4357,7 +4357,9 @@ export type PostAppsByAppIdApiEnableResponse = PostAppsByAppIdApiEnableResponses[keyof PostAppsByAppIdApiEnableResponses] export type PostAppsByAppIdAudioToTextData = { - body?: never + body: { + file: Blob | File + } path: { app_id: string } diff --git a/packages/contracts/generated/api/console/apps/zod.gen.ts b/packages/contracts/generated/api/console/apps/zod.gen.ts index 77ff787ed76..e705cfa9a6f 100644 --- a/packages/contracts/generated/api/console/apps/zod.gen.ts +++ b/packages/contracts/generated/api/console/apps/zod.gen.ts @@ -5094,6 +5094,10 @@ export const zPostAppsByAppIdApiEnablePath = z.object({ */ export const zPostAppsByAppIdApiEnableResponse = zAppDetail +export const zPostAppsByAppIdAudioToTextBody = z.object({ + file: z.custom(), +}) + export const zPostAppsByAppIdAudioToTextPath = z.object({ app_id: z.uuid(), }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 837e613c2d9..2395c36d72a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -84,6 +84,9 @@ catalogs: '@mdx-js/rollup': specifier: 3.1.1 version: 3.1.1 + '@mediabunny/mp3-encoder': + specifier: 1.50.7 + version: 1.50.7 '@monaco-editor/react': specifier: 4.7.0 version: 4.7.0 @@ -408,9 +411,6 @@ catalogs: jotai-tanstack-query: specifier: 0.11.0 version: 0.11.0 - js-audio-recorder: - specifier: 1.0.7 - version: 1.0.7 js-cookie: specifier: 3.0.8 version: 3.0.8 @@ -429,9 +429,6 @@ catalogs: ky: specifier: 2.0.2 version: 2.0.2 - lamejs: - specifier: 1.2.1 - version: 1.2.1 lexical: specifier: 0.46.0 version: 0.46.0 @@ -441,6 +438,9 @@ catalogs: loro-crdt: specifier: 1.13.6 version: 1.13.6 + mediabunny: + specifier: 1.50.7 + version: 1.50.7 mermaid: specifier: 11.16.0 version: 11.16.0 @@ -1155,6 +1155,9 @@ importers: '@lexical/utils': specifier: 'catalog:' version: 0.46.0(@typescript/typescript6@6.0.2) + '@mediabunny/mp3-encoder': + specifier: 'catalog:' + version: 1.50.7(mediabunny@1.50.7) '@monaco-editor/react': specifier: 'catalog:' version: 4.7.0(react-dom@19.2.7(react@19.2.7))(react@19.2.7) @@ -1299,9 +1302,6 @@ importers: jotai-tanstack-query: specifier: 'catalog:' version: 0.11.0(@tanstack/query-core@5.101.2)(@tanstack/react-query@5.101.2(react@19.2.7))(jotai@2.20.1(@babel/core@7.29.7)(@babel/template@7.29.7)(@types/react@19.2.17)(react@19.2.7))(react@19.2.7) - js-audio-recorder: - specifier: 'catalog:' - version: 1.0.7 js-cookie: specifier: 'catalog:' version: 3.0.8 @@ -1317,15 +1317,15 @@ importers: ky: specifier: 'catalog:' version: 2.0.2 - lamejs: - specifier: 'catalog:' - version: 1.2.1 lexical: specifier: 'catalog:' version: 0.46.0(@typescript/typescript6@6.0.2) loro-crdt: specifier: 'catalog:' version: 1.13.6 + mediabunny: + specifier: 'catalog:' + version: 1.50.7 mermaid: specifier: 'catalog:' version: 11.16.0 @@ -1752,7 +1752,6 @@ packages: '@antfu/eslint-config@9.1.0': resolution: {integrity: sha512-Vtjt+W/6/bV9hgQG8AiWG66OpAeLKmpo5bZaAmUdRvZsMyirVvIJeHnbM4XUJf4urIuLvK0gGSvXD74PI3ZnGQ==} - hasBin: true peerDependencies: '@angular-eslint/eslint-plugin': ^21.1.0 '@angular-eslint/eslint-plugin-template': ^21.1.0 @@ -1870,7 +1869,6 @@ packages: '@babel/parser@7.29.7': resolution: {integrity: sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==} engines: {node: '>=6.0.0'} - hasBin: true '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} @@ -1973,11 +1971,9 @@ packages: '@cucumber/cucumber@13.0.0': resolution: {integrity: sha512-lUD/IxGZXbfSP+pd7zaYvJIJXBaTZ36CmQxcLDYkilGH8y4ycaOnXe7ll0QFukYFh9/1x6S7pw6lYspJg6g7jw==} engines: {node: 22 || 24 || >=26} - hasBin: true '@cucumber/gherkin-streams@6.0.0': resolution: {integrity: sha512-HLSHMmdDH0vCr7vsVEURcDA4WwnRLdjkhqr6a4HQ3i4RFK1wiDGPjBGVdGJLyuXuRdJpJbFc6QxHvT8pU4t6jw==} - hasBin: true peerDependencies: '@cucumber/gherkin': '>=22.0.0' '@cucumber/message-streams': '>=4.0.0' @@ -1985,7 +1981,6 @@ packages: '@cucumber/gherkin-utils@11.0.0': resolution: {integrity: sha512-LJ+s4+TepHTgdKWDR4zbPyT7rQjmYIcukTwNbwNwgqr6i8Gjcmzf6NmtbYDA19m1ZFg6kWbFsmHnj37ZuX+kZA==} - hasBin: true '@cucumber/gherkin@38.0.0': resolution: {integrity: sha512-duEXK+KDfQUzu3vsSzXjkxQ2tirF5PRsc1Xrts6THKHJO6mjw4RjM8RV+vliuDasmhhrmdLcOcM7d9nurNTJKw==} @@ -2416,7 +2411,6 @@ packages: '@hey-api/openapi-ts@0.98.2': resolution: {integrity: sha512-2nVJXH8tpFPGTBOhxyjEd1Jw0hsRqJqeTQW3kltAjVdSU4YWxeu97x5sgNOmsbsfeg6Dqz7Wfzs26walBOuswA==} engines: {node: '>=22.18.0'} - hasBin: true peerDependencies: typescript: '>=5.5.3 || >=6.0.0 || 6.0.1-rc' @@ -3027,6 +3021,11 @@ packages: peerDependencies: rollup: 4.62.2 + '@mediabunny/mp3-encoder@1.50.7': + resolution: {integrity: sha512-nYhphp1q9xSBRL4rhMg2yw63Krw2lAy4i7Lv6WjqWYabJayOd5ckHV7jB6Rn8A0SeqdUl5fwwia5uRJGcujNrg==} + peerDependencies: + mediabunny: ^1.0.0 + '@mermaid-js/parser@1.2.0': resolution: {integrity: sha512-oYPyv8A4As1yH5Bx+04iQEQxXuIQDe0GKCNSRgao6z8AM9jixXIfP0vsppRLvGf+nKIOb9/LdpWA4YuJiVvESA==} @@ -4047,7 +4046,6 @@ packages: '@playwright/test@1.61.1': resolution: {integrity: sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==} engines: {node: '>=18'} - hasBin: true '@polka/url@1.0.0-next.29': resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==} @@ -4364,7 +4362,6 @@ packages: '@shuding/opentype.js@1.4.0-beta.0': resolution: {integrity: sha512-3NgmNyH3l/Hv6EvsWJbsvpcpUba6R8IREQ83nH83cyakCw7uM1arZKNfHwv1Wz6jgqrF/j4x5ELvR6PnK9nTcA==} engines: {node: '>= 8.0.0'} - hasBin: true '@sindresorhus/base62@1.0.0': resolution: {integrity: sha512-TeheYy0ILzBEI/CO55CP6zJCSdSWeRtGnHy8U8dWSUH4I68iqTsy7HkMktR4xakThc9jotkPQUXT4ITdbV7cHA==} @@ -4679,7 +4676,6 @@ packages: '@tanstack/devtools-event-client@0.4.4': resolution: {integrity: sha512-6T5Yop/793YI+H+5J8Hsyj4kCih9sl4t3ElLgKioW5hk3ocn+ZdSJ94tT7vL7uabxSugWYBZlOTMPzEw2puvQw==} engines: {node: '>=18'} - hasBin: true '@tanstack/eslint-plugin-query@5.101.2': resolution: {integrity: sha512-cPE99s3XZwlObfn8lCezT4j4JLj2CVzpIEywx0H4hzfPsX/o9QhdwaOwcDXxrQAqx2ds7TbvTinxhB8B/ywb6w==} @@ -4779,7 +4775,6 @@ packages: '@tsslint/cli@3.1.4': resolution: {integrity: sha512-svoLfFkoWmdsDrIRLllFnrxydfMjKKZ1UBjv7Sua1KjFkx6VaJ88+YGYqNiTbB/dDcU10qnSMXRavNTJ0fjBkQ==} engines: {node: '>=22.6.0'} - hasBin: true peerDependencies: typescript: '*' @@ -4791,7 +4786,6 @@ packages: '@tsslint/config@3.1.4': resolution: {integrity: sha512-6VcUimc170M1v3b0vmOhRW7NI/b7DqXB5Wpo+1wCNLprDTN1HwjsbfdFNm/nsd0jXWChNr/cJhmsnVp4xHDCKw==} engines: {node: '>=22.6.0'} - hasBin: true peerDependencies: '@tsslint/compat-eslint': ^3.0.0 tsl: ^1.0.28 @@ -4934,6 +4928,12 @@ packages: '@types/doctrine@0.0.9': resolution: {integrity: sha512-eOIHzCUSH7SMfonMG1LsC2f8vxBFtho6NGBznK41R84YzPuvSBzrhEps33IsQiOW9+VL6NQ9DbjQJznk/S4uRA==} + '@types/dom-mediacapture-transform@0.1.11': + resolution: {integrity: sha512-Y2p+nGf1bF2XMttBnsVPHUWzRRZzqUoJAKmiP10b5umnO6DDrWI0BrGDJy1pOHoOULVmGSfFNkQrAlC5dcj6nQ==} + + '@types/dom-webcodecs@0.1.13': + resolution: {integrity: sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==} + '@types/esrecurse@4.3.1': resolution: {integrity: sha512-xJBAbDifo5hpffDBuHl0Y8ywswbiAp/Wi7Y/GtAgSlZyIABppyurxVueOPE8LUQOxdlgi6Zqce7uoEpqNTeiUw==} @@ -5204,7 +5204,6 @@ packages: '@typescript/typescript6@6.0.2': resolution: {integrity: sha512-mbCddXd+jm7hfx7w2YU64/Av4/NqqeG3GoRZgxPcgoTxYjhrcfJRw9ULch71SS4G+Q3bOXFhRvPqjguN0Hyp5w==} - hasBin: true '@ungap/structured-clone@1.3.2': resolution: {integrity: sha512-5jsZFwgR5rTdKwidH9Qmat75RKwqfpKlWWB1frDkljN127mwqBu8K0PYo7/hFpF03IEJpfVPpCQDY/eDx3iHvA==} @@ -5496,7 +5495,6 @@ packages: acorn@8.17.0: resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} engines: {node: '>=0.4.0'} - hasBin: true agentation@3.0.2: resolution: {integrity: sha512-iGzBxFVTuZEIKzLY6AExSLAQH6i6SwxV4pAu7v7m3X6bInZ7qlZXAwrEqyc4+EfP4gM7z2RXBF6SF4DeH0f2lA==} @@ -5603,7 +5601,6 @@ packages: astring@1.9.0: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} - hasBin: true async-function@1.0.0: resolution: {integrity: sha512-hsU18Ae8CDTR6Kgu9DYf0EbCr/a5iGL0rytQDobUcdpYOKokk8LEjVphnXkDkgpi0wYVsqrXuP0bZxJaTqdgoA==} @@ -5648,7 +5645,6 @@ packages: baseline-browser-mapping@2.10.40: resolution: {integrity: sha512-BSSLZ9/Cjjv7Gtj5B68ZzXcXUg8iOf3fme+FCuh8rC/Go+Kmh8cox7M3A8dolou16s64QjLPOSdngh7GxXvkSw==} engines: {node: '>=6.0.0'} - hasBin: true birecord@0.1.1: resolution: {integrity: sha512-VUpsf/qykW0heRlC8LooCq28Kxn3mAqKohhDG/49rrsQ1dT1CXyj/pgXS+5BSRzFTR/3DyIBOqQOrGyZOh71Aw==} @@ -5676,7 +5672,6 @@ packages: browserslist@4.28.4: resolution: {integrity: sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} - hasBin: true buffer-crc32@0.2.13: resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==} @@ -5815,7 +5810,6 @@ packages: chromatic@16.10.0: resolution: {integrity: sha512-nFsztmnu7rFiGafUJgXvLUNpqmRylz92eNvzBoJNTKKQj4EQUyxznwnfpf1dTs7hXtWD8JwcH92jADydaHA1sw==} - hasBin: true peerDependencies: '@chromatic-com/cypress': ^0.*.* || ^1.0.0 '@chromatic-com/playwright': ^0.*.* || ^1.0.0 @@ -5888,7 +5882,6 @@ packages: color-support@1.1.3: resolution: {integrity: sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==} - hasBin: true comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -5930,7 +5923,6 @@ packages: concurrently@10.0.3: resolution: {integrity: sha512-hc3LH4UaKWd/bbyDK/IGVa4RB6PtQ3CUYwtrkzqHn+wIG3Hr5fhpRlk0L/gCa8ZE1L/Ufj50Zho69cI5w8SQBA==} engines: {node: '>=22'} - hasBin: true confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -5999,7 +5991,6 @@ packages: cssesc@3.0.0: resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==} engines: {node: '>=4'} - hasBin: true cssfontparser@1.2.1: resolution: {integrity: sha512-6tun4LoZnj7VN6YeegOVb67KBX/7JJsqvj+pv3ZA7F878/eN33AbGa5b/S/wXxS/tcp8nc40xRUrsPlxIyNUPg==} @@ -6067,7 +6058,6 @@ packages: d3-dsv@3.0.1: resolution: {integrity: sha512-UG6OvdI5afDIFP9w4G0mNq50dSOsXHJaRE8arAS5o9ApWnIElp8GZw1Dun8vP8OyHOZ/QJUKUJwxiiCCnUwm+Q==} engines: {node: '>=12'} - hasBin: true d3-ease@3.0.1: resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} @@ -6461,7 +6451,6 @@ packages: esbuild@0.28.1: resolution: {integrity: sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==} engines: {node: '>=18'} - hasBin: true escalade@3.2.0: resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} @@ -6743,7 +6732,6 @@ packages: eslint@10.6.0: resolution: {integrity: sha512-6lVbcqSodALYo+4ELD0heG6lFiFxnLMuLkiMi2qV8LMp54N8tE8FT1GMH+ev4Ti00nFjNze2+Su6DsV5OQW3Dg==} engines: {node: ^20.19.0 || ^22.13.0 || >=24} - hasBin: true peerDependencies: jiti: '*' peerDependenciesMeta: @@ -6761,7 +6749,6 @@ packages: esprima@4.0.1: resolution: {integrity: sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==} engines: {node: '>=4'} - hasBin: true esquery@1.7.0: resolution: {integrity: sha512-Ap6G0WQwcU/LHsvLwON1fAQX9Zp0A2Y6Y/cJBl9r/JbW90Zyg4/zbG6zzKa2OTALELarYHmKu0GhpM5EO+7T0g==} @@ -6827,7 +6814,6 @@ packages: extract-zip@2.0.1: resolution: {integrity: sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==} engines: {node: '>= 10.17.0'} - hasBin: true fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -6917,7 +6903,6 @@ packages: formatly@0.3.0: resolution: {integrity: sha512-9XNj/o4wrRFyhSMJOvsuyMwy8aUfBaZ1VrqHVfohyXf0Sw0e+yfKG+xZaY3arGCOMdwFsqObtzVOc1gU9KiT9w==} engines: {node: '>=18.3.0'} - hasBin: true foxact@0.3.8: resolution: {integrity: sha512-lFakEG8xm6A6wMH4ycWhXvwUybq4ATnMxJfBD4hj6hm9CKo72q2M/YaiEemoWGh9sUjQqX+pPXfq8S/jxedWzA==} @@ -7012,7 +6997,6 @@ packages: giget@3.3.0: resolution: {integrity: sha512-gzi2D96p+AMfDcmJHGDj3KJ9NRiwvlFAU5yfa3ROwWZmFUjX4P43x3BcyRaOMMLto1vUo7C+86+MFhYTl6Ryiw==} - hasBin: true github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} @@ -7061,7 +7045,6 @@ packages: h3@2.0.1-rc.22: resolution: {integrity: sha512-Esv0DMIuPkCTSWCA0vO73vcTqwzH1wjSrAO1TXNu/K3up1sZHa9EKMapbmxCDYBeymC3fVTk4qxp7ogQWQ+KgA==} engines: {node: '>=20.11.1'} - hasBin: true peerDependencies: crossws: ^0.4.1 peerDependenciesMeta: @@ -7222,7 +7205,6 @@ packages: image-size@2.0.2: resolution: {integrity: sha512-IRqXKlaXwgSMAMtpNzZa1ZAe8m+Sa1770Dhk8VkSsP9LS+iHD62Zd8FQKs8fbPiagBE7BzoFX23cxFnwshpV6w==} engines: {node: '>=16.x'} - hasBin: true immer@11.1.11: resolution: {integrity: sha512-qzXuyXAkPySAGYkfsAwodDPWT8Zm7/Uo5BNt4BjhMhG5WlWyZZ4wQqnWwdS8kjlQ1Cwu6gjw3A6+0gTQwlyYtw==} @@ -7322,7 +7304,6 @@ packages: is-docker@3.0.0: resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - hasBin: true is-document.all@1.0.0: resolution: {integrity: sha512-+XSoyS05OdBbhFuELhgTCpFNHkpBOJqtsZfUFFpe5QTw+9Sjbh8zitxhQkYAo6wV7e1Vb8cAPvpCk9jGam/82g==} @@ -7354,7 +7335,6 @@ packages: is-inside-container@1.0.0: resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} engines: {node: '>=14.16'} - hasBin: true is-installed-globally@1.0.0: resolution: {integrity: sha512-K55T22lfpQ63N4KEN57jZUAaAYqYHEe8veb/TycJRk9DdSCLLcovXz/mL6mOnhQaZsQGwPhuFopdQIlqGSEjiQ==} @@ -7456,7 +7436,6 @@ packages: jiti@2.7.0: resolution: {integrity: sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ==} - hasBin: true jotai-effect@2.3.1: resolution: {integrity: sha512-FBBVXDM2podbbxJsZ19+uDv45LxeXoVA5yh6CfkQ35AVCuRHj7Lanlcjiea3b67Q7+/MMGXWf8+GeB73JcbeMg==} @@ -7502,9 +7481,6 @@ packages: react: optional: true - js-audio-recorder@1.0.7: - resolution: {integrity: sha512-JiDODCElVHGrFyjGYwYyNi7zCbKk9va9C77w+zCPMmi4C6ix7zsX2h3ddHugmo4dOTOTCym9++b/wVW9nC0IaA==} - js-base64@3.8.0: resolution: {integrity: sha512-65kvbemyZhj+ExQt1PEFyBEjL5vAHysu1lJdW1AwhhChkO8ZBPizYk/m9GVrpbS2Je1hF+UYZ+6KywqtZV8mHw==} @@ -7522,11 +7498,9 @@ packages: js-yaml@4.3.0: resolution: {integrity: sha512-1td788aAnnZ5qs7V2QIRl1owjtYpbKt749Y3xauqQgwIIGF/xXWz1wMTEBx5O3LK3lXLVuqXPdPxj2BoFHaW9Q==} - hasBin: true js-yaml@5.2.1: resolution: {integrity: sha512-zfLtNfQqxVqq3uaTqSkh4x4hZw3KHobGUA0fJUj4wawW8bsQLTVqpHdXSIzidh7o+4lEW36tANuAGdaFx6Zgnw==} - hasBin: true jsdoc-type-pratt-parser@7.1.1: resolution: {integrity: sha512-/2uqY7x6bsrpi3i9LVU6J89352C0rpMk0as8trXxCtvd4kPk1ke/Eyif6wqfSLvoNJqcDG9Vk4UsXgygzCt2xA==} @@ -7539,7 +7513,6 @@ packages: jsesc@3.1.0: resolution: {integrity: sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==} engines: {node: '>=6'} - hasBin: true json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} @@ -7553,7 +7526,6 @@ packages: json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} - hasBin: true jsonc-eslint-parser@3.1.0: resolution: {integrity: sha512-75EA7EWZExL/j+MDKQrRbdzcRI2HOkRlmUw8fZJc1ioqFEOvBsq7Rt+A6yCxOt9w/TYNpkt52gC6nm/g5tFIng==} @@ -7571,11 +7543,9 @@ packages: katex@0.16.47: resolution: {integrity: sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==} - hasBin: true katex@0.17.0: resolution: {integrity: sha512-Vdw0ATsQ9V+LuegM/BTwQqV/6cTl5lbGcIrU+BCgLxyf6bo38ybOr372tuSIxir3CN720flu1meYR6XzNMwQnw==} - hasBin: true keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -7586,7 +7556,6 @@ packages: knip@6.25.0: resolution: {integrity: sha512-Q3n41VjOOB/aqsbxb8kallAcFKrUz3b2S5fD5pTODljVpP01t+rvAgy2x3j0Cq8yEpRRHNdar1vHuqFfGuIakQ==} engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true knuth-shuffle-seeded@1.0.6: resolution: {integrity: sha512-9pFH0SplrfyKyojCLxZfMcvkhf5hH0d+UwR9nTVJ/DDQJGuzcXjTwB7TP7sDfehSudlGGaOLblmEWqv04ERVWg==} @@ -7598,9 +7567,6 @@ packages: resolution: {integrity: sha512-/GmXpo9F9W+f8n4Ivr2iH+7h7wL7jLbLKWkMlpflcCRb6kGjBfTlASEXaZ9qUgNTn4VgS0P2pwxxzQ4EM6Ulgg==} engines: {node: '>=22'} - lamejs@1.2.1: - resolution: {integrity: sha512-s7bxvjvYthw6oPLCm5pFxvA84wUROODB8jEO2+CE1adhKgrIvVOlmMgY8zyugxGrvRaDHNJanOiS21/emty6dQ==} - language-subtag-registry@0.3.23: resolution: {integrity: sha512-0K65Lea881pHotoGEa5gDlMxt3pctLi2RplBb7Ezh4rRdLEOtgi7n4EwK9lamnUCkKBqaeKRVebTq6BAxSkpXQ==} @@ -7747,7 +7713,6 @@ packages: loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} - hasBin: true loro-crdt@1.13.6: resolution: {integrity: sha512-OsnKW64J+1XdCMafAqR+aASn/wQEgDzwcDwu6dDOVSxxbLPFgnU6LX8ja+hd5PnEcT+kuz9eXSagvlYXsRbQIw==} @@ -7768,7 +7733,6 @@ packages: lz-string@1.5.0: resolution: {integrity: sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ==} - hasBin: true magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} @@ -7790,12 +7754,10 @@ packages: marked@16.4.2: resolution: {integrity: sha512-TI3V8YYWvkVf3KJe1dRkpnjs68JUPyEa5vjKrp1XEEJUAOaQc+Qj+L1qWbPd0SJuAdQkFU0h73sXXqwDYxsiDA==} engines: {node: '>= 20'} - hasBin: true marked@17.0.6: resolution: {integrity: sha512-gB0gkNafnonOw0obSTEGZTT86IuhILt2Wfx0mWH/1Au83kybTayroZ/V6nS25mN7u8ASy+5fMhgB3XPNrOZdmA==} engines: {node: '>= 20'} - hasBin: true math-intrinsics@1.1.0: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} @@ -7870,6 +7832,9 @@ packages: mdn-data@2.28.1: resolution: {integrity: sha512-U9w+PzSZ00Z5m9rZ5ARVFL5xOfuCHdKYi/1RRwDCJsboFgJDNT3zT6PIPD7mZQYaQLhsZM3GfDRgSMRHhSmVng==} + mediabunny@1.50.7: + resolution: {integrity: sha512-mv+FXkQNOobx4b43GJV1t0n8cx26d8nZpBMtCTaNKk6xzVpNPrwrbDq+dggm7Zpjvtx2ycgtYciFcySUsO5QGA==} + merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -7998,12 +7963,10 @@ packages: mime@3.0.0: resolution: {integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==} engines: {node: '>=10.0.0'} - hasBin: true mime@4.1.0: resolution: {integrity: sha512-X5ju04+cAzsojXKes0B/S4tcYtFAJ6tTMuSPBEn9CPGlrWr8Fiw7qYeLT0XyH80HSoAoqWCaz+MWKh22P7G1cw==} engines: {node: '>=16'} - hasBin: true mimic-function@5.0.1: resolution: {integrity: sha512-VP79XUPxV2CigYP3jWwAUFSku2aKqBH7uTAapFWCBqutsbmDo96KY5o8uh6U+/YSIn5OxJnXp73beVkpqMIGhA==} @@ -8044,7 +8007,6 @@ packages: mkdirp@3.0.1: resolution: {integrity: sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==} engines: {node: '>=10'} - hasBin: true mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} @@ -8088,7 +8050,6 @@ packages: nanoid@3.3.15: resolution: {integrity: sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true napi-build-utils@2.0.0: resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} @@ -8116,7 +8077,6 @@ packages: next@16.2.10: resolution: {integrity: sha512-2som5AVXb3kE6Yjine3/mNbBayYF58eguBWIVVUdr1y/L426xyVEgYxgBG+1QC34P2x5E+tcDup6XkuOAX3dCA==} engines: {node: '>=20.9.0'} - hasBin: true peerDependencies: '@opentelemetry/api': ^1.1.0 '@playwright/test': ^1.51.1 @@ -8411,7 +8371,6 @@ packages: oxfmt@0.57.0: resolution: {integrity: sha512-ZB7Bi+rGDSqmVIo9jwcLyFgjxXvQhDdU+jx+ZrVy6VRiVXK2+CHc4hO3J4dUQjHe7V0ymHB+MDuv5z+NhK07HA==} engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true peerDependencies: svelte: ^5.0.0 vite-plus: '*' @@ -8423,12 +8382,10 @@ packages: oxlint-tsgolint@0.24.0: resolution: {integrity: sha512-giCk5sEvG02d5tzPmFMX3hem8ndzEEu1xvGYS5OwNfO2WGl6ZVxt5LjE0yiMDoz94INI7XkXwgFAQiydPvVHDw==} - hasBin: true oxlint@1.72.0: resolution: {integrity: sha512-1rhdZIP/EvoI91ABIwNU5Q8+bWf8mjrS5UzIOZld4d4bXxJvtlUhlQvaoTogIGin/qdErMOrwaIJvCSIAKTLhA==} engines: {node: ^20.19.0 || >=22.12.0} - hasBin: true peerDependencies: oxlint-tsgolint: '>=0.22.1' vite-plus: '*' @@ -8556,12 +8513,10 @@ packages: playwright-core@1.61.1: resolution: {integrity: sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==} engines: {node: '>=18'} - hasBin: true playwright@1.61.1: resolution: {integrity: sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==} engines: {node: '>=18'} - hasBin: true pluralize@8.0.0: resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} @@ -8611,7 +8566,6 @@ packages: resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} engines: {node: '>=10'} deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. - hasBin: true prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -8661,7 +8615,6 @@ packages: rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true re-resizable@6.11.2: resolution: {integrity: sha512-2xI2P3OHs5qw7K0Ud1aLILK6MQxW50TcO+DetD9eIV58j84TqYeHoZcL9H4GXFXXIh7afhH8mv5iUCXII7OW7A==} @@ -8871,7 +8824,6 @@ packages: regexp-tree@0.1.27: resolution: {integrity: sha512-iETxpjK6YoRWJG5o6hXLwvjYAoW+FEZn9os0PD/b6AP6xQwsa/Y7lCVgIixBbUPMfhu+i2LtdeAqVTgGlQarfA==} - hasBin: true regexp.prototype.flags@1.5.4: resolution: {integrity: sha512-dYqgNSZbDwkaJ2ceRd9ojCGjBq+mOm9LmtXnAnEGyHhN/5R7iDW2TRw3h+o/jCFxus3P2LfWIIiwowAjANm7IA==} @@ -8879,7 +8831,6 @@ packages: regjsparser@0.13.2: resolution: {integrity: sha512-NgRBy2Nx/bE+9F27nVHnqcN5HjyLmecqsqx2PJHu3/IEtADD4WuxuXIVExD5PoSDFVrl78dOonfcOe5O+5nbzQ==} - hasBin: true rehype-harden@1.1.8: resolution: {integrity: sha512-Qn7vR1xrf6fZCrkm9TDWi/AB4ylrHy+jqsNm1EHOAmbARYA6gsnVJBq/sdBh6kmT4NEZxH5vgIjrscefJAOXcw==} @@ -8943,7 +8894,6 @@ packages: resolve@1.22.12: resolution: {integrity: sha512-TyeJ1zif53BPfHootBGwPRYT1RUt6oGWsaQr8UyZW/eAm9bKoijtvruSDEmZHm92CwS9nj7/fWttqPCgzep8CA==} engines: {node: '>= 0.4'} - hasBin: true restore-cursor@5.1.0: resolution: {integrity: sha512-oMA2dcrw6u0YfxJQXm342bFKX/E4sG9rbTzO9ptUcR/e8A33cHuvStiYOwH7fszkZlZ1z/ta9AAoPk2F4qIOHA==} @@ -9017,22 +8967,18 @@ packages: semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} - hasBin: true semver@7.8.1: resolution: {integrity: sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg==} engines: {node: '>=10'} - hasBin: true semver@7.8.2: resolution: {integrity: sha512-c8jsqUZm3omBOI66G90z1Dyw5z622G8oLG+omfsHBJf3CWQTlOcwOjvOG6wtiNfW6anKm/eA39LMwMtMez2TiQ==} engines: {node: '>=10'} - hasBin: true semver@7.8.5: resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} engines: {node: '>=10'} - hasBin: true server-only@0.0.1: resolution: {integrity: sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA==} @@ -9155,7 +9101,6 @@ packages: srvx@0.11.17: resolution: {integrity: sha512-43yM4luKfCJamyCMhrUeHUPOrf8TdZe7kN8s5zayZCH5OeprYqi49Aso5ZvHXR4aB+DHaRNO/diNFgZSMNG8Xw==} engines: {node: '>=20.16.0'} - hasBin: true stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} @@ -9183,7 +9128,6 @@ packages: storybook@10.4.6: resolution: {integrity: sha512-6wkA6LxfDSSilloITsrFOJfsnw0mDUP2h8Ls+lRt8oRsudtz2RWFhLv+Toiwg6NW7hUpdTDc2hzR7DztJid6+A==} - hasBin: true peerDependencies: '@types/react': ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 prettier: ^2 || ^3 @@ -9302,7 +9246,6 @@ packages: svgo@3.3.3: resolution: {integrity: sha512-+wn7I4p7YgJhHs38k2TNjy1vCfPIfLIJWR5MnCStsN8WuuTcBnRKcMHQLMM2ijxGZmDoZwNv8ipl5aTTen62ng==} engines: {node: '>=14.0.0'} - hasBin: true synckit@0.11.13: resolution: {integrity: sha512-eNRKgb3z66Yp3D2CixVujOUvXLFUTij/zVnV8KRyvFdQwpz7I5DS8UfRkTeLzb64u+dkzDSdelE24izu+zSSUg==} @@ -9389,7 +9332,6 @@ packages: tldts@7.4.7: resolution: {integrity: sha512-56L0/9HELHSsG1bFCzay8UoLxzRL7kpFf7Wl5q/kSYwiSJGACvro61xnKzPNM+SadxllzdtXsKDSXE7HPeqIAw==} - hasBin: true to-regex-range@5.0.1: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} @@ -9412,7 +9354,6 @@ packages: tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} - hasBin: true trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -9440,7 +9381,6 @@ packages: resolution: {integrity: sha512-ks6Vjr/jEw0P1gmOVwutM3B7fWxoWBL2KRDb1JfqGVawBmO5UsvmWOQFGHBPl5yxYz4eERr19E6L7NMv+Fej4w==} engines: {node: ^18 || >=20} deprecated: unmaintained - hasBin: true peerDependencies: typescript: ^5.0.0 peerDependenciesMeta: @@ -9467,7 +9407,6 @@ packages: tsx@4.23.0: resolution: {integrity: sha512-eUdUIaCr963q2h5u3+QwvYp0+eqPvn+egeqZUm0hwERCqqx1E3kK5ehbGCvqSE5MQAULr67ww0cA3jKc3YkM1w==} engines: {node: '>=18.0.0'} - hasBin: true tunnel-agent@0.6.0: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} @@ -9515,7 +9454,6 @@ packages: typescript@7.0.2: resolution: {integrity: sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==} engines: {node: '>=16.20.0'} - hasBin: true ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} @@ -9523,7 +9461,6 @@ packages: uglify-js@3.19.3: resolution: {integrity: sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==} engines: {node: '>=0.8.0'} - hasBin: true unbash@4.0.2: resolution: {integrity: sha512-8gwNZ29+0/3zmXw7ToIHZtg6wK37xnniRUdBt7B27xZxaxfgR5tGMaGHT0t0dLtBV9fXE7zurh0s6Z1DHVjfWg==} @@ -9627,7 +9564,6 @@ packages: update-browserslist-db@1.2.3: resolution: {integrity: sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==} - hasBin: true peerDependencies: browserslist: '>= 4.21.0' @@ -9687,9 +9623,6 @@ packages: '@types/react': optional: true - use-strict@1.0.1: - resolution: {integrity: sha512-IeiWvvEXfW5ltKVMkxq6FvNf2LojMKvB2OCeja6+ct24S1XOmQw2dGr2JyndwACWAGJva9B7yPHwAmeA9QCqAQ==} - use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -9703,7 +9636,6 @@ packages: uuid@14.0.1: resolution: {integrity: sha512-6ZxzVpzDXDa3bJWaHilVayA+BH/1zmxCJoVgvmqJnid/gPoKHxUrS/aC/T6LGQtNHT+XHG9fXPJB4d+IrU30Ew==} - hasBin: true valibot@1.4.2: resolution: {integrity: sha512-gjdCvJ6d3RyHAneqxMYMW9QMCwYMb3jpOO0IyHZV1bnRHFBHrX3VkIILt5XYR0WhwHiH7Mty8ovuPZ/O3gamrg==} @@ -9728,7 +9660,6 @@ packages: vinext@1.0.0-beta.0: resolution: {integrity: sha512-O0YkPs435h9kSYRQPjgPvtewpiKmUUJa6eBOoP3eQtgg0wR5L6w8HezJGTq0T/mHL1EaGCDpYbED3TFhriFklg==} engines: {node: '>=22'} - hasBin: true peerDependencies: '@mdx-js/rollup': ^3.0.0 '@vitejs/plugin-react': ^5.1.4 || ^6.0.0 @@ -9771,7 +9702,6 @@ packages: vite-plus@0.2.4: resolution: {integrity: sha512-gaBBjOXIq9lLRU44oAYdIr99p+JBLX1kxs+l/6LqGgSXwcVKAdDa1boSrOTELqYCkQQ0fpppXUGWi9o6JDT5zw==} engines: {node: ^20.19.0 || ^22.18.0 || >=24.11.0} - hasBin: true peerDependencies: '@vitest/browser-playwright': 4.1.10 '@vitest/browser-webdriverio': 4.1.10 @@ -9819,7 +9749,6 @@ packages: vitest@4.1.10: resolution: {integrity: sha512-R9jUTe5S4Qb0HCd4TNqpC7oGcrMssMRGXLW80ubjWsW9VH5GF8y1Y0SFLY9AbqSk6nt0PnOx4H4WNJYZ13GUPw==} engines: {node: ^20.0.0 || ^22.0.0 || >=24.0.0} - hasBin: true peerDependencies: '@edge-runtime/vm': '*' '@opentelemetry/api': ^1.9.0 @@ -9922,12 +9851,10 @@ packages: which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} - hasBin: true why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} - hasBin: true word-wrap@1.2.5: resolution: {integrity: sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==} @@ -9990,7 +9917,6 @@ packages: yaml@2.9.0: resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} engines: {node: '>= 14.6'} - hasBin: true yargs-parser@22.0.0: resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} @@ -11882,6 +11808,10 @@ snapshots: transitivePeerDependencies: - supports-color + '@mediabunny/mp3-encoder@1.50.7(mediabunny@1.50.7)': + dependencies: + mediabunny: 1.50.7 + '@mermaid-js/parser@1.2.0': dependencies: '@chevrotain/types': 11.1.2 @@ -13517,6 +13447,12 @@ snapshots: '@types/doctrine@0.0.9': {} + '@types/dom-mediacapture-transform@0.1.11': + dependencies: + '@types/dom-webcodecs': 0.1.13 + + '@types/dom-webcodecs@0.1.13': {} + '@types/esrecurse@4.3.1': {} '@types/estree-jsx@1.0.5': @@ -16959,8 +16895,6 @@ snapshots: '@types/react': 19.2.17 react: 19.2.7 - js-audio-recorder@1.0.7: {} - js-base64@3.8.0: {} js-cookie@3.0.8: {} @@ -17052,10 +16986,6 @@ snapshots: ky@2.0.2: {} - lamejs@1.2.1: - dependencies: - use-strict: 1.0.1 - language-subtag-registry@0.3.23: {} language-tags@1.0.9: @@ -17425,6 +17355,11 @@ snapshots: mdn-data@2.28.1: {} + mediabunny@1.50.7: + dependencies: + '@types/dom-mediacapture-transform': 0.1.11 + '@types/dom-webcodecs': 0.1.13 + merge2@1.4.1: {} mermaid@11.16.0: @@ -19777,8 +19712,6 @@ snapshots: optionalDependencies: '@types/react': 19.2.17 - use-strict@1.0.1: {} - use-sync-external-store@1.6.0(react@19.2.7): dependencies: react: 19.2.7 @@ -20501,6 +20434,7 @@ time: '@mdx-js/loader@3.1.1': '2025-08-29T18:03:05.606Z' '@mdx-js/react@3.1.1': '2025-08-29T18:02:56.462Z' '@mdx-js/rollup@3.1.1': '2025-08-29T18:03:10.680Z' + '@mediabunny/mp3-encoder@1.50.7': '2026-07-07T19:31:44.869Z' '@monaco-editor/react@4.7.0': '2025-02-13T16:13:41.390Z' '@napi-rs/keyring@1.3.0': '2026-04-30T09:56:44.246Z' '@next/eslint-plugin-next@16.2.10': '2026-07-01T20:11:43.814Z' @@ -20610,17 +20544,16 @@ time: jotai-scope@0.11.0: '2026-05-13T18:43:15.331Z' jotai-tanstack-query@0.11.0: '2025-08-01T02:55:49.826Z' jotai@2.20.1: '2026-06-11T06:30:45.782Z' - js-audio-recorder@1.0.7: '2021-01-09T10:20:49.923Z' js-cookie@3.0.8: '2026-05-29T10:51:39.065Z' js-yaml@5.2.1: '2026-07-02T00:55:21.714Z' jsonschema@1.5.0: '2025-01-07T15:09:11.287Z' katex@0.17.0: '2026-05-22T08:06:26.967Z' knip@6.25.0: '2026-07-07T08:47:13.416Z' ky@2.0.2: '2026-04-21T08:58:46.923Z' - lamejs@1.2.1: '2021-12-02T15:44:40.036Z' lexical@0.46.0: '2026-06-26T04:53:00.532Z' lockfile@1.0.4: '2018-04-17T00:36:12.565Z' loro-crdt@1.13.6: '2026-06-21T15:40:04.671Z' + mediabunny@1.50.7: '2026-07-07T19:31:25.625Z' mermaid@11.16.0: '2026-06-25T11:30:40.280Z' mime@4.1.0: '2025-09-12T17:53:01.376Z' mitt@3.0.1: '2023-07-04T17:31:47.638Z' diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 32c247a48b2..038c27e021a 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -77,6 +77,7 @@ catalog: '@mdx-js/loader': 3.1.1 '@mdx-js/react': 3.1.1 '@mdx-js/rollup': 3.1.1 + '@mediabunny/mp3-encoder': 1.50.7 '@monaco-editor/react': 4.7.0 '@napi-rs/keyring': 1.3.0 '@next/eslint-plugin-next': 16.2.10 @@ -185,17 +186,16 @@ catalog: jotai-effect: 2.3.1 jotai-scope: 0.11.0 jotai-tanstack-query: 0.11.0 - js-audio-recorder: 1.0.7 js-cookie: 3.0.8 js-yaml: 5.2.1 jsonschema: 1.5.0 katex: 0.17.0 knip: 6.25.0 ky: 2.0.2 - lamejs: 1.2.1 lexical: 0.46.0 lockfile: 1.0.4 loro-crdt: 1.13.6 + mediabunny: 1.50.7 mermaid: 11.16.0 mime: 4.1.0 mitt: 3.0.1 diff --git a/web/app/components/app/configuration/debug/debug-with-multiple-model/index.tsx b/web/app/components/app/configuration/debug/debug-with-multiple-model/index.tsx index 5fe91d7681d..4cd775f22e6 100644 --- a/web/app/components/app/configuration/debug/debug-with-multiple-model/index.tsx +++ b/web/app/components/app/configuration/debug/debug-with-multiple-model/index.tsx @@ -19,6 +19,7 @@ const DebugWithMultipleModel = () => { mode, inputs, modelConfig, + appId, readonly, canTestAndRun = false, } = useDebugConfigurationContext() @@ -135,6 +136,7 @@ const DebugWithMultipleModel = () => { onFeatureBarClick={setShowAppConfigureFeaturesModal} onSend={handleSend} speechToTextConfig={speech2text as any} + speechToTextTarget={{ type: 'consoleApp', appId }} visionConfig={file} inputs={inputs} inputsForm={inputsForm} diff --git a/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx b/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx index a9bbf6e6c80..533bcbc70c4 100644 --- a/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx +++ b/web/app/components/app/configuration/debug/debug-with-single-model/index.tsx @@ -185,6 +185,7 @@ const DebugWithSingleModel = ({ { imageUrl={appData.site.icon_url} /> ) : null + const speechToTextTarget: SpeechToTextTarget | undefined = + appSourceType === AppSourceType.webApp + ? { type: 'app' as const, appSourceType: AppSourceType.webApp } + : appId + ? { type: 'app' as const, appId, appSourceType } + : undefined return (
({ disabled, readonly, footerNotice, + onBeforeSpeechToText, + speechToTextTarget, }: { customPlaceholder?: string disabled?: boolean readonly?: boolean footerNotice?: string + onBeforeSpeechToText?: () => Promise + speechToTextTarget?: SpeechToTextTarget }) => (
{footerNotice}
@@ -401,6 +415,29 @@ describe('Chat', () => { renderChat({ readonly: true }) expect(screen.getByTestId('chat-input-area')).toHaveAttribute('data-readonly', 'true') }) + + it('should pass the explicit speech-to-text target to ChatInputArea', () => { + renderChat({ + speechToTextTarget: { + type: 'consoleApp', + appId: 'app-123', + }, + }) + expect(screen.getByTestId('chat-input-area')).toHaveAttribute('data-speech-app-id', 'app-123') + expect(screen.getByTestId('chat-input-area')).toHaveAttribute( + 'data-speech-source', + 'consoleApp', + ) + }) + + it('should pass the save-before-transcribe callback to ChatInputArea', () => { + renderChat({ onBeforeSpeechToText: vi.fn().mockResolvedValue(undefined) }) + + expect(screen.getByTestId('chat-input-area')).toHaveAttribute( + 'data-has-before-speech', + 'true', + ) + }) }) describe('PromptLogModal', () => { diff --git a/web/app/components/base/chat/chat/chat-input-area/__tests__/index.spec.tsx b/web/app/components/base/chat/chat/chat-input-area/__tests__/index.spec.tsx index f5ecb3a45f0..7a53403c82e 100644 --- a/web/app/components/base/chat/chat/chat-input-area/__tests__/index.spec.tsx +++ b/web/app/components/base/chat/chat/chat-input-area/__tests__/index.spec.tsx @@ -1,8 +1,11 @@ import type { FileUpload } from '@/app/components/base/features/types' import type { FileEntity } from '@/app/components/base/file-uploader/types' -import { fireEvent, render, screen, waitFor } from '@testing-library/react' +import type { VoiceRecorder } from '@/app/components/base/voice-input/recorder' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import * as React from 'react' +import { transcribeAudio } from '@/app/components/base/voice-input/api' +import { startVoiceRecorder } from '@/app/components/base/voice-input/recorder' import { TransferMethod } from '@/types/app' import ChatInputArea from '../index' @@ -12,82 +15,28 @@ vi.setConfig({ testTimeout: 60000 }) // External dependency mocks // --------------------------------------------------------------------------- -// Track whether getPermission should reject -const { mockGetPermissionConfig } = vi.hoisted(() => ({ - mockGetPermissionConfig: { shouldReject: false }, +const recorderStop = vi.fn<() => Promise>() +const recorderCancel = vi.fn<() => Promise>() +const recorder: VoiceRecorder = { + analyser: { + frequencyBinCount: 8, + getByteFrequencyData: vi.fn(), + } as unknown as AnalyserNode, + stop: recorderStop, + cancel: recorderCancel, +} + +vi.mock('@/app/components/base/voice-input/recorder', () => ({ + startVoiceRecorder: vi.fn(), })) -vi.mock('js-audio-recorder', () => ({ - default: class MockRecorder { - static getPermission = vi.fn().mockImplementation(() => { - if (mockGetPermissionConfig.shouldReject) { - return Promise.reject(new Error('Permission denied')) - } - return Promise.resolve(undefined) - }) +vi.mock('@/app/components/base/voice-input/api', () => ({ transcribeAudio: vi.fn() })) - start = vi.fn().mockResolvedValue(undefined) - stop = vi.fn() - getWAVBlob = vi.fn().mockReturnValue(new Blob([''], { type: 'audio/wav' })) - getRecordAnalyseData = vi.fn().mockReturnValue(new Uint8Array(128)) - getChannelData = vi - .fn() - .mockReturnValue({ left: new Float32Array(0), right: new Float32Array(0) }) - getWAV = vi.fn().mockReturnValue(new ArrayBuffer(0)) - destroy = vi.fn() - }, -})) - -vi.mock('@/app/components/base/voice-input/utils', () => ({ - convertToMp3: vi.fn().mockReturnValue(new Blob([''], { type: 'audio/mp3' })), -})) - -// Mock VoiceInput component - simplified version -vi.mock('@/app/components/base/voice-input', () => { - const VoiceInputMock = ({ - onCancel, - onConverted, - }: { - onCancel: () => void - onConverted: (text: string) => void - }) => { - // Use module-level state for simplicity - const [showStop, setShowStop] = React.useState(true) - - const handleStop = () => { - setShowStop(false) - // Simulate async conversion - setTimeout(() => { - onConverted('Converted voice text') - setShowStop(true) - }, 100) - } - - return ( -
-
voiceInput.speaking
-
voiceInput.converting
- {showStop && ( - - )} - -
- ) - } - - return { - default: VoiceInputMock, - } -}) - -vi.stubGlobal('requestAnimationFrame', (cb: FrameRequestCallback) => - setTimeout(() => cb(Date.now()), 16), +vi.stubGlobal( + 'requestAnimationFrame', + vi.fn(() => 1), ) -vi.stubGlobal('cancelAnimationFrame', (id: number) => clearTimeout(id)) +vi.stubGlobal('cancelAnimationFrame', vi.fn()) vi.stubGlobal('devicePixelRatio', 1) // Mock Canvas @@ -106,11 +55,6 @@ HTMLCanvasElement.prototype.getBoundingClientRect = vi.fn().mockReturnValue({ height: 50, }) -vi.mock('@/service/share', () => ({ - audioToText: vi.fn().mockResolvedValue({ text: 'Converted voice text' }), - AppSourceType: { webApp: 'webApp', installedApp: 'installedApp' }, -})) - // --------------------------------------------------------------------------- // File-uploader store // --------------------------------------------------------------------------- @@ -253,6 +197,11 @@ const mockVisionConfig: FileUpload = { }, } +const speechToTextTarget = { + type: 'consoleApp' as const, + appId: 'app-123', +} + const makeFile = (overrides: Partial = {}): FileEntity => ({ id: 'file-1', @@ -278,6 +227,10 @@ const getTextarea = () => describe('ChatInputArea', () => { beforeEach(() => { vi.clearAllMocks() + vi.mocked(startVoiceRecorder).mockResolvedValue(recorder) + recorderStop.mockResolvedValue(new Blob(['mp3-data'], { type: 'audio/mp3' })) + recorderCancel.mockResolvedValue() + vi.mocked(transcribeAudio).mockResolvedValue({ text: 'Converted voice text' }) mockFileStore.files = [] mockIsDragActive.value = false mockIsMultipleLine.value = false @@ -536,88 +489,360 @@ describe('ChatInputArea', () => { // ------------------------------------------------------------------------- describe('Voice Input', () => { - it('should render the voice input button when enabled', () => { + it('should hide the voice input button without an API target', () => { render( , ) - expect(screen.getByRole('button', { name: 'common.voiceInput.start' })).toBeTruthy() + expect( + screen.queryByRole('button', { name: 'common.voiceInput.start' }), + ).not.toBeInTheDocument() + }) + + it('should render the voice input button when enabled with an API target', () => { + render( + , + ) + expect(screen.getByRole('button', { name: 'common.voiceInput.start' })).toBeInTheDocument() + }) + + it('should keep the active recorder when the chat input rerenders', async () => { + const user = userEvent.setup({ delay: null }) + const { rerender } = render( + , + ) + await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) + await screen.findByRole('button', { name: 'common.voiceInput.stop' }) + + rerender( + , + ) + + expect(startVoiceRecorder).toHaveBeenCalledTimes(1) + expect(recorderCancel).not.toHaveBeenCalled() }) it('should handle stop recording in VoiceInput', async () => { const user = userEvent.setup({ delay: null }) render( - , + , ) await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) - // Wait for VoiceInput to show speaking - await screen.findByText(/voiceInput.speaking/i) - const stopBtn = screen.getByTestId('voice-input-stop') - await user.click(stopBtn) - - // Converting should show up - await screen.findByText(/voiceInput.converting/i) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) await waitFor(() => { expect(getTextarea()!).toHaveValue('Converted voice text') }) }) - it('should handle cancel in VoiceInput', async () => { + it('should focus the textarea at the end of converted voice text', async () => { const user = userEvent.setup({ delay: null }) render( - , + , + ) + const textarea = getTextarea()! + + await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + + await waitFor(() => expect(textarea).toHaveValue('Converted voice text')) + await waitFor(() => expect(textarea).toHaveFocus()) + expect(textarea.selectionStart).toBe(textarea.value.length) + expect(textarea.selectionEnd).toBe(textarea.value.length) + }) + + it('should focus the textarea when conversion completes while focus is in the voice input', async () => { + const user = userEvent.setup({ delay: null }) + let resolveTranscription: ((value: { text: string }) => void) | undefined + vi.mocked(transcribeAudio).mockReturnValueOnce( + new Promise((resolve) => { + resolveTranscription = resolve + }), + ) + render( + , + ) + const textarea = getTextarea()! + + await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + const cancelButton = await screen.findByRole('button', { name: 'common.operation.cancel' }) + cancelButton.focus() + await act(async () => resolveTranscription?.({ text: 'Converted voice text' })) + + await waitFor(() => expect(textarea).toHaveFocus()) + expect(textarea).toHaveValue('Converted voice text') + }) + + it('should preserve focus when the user moves elsewhere during transcription', async () => { + const user = userEvent.setup({ delay: null }) + let resolveTranscription: ((value: { text: string }) => void) | undefined + vi.mocked(transcribeAudio).mockReturnValueOnce( + new Promise((resolve) => { + resolveTranscription = resolve + }), + ) + render( + <> + + + , + ) + const textarea = getTextarea()! + const elsewhereButton = screen.getByRole('button', { name: 'Elsewhere' }) + + await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + await user.click(elsewhereButton) + await act(async () => resolveTranscription?.({ text: 'Converted voice text' })) + + await waitFor(() => expect(textarea).toHaveValue('Converted voice text')) + expect(elsewhereButton).toHaveFocus() + }) + + it('should wait for the owning draft before transcription', async () => { + const user = userEvent.setup({ delay: null }) + const onBeforeSpeechToText = vi.fn().mockResolvedValue(undefined) + render( + , ) await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) - await screen.findByText(/voiceInput.speaking/i) - const stopBtn = screen.getByTestId('voice-input-stop') - await user.click(stopBtn) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) - // Wait for converting and cancel button - const cancelBtn = await screen.findByTestId('voice-input-cancel') + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + expect(onBeforeSpeechToText).toHaveBeenCalledTimes(1) + expect(onBeforeSpeechToText.mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(transcribeAudio).mock.invocationCallOrder[0]!, + ) + }) + + it('should preserve the current query and report transcription failure', async () => { + const user = userEvent.setup({ delay: null }) + vi.mocked(transcribeAudio).mockRejectedValueOnce(new Error('API error')) + render( + , + ) + const voiceInputButton = screen.getByRole('button', { name: 'common.voiceInput.start' }) + await user.type(getTextarea()!, 'Keep this text') + + await user.click(voiceInputButton) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith({ + type: 'error', + message: 'common.api.actionFailed', + }) + }) + expect(getTextarea()!).toHaveValue('Keep this text') + await waitFor(() => expect(voiceInputButton).toHaveFocus()) + }) + + it('should restore focus to the voice input trigger when conversion is cancelled', async () => { + const user = userEvent.setup({ delay: null }) + vi.mocked(transcribeAudio).mockImplementationOnce(() => new Promise(() => {})) + render( + , + ) + const voiceInputButton = screen.getByRole('button', { name: 'common.voiceInput.start' }) + + await user.click(voiceInputButton) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + + const cancelBtn = await screen.findByRole('button', { name: 'common.operation.cancel' }) await user.click(cancelBtn) await waitFor(() => { - expect(screen.queryByTestId('voice-input-stop')).toBeNull() + expect(screen.queryByText(/voiceInput.converting/i)).not.toBeInTheDocument() }) + await waitFor(() => expect(voiceInputButton).toHaveFocus()) + }) + + it('should restore focus to the voice input trigger when setup is cancelled', async () => { + const user = userEvent.setup({ delay: null }) + vi.mocked(startVoiceRecorder).mockReturnValueOnce(new Promise(() => {})) + render( + , + ) + const voiceInputButton = screen.getByRole('button', { name: 'common.voiceInput.start' }) + + await user.click(voiceInputButton) + await user.click(await screen.findByRole('button', { name: 'common.operation.cancel' })) + + await waitFor(() => expect(voiceInputButton).toHaveFocus()) + }) + + it('should restore focus to the voice input trigger when stopping the recorder fails', async () => { + const user = userEvent.setup({ delay: null }) + recorderStop.mockRejectedValueOnce(new Error('Recorder failed')) + render( + , + ) + const voiceInputButton = screen.getByRole('button', { name: 'common.voiceInput.start' }) + + await user.click(voiceInputButton) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith({ + type: 'error', + message: 'common.api.actionFailed', + }) + }) + await waitFor(() => expect(voiceInputButton).toHaveFocus()) + }) + + it('should preserve focus when the user moves elsewhere before transcription fails', async () => { + const user = userEvent.setup({ delay: null }) + let rejectTranscription: ((reason?: unknown) => void) | undefined + vi.mocked(transcribeAudio).mockReturnValueOnce( + new Promise((_resolve, reject) => { + rejectTranscription = reject + }), + ) + render( + <> + + + , + ) + const elsewhereButton = screen.getByRole('button', { name: 'Elsewhere' }) + + await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + await user.click(elsewhereButton) + await act(async () => rejectTranscription?.(new Error('API error'))) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith({ + type: 'error', + message: 'common.api.actionFailed', + }) + }) + expect(elsewhereButton).toHaveFocus() }) it('should show error toast when voice permission is denied', async () => { const user = userEvent.setup({ delay: null }) - mockGetPermissionConfig.shouldReject = true + vi.mocked(startVoiceRecorder).mockRejectedValueOnce( + new DOMException('Permission denied', 'NotAllowedError'), + ) render( - , + , ) await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) - // Permission denied should trigger error toast await waitFor(() => { - expect(mockNotify).toHaveBeenCalledWith(expect.objectContaining({ type: 'error' })) + expect(mockNotify).toHaveBeenCalledWith({ + type: 'error', + message: 'common.voiceInput.notAllow', + }) }) + }) - mockGetPermissionConfig.shouldReject = false + it('should show a generic error when voice input setup fails after permission', async () => { + const user = userEvent.setup({ delay: null }) + vi.mocked(startVoiceRecorder).mockRejectedValueOnce(new Error('AudioWorklet unavailable')) + + render( + , + ) + const voiceInputButton = screen.getByRole('button', { name: 'common.voiceInput.start' }) + + await user.click(voiceInputButton) + + await waitFor(() => { + expect(mockNotify).toHaveBeenCalledWith({ + type: 'error', + message: 'common.api.actionFailed', + }) + }) + expect(voiceInputButton).toHaveFocus() }) it('should handle empty converted text in VoiceInput', async () => { const user = userEvent.setup({ delay: null }) - // Mock failure or empty result - const { audioToText } = await import('@/service/share') - vi.mocked(audioToText).mockResolvedValueOnce({ text: '' }) + vi.mocked(transcribeAudio).mockResolvedValueOnce({ text: '' }) render( - , + , ) await user.click(screen.getByRole('button', { name: 'common.voiceInput.start' })) - await screen.findByText(/voiceInput.speaking/i) - const stopBtn = screen.getByTestId('voice-input-stop') - await user.click(stopBtn) + await user.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) await waitFor(() => { - expect(screen.queryByTestId('voice-input-stop')).toBeNull() + expect(screen.queryByText(/voiceInput.converting/i)).not.toBeInTheDocument() }) expect(getTextarea()!).toHaveValue('') }) diff --git a/web/app/components/base/chat/chat/chat-input-area/__tests__/operation.spec.tsx b/web/app/components/base/chat/chat/chat-input-area/__tests__/operation.spec.tsx index b026634bb80..de1cd921785 100644 --- a/web/app/components/base/chat/chat/chat-input-area/__tests__/operation.spec.tsx +++ b/web/app/components/base/chat/chat/chat-input-area/__tests__/operation.spec.tsx @@ -43,10 +43,12 @@ describe('Operation', () => { expect(screen.queryByTestId('file-uploader')).not.toBeInTheDocument() }) - it('should render voice input button when speechToTextConfig.enabled is true', () => { + it('should render voice input button when speech-to-text is enabled with a handler', () => { const speechConfig: EnableType = { enabled: true } - render() + render( + , + ) expect(screen.getByRole('button', { name: 'common.voiceInput.start' })).toBeInTheDocument() expect(screen.getByRole('button', { name: 'common.operation.send' })).toBeInTheDocument() @@ -57,7 +59,12 @@ describe('Operation', () => { const speechConfig: EnableType = { enabled: true } render( - , + , ) const fileUploader = screen.getByTestId('file-uploader') diff --git a/web/app/components/base/chat/chat/chat-input-area/index.tsx b/web/app/components/base/chat/chat/chat-input-area/index.tsx index 74f6cf0bc91..c22681bf4bf 100644 --- a/web/app/components/base/chat/chat/chat-input-area/index.tsx +++ b/web/app/components/base/chat/chat/chat-input-area/index.tsx @@ -3,11 +3,11 @@ import type { Theme } from '../../embedded-chatbot/theme/theme-context' import type { EnableType, OnSend } from '../../types' import type { InputForm } from '../type' import type { FileUpload } from '@/app/components/base/features/types' +import type { SpeechToTextTarget } from '@/app/components/base/voice-input/types' import { cn } from '@langgenius/dify-ui/cn' import { toast } from '@langgenius/dify-ui/toast' import { noop } from 'es-toolkit/function' import { decode } from 'html-entities' -import Recorder from 'js-audio-recorder' import { useCallback, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import Textarea from 'react-textarea-autosize' @@ -22,12 +22,12 @@ import { useCheckInputsForms } from '../check-input-forms-hooks' import { useTextAreaHeight } from './hooks' import Operation from './operation' -type AudioRecorderWithPermission = typeof Recorder & { - getPermission: () => Promise -} - type SendAcceptance = void | boolean | Promise +function isMicrophonePermissionDenied(error: unknown) { + return error instanceof DOMException && error.name === 'NotAllowedError' +} + type ChatInputAreaProps = { readonly?: boolean botName?: string @@ -39,6 +39,8 @@ type ChatInputAreaProps = { onFeatureBarClick?: (state: boolean) => void visionConfig?: FileUpload speechToTextConfig?: EnableType + speechToTextTarget?: SpeechToTextTarget + onBeforeSpeechToText?: () => Promise onSend?: OnSend inputs?: Record inputsForm?: InputForm[] @@ -69,6 +71,8 @@ const ChatInputArea = ({ onFeatureBarClick, visionConfig, speechToTextConfig = { enabled: true }, + speechToTextTarget, + onBeforeSpeechToText, onSend, inputs = {}, inputsForm = [], @@ -108,6 +112,9 @@ const ChatInputArea = ({ const [currentIndex, setCurrentIndex] = useState(-1) const isComposingRef = useRef(false) const queryRef = useRef('') + const voiceInputRef = useRef(null) + const voiceInputReturnFocusRef = useRef(null) + const voiceInputCompletedRef = useRef(false) const handleQueryChange = useCallback( (value: string) => { queryRef.current = value @@ -199,22 +206,85 @@ const ChatInputArea = ({ } } const handleShowVoiceInput = useCallback(() => { - ;(Recorder as AudioRecorderWithPermission).getPermission().then( - () => { - setShowVoiceInput(true) - }, - () => { - toast.error(t(($) => $['voiceInput.notAllow'], { ns: 'common' })) - }, - ) + const textareaElement = textareaRef.current + if (textareaElement) { + const activeElement = textareaElement.ownerDocument.activeElement + voiceInputReturnFocusRef.current = + activeElement instanceof HTMLElement && activeElement !== textareaElement.ownerDocument.body + ? activeElement + : textareaElement + } + voiceInputCompletedRef.current = false + setShowVoiceInput(true) + }, [textareaRef]) + const handleVoiceInputStartError = useCallback( + (error: unknown) => { + toast.error( + t( + ($) => + $[isMicrophonePermissionDenied(error) ? 'voiceInput.notAllow' : 'api.actionFailed'], + { ns: 'common' }, + ), + ) + }, + [t], + ) + const handleVoiceInputError = useCallback(() => { + toast.error(t(($) => $['api.actionFailed'], { ns: 'common' })) }, [t]) + const handleHideVoiceInput = useCallback(() => { + const textareaElement = textareaRef.current + const documentElement = textareaElement?.ownerDocument + const activeElement = documentElement?.activeElement + const shouldRestoreFocus = + !!documentElement && + (activeElement === documentElement.body || + !!(activeElement && voiceInputRef.current?.contains(activeElement))) + const returnFocusElement = voiceInputReturnFocusRef.current + const voiceInputCompleted = voiceInputCompletedRef.current + voiceInputReturnFocusRef.current = null + voiceInputCompletedRef.current = false + setShowVoiceInput(false) + if (voiceInputCompleted || !shouldRestoreFocus || !documentElement) return + + queueMicrotask(() => { + const currentActiveElement = documentElement.activeElement + if (currentActiveElement !== documentElement.body && currentActiveElement !== activeElement) + return + + const elementToFocus = returnFocusElement?.isConnected ? returnFocusElement : textareaElement + elementToFocus?.focus({ preventScroll: true }) + }) + }, [textareaRef]) + const handleVoiceInputConverted = (text: string) => { + voiceInputCompletedRef.current = true + handleQueryChange(text) + queueMicrotask(() => { + const textareaElement = textareaRef.current + if (!textareaElement) return + + const activeElement = textareaElement.ownerDocument.activeElement + if ( + activeElement && + activeElement !== textareaElement.ownerDocument.body && + activeElement !== textareaElement && + !voiceInputRef.current?.contains(activeElement) + ) { + return + } + + textareaElement.focus({ preventScroll: true }) + const caretPosition = textareaElement.value.length + textareaElement.setSelectionRange(caretPosition, caretPosition) + }) + } const operation = ( {!isMultipleLine && operation}
- {showVoiceInput && ( + {showVoiceInput && speechToTextTarget && ( setShowVoiceInput(false)} - onConverted={(text) => handleQueryChange(text)} + ref={voiceInputRef} + target={speechToTextTarget} + onCancel={handleHideVoiceInput} + onBeforeTranscribe={onBeforeSpeechToText} + onConverted={handleVoiceInputConverted} + onError={handleVoiceInputError} + onStartError={handleVoiceInputStartError} /> )} diff --git a/web/app/components/base/chat/chat/chat-input-area/operation.tsx b/web/app/components/base/chat/chat/chat-input-area/operation.tsx index eb80ef85a62..2e9a10605bb 100644 --- a/web/app/components/base/chat/chat/chat-input-area/operation.tsx +++ b/web/app/components/base/chat/chat/chat-input-area/operation.tsx @@ -42,8 +42,9 @@ const Operation: FC = ({ {fileConfig?.enabled && ( )} - {speechToTextConfig?.enabled && ( + {speechToTextConfig?.enabled && onShowVoiceInput && ( $['voiceInput.start'], { ns: 'common' })} disabled={readonly} diff --git a/web/app/components/base/chat/chat/index.tsx b/web/app/components/base/chat/chat/index.tsx index 00c9e7505a2..2c35864382a 100644 --- a/web/app/components/base/chat/chat/index.tsx +++ b/web/app/components/base/chat/chat/index.tsx @@ -3,6 +3,7 @@ import type { ThemeBuilder } from '../embedded-chatbot/theme/theme-context' import type { ChatConfig, ChatItem, Feedback, OnRegenerate, OnSend } from '../types' import type { HumanInputFormSubmitData } from './answer/human-input-content/type' import type { InputForm } from './type' +import type { SpeechToTextTarget } from '@/app/components/base/voice-input/types' import type { HumanInputNodeType } from '@/app/components/workflow/nodes/human-input/types' import type { Node } from '@/app/components/workflow/types' import type { AppData, ToolIcon } from '@/models/share' @@ -75,6 +76,8 @@ export type ChatProps = { sidebarCollapseState?: boolean hideAvatar?: boolean sendOnEnter?: boolean + speechToTextTarget?: SpeechToTextTarget + onBeforeSpeechToText?: () => Promise renderAgentContent?: (props: { item: ChatItem responding?: boolean @@ -132,6 +135,8 @@ const Chat: FC = ({ sidebarCollapseState, hideAvatar, sendOnEnter, + speechToTextTarget, + onBeforeSpeechToText, renderAgentContent, onHumanInputFormSubmit, getHumanInputNodeData, @@ -281,6 +286,8 @@ const Chat: FC = ({ onFeatureBarClick={onFeatureBarClick} visionConfig={config?.file_upload} speechToTextConfig={config?.speech_to_text} + speechToTextTarget={speechToTextTarget} + onBeforeSpeechToText={onBeforeSpeechToText} onSend={onSend} inputs={inputs} inputsForm={inputsForm} diff --git a/web/app/components/base/chat/embedded-chatbot/chat-wrapper.tsx b/web/app/components/base/chat/embedded-chatbot/chat-wrapper.tsx index aaffc1331f1..e4c089c0d1a 100644 --- a/web/app/components/base/chat/embedded-chatbot/chat-wrapper.tsx +++ b/web/app/components/base/chat/embedded-chatbot/chat-wrapper.tsx @@ -420,12 +420,19 @@ const ChatWrapper = () => { imageUrl={appData.site.icon_url} /> ) : null + const speechToTextTarget = + appSourceType === AppSourceType.webApp + ? { type: 'app' as const, appSourceType } + : appId + ? { type: 'app' as const, appId, appSourceType } + : undefined return ( { + const original = await importOriginal() + return { + ...original, + audioToText: vi.fn(), + } +}) + +vi.mock('@/service/client', () => ({ + consoleClient: { + apps: { + byAppId: { + audioToText: { + post: vi.fn(), + }, + }, + }, + agent: { + byAgentId: { + audioToText: { + post: vi.fn(), + }, + }, + }, + }, +})) + +describe('transcribeAudio', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + // App surfaces retain their existing transport-specific endpoints. + it('should send app audio through the existing app transport', async () => { + const file = new File(['mp3'], 'temp.mp3', { type: 'audio/mp3' }) + const signal = new AbortController().signal + vi.mocked(audioToText).mockResolvedValue({ text: 'app transcript' }) + + const result = await transcribeAudio( + { + type: 'app', + appId: 'app-1', + appSourceType: AppSourceType.installedApp, + }, + file, + signal, + ) + + expect(result).toEqual({ text: 'app transcript' }) + const formData = vi.mocked(audioToText).mock.calls[0]![2] + expect(audioToText).toHaveBeenCalledWith(AppSourceType.installedApp, 'app-1', formData, signal) + expect(formData.get('file')).toBe(file) + }) + + it('should send console App audio through its generated multipart contract', async () => { + const file = new File(['mp3'], 'temp.mp3', { type: 'audio/mp3' }) + const signal = new AbortController().signal + const post = consoleClient.apps.byAppId.audioToText.post + vi.mocked(post).mockResolvedValue({ text: 'console app transcript' }) + + const result = await transcribeAudio({ type: 'consoleApp', appId: 'app-1' }, file, signal) + + expect(result).toEqual({ text: 'console app transcript' }) + expect(post).toHaveBeenCalledWith( + { + body: { file }, + params: { app_id: 'app-1' }, + }, + { signal }, + ) + expect(audioToText).not.toHaveBeenCalled() + }) + + // Agent debug audio uses its generated ID-first multipart contract. + it('should send Agent audio with the requested draft type', async () => { + const file = new File(['mp3'], 'temp.mp3', { type: 'audio/mp3' }) + const signal = new AbortController().signal + const post = consoleClient.agent.byAgentId.audioToText.post + vi.mocked(post).mockResolvedValue({ text: 'agent transcript' }) + + const result = await transcribeAudio( + { + type: 'agent', + agentId: 'agent-1', + draftType: 'debug_build', + }, + file, + signal, + ) + + expect(result).toEqual({ text: 'agent transcript' }) + expect(post).toHaveBeenCalledWith( + { + body: { + draft_type: 'debug_build', + file, + }, + params: { + agent_id: 'agent-1', + }, + }, + { signal }, + ) + expect(audioToText).not.toHaveBeenCalled() + }) +}) diff --git a/web/app/components/base/voice-input/__tests__/index.spec.tsx b/web/app/components/base/voice-input/__tests__/index.spec.tsx index 72f830f7247..699d78f6076 100644 --- a/web/app/components/base/voice-input/__tests__/index.spec.tsx +++ b/web/app/components/base/voice-input/__tests__/index.spec.tsx @@ -1,523 +1,318 @@ -import { act, render, screen, waitFor } from '@testing-library/react' -import userEvent from '@testing-library/user-event' -import { audioToText } from '@/service/share' +import type { VoiceRecorder } from '../recorder' +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react' +import { StrictMode } from 'react' +import { transcribeAudio } from '../api' import VoiceInput from '../index' +import { startVoiceRecorder } from '../recorder' -const { mockState, MockRecorder, rafState } = vi.hoisted(() => { - const state = { - params: {} as Record, - pathname: '/test', - recorderInstances: [] as unknown[], - startOverride: null as (() => Promise) | null, - analyseData: new Uint8Array(1024).fill(150) as Uint8Array, - } - const rafStateObj = { - callback: null as (() => void) | null, - } +vi.mock('../api', () => ({ transcribeAudio: vi.fn() })) - class MockRecorderClass { - start = vi.fn((..._args: unknown[]) => { - if (state.startOverride) return state.startOverride() - return Promise.resolve() - }) - - stop = vi.fn() - getRecordAnalyseData = vi.fn(() => state.analyseData) - getWAV = vi.fn(() => new ArrayBuffer(0)) - getChannelData = vi.fn(() => ({ - left: { buffer: new ArrayBuffer(2048), byteLength: 2048 }, - right: { buffer: new ArrayBuffer(2048), byteLength: 2048 }, - })) - - constructor() { - state.recorderInstances.push(this) - } - } - - return { mockState: state, MockRecorder: MockRecorderClass, rafState: rafStateObj } -}) - -vi.mock('js-audio-recorder', () => ({ - default: MockRecorder, +vi.mock('../recorder', () => ({ + startVoiceRecorder: vi.fn(), })) -vi.mock('@/service/share', () => ({ - AppSourceType: { webApp: 'webApp', installedApp: 'installedApp' }, - audioToText: vi.fn(), -})) +const recorderStop = vi.fn<() => Promise>() +const recorderCancel = vi.fn<() => Promise>() +const getByteFrequencyData = vi.fn() +const recorder: VoiceRecorder = { + analyser: { + frequencyBinCount: 8, + getByteFrequencyData, + } as unknown as AnalyserNode, + stop: recorderStop, + cancel: recorderCancel, +} +const target = { + type: 'consoleApp' as const, + appId: 'app-123', +} +const canvasContext = { + beginPath: vi.fn(), + clearRect: vi.fn(), + closePath: vi.fn(), + fill: vi.fn(), + rect: vi.fn(), + roundRect: vi.fn(), + scale: vi.fn(), + get fillStyle() { + return '' + }, + set fillStyle(_value: string) {}, +} as unknown as CanvasRenderingContext2D -vi.mock('@/next/navigation', () => ({ - useParams: vi.fn(() => mockState.params), - usePathname: vi.fn(() => mockState.pathname), -})) - -vi.mock('../utils', () => ({ - convertToMp3: vi.fn(() => new Blob(['test'], { type: 'audio/mp3' })), -})) - -vi.mock('ahooks', async (importOriginal) => { - const actual = await importOriginal() - return { - ...actual, - useRafInterval: vi.fn((fn) => { - rafState.callback = fn - return vi.fn() - }), +const renderVoiceInput = (overrides: Partial> = {}) => { + const props: React.ComponentProps = { + onCancel: vi.fn(), + onBeforeTranscribe: vi.fn().mockResolvedValue(undefined), + onConverted: vi.fn(), + onError: vi.fn(), + onStartError: vi.fn(), + target, + ...overrides, } -}) + return { ...render(), props } +} describe('VoiceInput', () => { - const onConverted = vi.fn() - const onCancel = vi.fn() - beforeEach(() => { vi.clearAllMocks() - mockState.params = {} - mockState.pathname = '/test' - mockState.recorderInstances = [] - mockState.startOverride = null - rafState.callback = null - - // Ensure canvas has non-zero dimensions for initCanvas() - HTMLCanvasElement.prototype.getBoundingClientRect = vi.fn(() => ({ - width: 300, - height: 32, - top: 0, + vi.mocked(startVoiceRecorder).mockResolvedValue(recorder) + recorderStop.mockResolvedValue(new Blob(['mp3-data'], { type: 'audio/mp3' })) + recorderCancel.mockResolvedValue() + vi.mocked(transcribeAudio).mockResolvedValue({ text: 'transcribed text' }) + vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue(canvasContext) + vi.spyOn(HTMLCanvasElement.prototype, 'getBoundingClientRect').mockReturnValue({ + bottom: 16, + height: 16, left: 0, right: 300, - bottom: 32, + top: 0, + width: 300, x: 0, y: 0, - toJSON: vi.fn(), - })) - - vi.spyOn(window, 'requestAnimationFrame').mockImplementation(() => 1) - vi.spyOn(window, 'cancelAnimationFrame').mockImplementation(() => {}) - }) - - it('should start recording on mount and show speaking state', async () => { - render() - // eslint-disable-next-line ts/no-explicit-any - const recorder = mockState.recorderInstances[0] as any - expect(recorder.start).toHaveBeenCalled() - expect(await screen.findByText('common.voiceInput.speaking'))!.toBeInTheDocument() - expect(screen.getByTestId('voice-input-stop'))!.toBeInTheDocument() - expect(screen.getByTestId('voice-input-timer'))!.toHaveTextContent('00:00') - }) - - it('should call onCancel when recording start fails', async () => { - mockState.startOverride = () => Promise.reject(new Error('Permission denied')) - - render() - await waitFor(() => { - expect(onCancel).toHaveBeenCalled() + toJSON: () => ({}), }) + vi.stubGlobal( + 'requestAnimationFrame', + vi.fn(() => 1), + ) + vi.stubGlobal('cancelAnimationFrame', vi.fn()) }) - it('should stop recording and convert audio on stop click', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'hello world' }) - mockState.params = { token: 'abc' } - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - // eslint-disable-next-line ts/no-explicit-any - const recorder = mockState.recorderInstances[0] as any - expect(await screen.findByTestId('voice-input-converting-text'))!.toBeInTheDocument() - expect(screen.getByText('common.voiceInput.converting'))!.toBeInTheDocument() - expect(screen.getByTestId('voice-input-loader'))!.toBeInTheDocument() - - await waitFor(() => { - expect(recorder.stop).toHaveBeenCalled() - expect(onConverted).toHaveBeenCalledWith('hello world') - expect(onCancel).toHaveBeenCalled() - }) - }) - - it('should call onConverted with empty string on conversion failure', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockRejectedValueOnce(new Error('API error')) - mockState.params = { token: 'abc' } - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - await waitFor(() => { - expect(onConverted).toHaveBeenCalledWith('') - expect(onCancel).toHaveBeenCalled() - }) - }) - - it('should show cancel button during conversion and cancel on click', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockImplementation(() => new Promise(() => {})) - mockState.params = { token: 'abc' } - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - const cancelBtn = await screen.findByTestId('voice-input-cancel') - await user.click(cancelBtn) - - expect(onCancel).toHaveBeenCalled() - }) - - it('should draw on canvas with low data values triggering v < 128 clamp', async () => { - mockState.analyseData = new Uint8Array(1024).fill(50) - - let rafCalls = 0 - vi.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => { - rafCalls++ - if (rafCalls <= 2) cb(0) - return rafCalls - }) - - render() - await screen.findByTestId('voice-input-stop') - - // eslint-disable-next-line ts/no-explicit-any - const firstRecorder = mockState.recorderInstances[0] as any - expect(firstRecorder.getRecordAnalyseData).toHaveBeenCalled() - }) - - it('should draw on canvas with high data values triggering v > 178 clamp', async () => { - mockState.analyseData = new Uint8Array(1024).fill(250) - - let rafCalls = 0 - vi.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => { - rafCalls++ - if (rafCalls <= 2) cb(0) - return rafCalls - }) - - render() - await screen.findByTestId('voice-input-stop') - - // eslint-disable-next-line ts/no-explicit-any - const firstRecorder = mockState.recorderInstances[0] as any - expect(firstRecorder.getRecordAnalyseData).toHaveBeenCalled() - }) - - it('should pass wordTimestamps in form data', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { token: 'abc' } - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - await waitFor(() => { - expect(audioToText).toHaveBeenCalled() - const formData = vi.mocked(audioToText).mock.calls[0]![2] as FormData - expect(formData.get('word_timestamps')).toBe('enabled') - }) - }) - - describe('URL patterns', () => { - it('should use webApp source with /audio-to-text for token-based URL', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { token: 'my-token' } - - render() - await user.click(await screen.findByTestId('voice-input-stop')) - - await waitFor(() => { - expect(audioToText).toHaveBeenCalledWith('/audio-to-text', 'webApp', expect.any(FormData)) + // The component owns the local recording state and browser-resource lifecycle. + describe('Recording state', () => { + it('should expose a cancellable starting state while microphone permission is pending', async () => { + let setupSignal: AbortSignal | undefined + vi.mocked(startVoiceRecorder).mockImplementationOnce((signal) => { + setupSignal = signal + return new Promise(() => {}) }) + const { props } = renderVoiceInput() + + expect(screen.getByRole('status')).toHaveTextContent('common.voiceInput.starting') + fireEvent.click(screen.getByRole('button', { name: 'common.operation.cancel' })) + + expect(props.onCancel).toHaveBeenCalledTimes(1) + expect(setupSignal?.aborted).toBe(true) }) - it('should use installed-apps URL when pathname is an installed app route', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { appId: 'app-123' } - mockState.pathname = '/installed/app-123' + it('should show the recording state after microphone setup succeeds', async () => { + renderVoiceInput() - render() - await user.click(await screen.findByTestId('voice-input-stop')) + expect(await screen.findByText('common.voiceInput.speaking')).toBeInTheDocument() + expect(screen.getByRole('button', { name: 'common.voiceInput.stop' })).toBeInTheDocument() + expect(screen.getByTestId('voice-input-timer')).toHaveTextContent('00:00') + expect(getByteFrequencyData).toHaveBeenCalledTimes(1) + }) - await waitFor(() => { - expect(audioToText).toHaveBeenCalledWith( - '/installed-apps/app-123/audio-to-text', - 'installedApp', - expect.any(FormData), + it('should report the original setup failure and close the voice input', async () => { + const startError = new DOMException('microphone denied', 'NotAllowedError') + vi.mocked(startVoiceRecorder).mockRejectedValueOnce(startError) + const { props } = renderVoiceInput() + + await waitFor(() => expect(props.onStartError).toHaveBeenCalledWith(startError)) + expect(props.onCancel).toHaveBeenCalledTimes(1) + }) + + it('should cancel recorder resources when unmounted', async () => { + const { unmount } = renderVoiceInput() + await screen.findByText('common.voiceInput.speaking') + + unmount() + + expect(recorderCancel).toHaveBeenCalledTimes(1) + }) + + it('should discard a stale recorder when effects are replayed', async () => { + let resolveFirstRecorder: (value: VoiceRecorder) => void = () => {} + let resolveSecondRecorder: (value: VoiceRecorder) => void = () => {} + const staleCancel = vi.fn().mockResolvedValue(undefined) + const activeCancel = vi.fn().mockResolvedValue(undefined) + vi.mocked(startVoiceRecorder) + .mockReturnValueOnce( + new Promise((resolve) => { + resolveFirstRecorder = resolve + }), ) - }) - }) - - it('should use /apps URL for non-explore paths with appId', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { appId: 'app-456' } - mockState.pathname = '/dashboard/apps' - - render() - await user.click(await screen.findByTestId('voice-input-stop')) - - await waitFor(() => { - expect(audioToText).toHaveBeenCalledWith( - '/apps/app-456/audio-to-text', - 'installedApp', - expect.any(FormData), + .mockReturnValueOnce( + new Promise((resolve) => { + resolveSecondRecorder = resolve + }), ) - }) - }) - }) - it('should use fallback rect when canvas roundRect is not available', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { token: 'abc' } - mockState.analyseData = new Uint8Array(1024).fill(150) - - const oldGetContext = HTMLCanvasElement.prototype.getContext - HTMLCanvasElement.prototype.getContext = vi.fn(() => ({ - scale: vi.fn(), - clearRect: vi.fn(), - beginPath: vi.fn(), - moveTo: vi.fn(), - rect: vi.fn(), - fill: vi.fn(), - closePath: vi.fn(), - })) as unknown as typeof HTMLCanvasElement.prototype.getContext - - let rafCalls = 0 - vi.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => { - rafCalls++ - if (rafCalls <= 1) cb(0) - return rafCalls - }) - - render() - await user.click(await screen.findByTestId('voice-input-stop')) - - await waitFor(() => { - expect(onConverted).toHaveBeenCalled() - }) - HTMLCanvasElement.prototype.getContext = oldGetContext - }) - - it('should display timer in MM:SS format correctly', async () => { - mockState.params = { token: 'abc' } - - render() - const timer = await screen.findByTestId('voice-input-timer') - expect(timer)!.toHaveTextContent('00:00') - - await act(async () => { - if (rafState.callback) rafState.callback() - }) - expect(timer)!.toHaveTextContent('00:01') - - for (let i = 0; i < 9; i++) { - await act(async () => { - if (rafState.callback) rafState.callback() - }) - } - expect(timer)!.toHaveTextContent('00:10') - }) - - it('should show timer element with formatted time', async () => { - mockState.params = { token: 'abc' } - - render() - const timer = screen.getByTestId('voice-input-timer') - expect(timer)!.toBeInTheDocument() - // Initial state should show 00:00 - expect(timer.textContent).toMatch(/0\d:\d{2}/) - }) - - it('should handle data values in normal range (between 128 and 178)', async () => { - mockState.analyseData = new Uint8Array(1024).fill(150) - - let rafCalls = 0 - vi.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => { - rafCalls++ - if (rafCalls <= 2) cb(0) - return rafCalls - }) - - render() - await screen.findByTestId('voice-input-stop') - - // eslint-disable-next-line ts/no-explicit-any - const recorder = mockState.recorderInstances[0] as any - expect(recorder.getRecordAnalyseData).toHaveBeenCalled() - }) - - it('should handle canvas context and device pixel ratio', async () => { - const dprSpy = vi.spyOn(window, 'devicePixelRatio', 'get') - dprSpy.mockReturnValue(2) - - render() - await screen.findByTestId('voice-input-stop') - - expect(screen.getByTestId('voice-input-stop'))!.toBeInTheDocument() - - dprSpy.mockRestore() - }) - - it('should handle empty params with no token or appId', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = {} - mockState.pathname = '/test' - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - await waitFor(() => { - // Should call audioToText with empty URL when neither token nor appId is present - expect(audioToText).toHaveBeenCalledWith('', 'installedApp', expect.any(FormData)) - }) - }) - - it('should render speaking state indicator', async () => { - render() - expect(await screen.findByText('common.voiceInput.speaking'))!.toBeInTheDocument() - }) - - it('should cleanup on unmount', () => { - const { unmount } = render() - // eslint-disable-next-line ts/no-explicit-any - const recorder = mockState.recorderInstances[0] as any - - unmount() - - expect(recorder.stop).toHaveBeenCalled() - }) - - it('should handle all data in recordAnalyseData for canvas drawing', async () => { - const allDataValues = [] - for (let i = 0; i < 256; i++) { - allDataValues.push(i) - } - mockState.analyseData = new Uint8Array(allDataValues) - - let rafCalls = 0 - vi.spyOn(window, 'requestAnimationFrame').mockImplementation((cb) => { - rafCalls++ - if (rafCalls <= 2) cb(0) - return rafCalls - }) - - render() - await screen.findByTestId('voice-input-stop') - - // eslint-disable-next-line ts/no-explicit-any - const recorder = mockState.recorderInstances[0] as any - expect(recorder.getRecordAnalyseData).toHaveBeenCalled() - }) - - it('should pass multiple props correctly', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { token: 'token123' } - - render() - - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - await waitFor(() => { - const calls = vi.mocked(audioToText).mock.calls - expect(calls.length).toBeGreaterThan(0) - const [url, sourceType, formData] = (calls[0] ?? []) as [any, any, any] - expect(url).toBe('/audio-to-text') - expect(sourceType).toBe('webApp') - expect(formData.get('word_timestamps')).toBe('enabled') - }) - }) - - it('should handle legacy explore installed pathname when appId exists', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'test' }) - mockState.params = { appId: 'app-id-123' } - mockState.pathname = '/explore/installed/app-details' - - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) - - await waitFor(() => { - expect(audioToText).toHaveBeenCalledWith( - '/installed-apps/app-id-123/audio-to-text', - 'installedApp', - expect.any(FormData), + render( + + + , ) + resolveSecondRecorder({ ...recorder, cancel: activeCancel }) + expect(await screen.findByText('common.voiceInput.speaking')).toBeInTheDocument() + + resolveFirstRecorder({ ...recorder, cancel: staleCancel }) + + await waitFor(() => expect(staleCancel).toHaveBeenCalledTimes(1)) + expect(activeCancel).not.toHaveBeenCalled() + }) + + it('should keep the active recorder when error callbacks change', async () => { + const { props, rerender } = renderVoiceInput() + await screen.findByText('common.voiceInput.speaking') + + rerender() + + expect(startVoiceRecorder).toHaveBeenCalledTimes(1) + expect(recorderCancel).not.toHaveBeenCalled() + expect(screen.getByRole('button', { name: 'common.voiceInput.stop' })).toBeInTheDocument() }) }) - it('should render timer with initial 00:00 value', () => { - render() - const timer = screen.getByTestId('voice-input-timer') - expect(timer)!.toHaveTextContent('00:00') - }) + // Stopping must upload a real MP3 file to the explicit owner-provided target. + describe('Conversion', () => { + it('should upload MP3 bytes to the explicit app target', async () => { + const { props } = renderVoiceInput() + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) - it('should render stop button during recording', async () => { - render() - expect(await screen.findByTestId('voice-input-stop'))!.toBeInTheDocument() - }) + expect(await screen.findByRole('status')).toHaveTextContent('common.voiceInput.converting') + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + const [requestTarget, file] = vi.mocked(transcribeAudio).mock.calls[0]! + expect(requestTarget).toBe(target) + expect(file.name).toBe('temp.mp3') + expect(file.type).toBe('audio/mp3') + expect(props.onBeforeTranscribe).toHaveBeenCalledTimes(1) + expect(vi.mocked(props.onBeforeTranscribe!).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(transcribeAudio).mock.invocationCallOrder[0]!, + ) + await waitFor(() => expect(props.onConverted).toHaveBeenCalledWith('transcribed text')) + expect(props.onCancel).toHaveBeenCalledTimes(1) + }) - it('should render converting UI after stopping', async () => { - const user = userEvent.setup() - vi.mocked(audioToText).mockImplementation(() => new Promise(() => {})) - mockState.params = { token: 'abc' } + it('should ignore repeated stop requests', async () => { + let resolveStop: (blob: Blob) => void = () => {} + recorderStop.mockReturnValueOnce( + new Promise((resolve) => { + resolveStop = resolve + }), + ) + renderVoiceInput() + const stopButton = await screen.findByRole('button', { name: 'common.voiceInput.stop' }) - render() - const stopBtn = await screen.findByTestId('voice-input-stop') - await user.click(stopBtn) + fireEvent.click(stopButton) + fireEvent.click(stopButton) - await screen.findByTestId('voice-input-loader') - expect(screen.getByTestId('voice-input-converting-text'))!.toBeInTheDocument() - expect(screen.getByTestId('voice-input-cancel'))!.toBeInTheDocument() - }) + expect(recorderStop).toHaveBeenCalledTimes(1) + resolveStop(new Blob(['mp3-data'], { type: 'audio/mp3' })) + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + }) - it('should auto-stop recording and convert audio when duration reaches 10 minutes (600s)', async () => { - vi.mocked(audioToText).mockResolvedValueOnce({ text: 'auto-stopped text' }) - mockState.params = { token: 'abc' } + it('should report conversion failure without replacing the current text', async () => { + vi.mocked(transcribeAudio).mockRejectedValueOnce(new Error('API error')) + const { props } = renderVoiceInput() + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) - render() - expect(await screen.findByTestId('voice-input-stop'))!.toBeInTheDocument() + await waitFor(() => expect(props.onError).toHaveBeenCalledTimes(1)) + expect(props.onConverted).not.toHaveBeenCalled() + expect(props.onCancel).toHaveBeenCalledTimes(1) + }) - for (let i = 0; i < 601; i++) { - await act(async () => { - if (rafState.callback) rafState.callback() + it('should not upload when saving the owning draft fails', async () => { + const onBeforeTranscribe = vi.fn().mockRejectedValue(new Error('save failed')) + const { props } = renderVoiceInput({ onBeforeTranscribe }) + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + + await waitFor(() => expect(props.onCancel).toHaveBeenCalledTimes(1)) + expect(transcribeAudio).not.toHaveBeenCalled() + expect(props.onConverted).not.toHaveBeenCalled() + expect(props.onError).not.toHaveBeenCalled() + }) + + it('should close without publishing late results when conversion is cancelled', async () => { + let requestSignal: AbortSignal | undefined + vi.mocked(transcribeAudio).mockImplementationOnce((_target, _file, signal) => { + requestSignal = signal + return new Promise(() => {}) }) - } + const { props } = renderVoiceInput() + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + await waitFor(() => expect(transcribeAudio).toHaveBeenCalledTimes(1)) + fireEvent.click(await screen.findByRole('button', { name: 'common.operation.cancel' })) - expect(await screen.findByTestId('voice-input-converting-text'))!.toBeInTheDocument() - await waitFor(() => { - expect(onConverted).toHaveBeenCalledWith('auto-stopped text') + expect(recorderCancel).toHaveBeenCalledTimes(1) + expect(requestSignal?.aborted).toBe(true) + expect(props.onCancel).toHaveBeenCalledTimes(1) + expect(props.onConverted).not.toHaveBeenCalled() }) - }, 10000) - it('should handle null canvas element gracefully during initialization', async () => { - const getElementByIdMock = vi.spyOn(document, 'getElementById').mockReturnValue(null) + it('should not upload when conversion is cancelled while the recorder is stopping', async () => { + let resolveStop: (blob: Blob) => void = () => {} + recorderStop.mockReturnValueOnce( + new Promise((resolve) => { + resolveStop = resolve + }), + ) + const { props } = renderVoiceInput() - const { unmount } = render() - await screen.findByTestId('voice-input-stop') + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + fireEvent.click(await screen.findByRole('button', { name: 'common.operation.cancel' })) + resolveStop(new Blob(['mp3-data'], { type: 'audio/mp3' })) - unmount() + await act(async () => {}) + expect(transcribeAudio).not.toHaveBeenCalled() + expect(props.onBeforeTranscribe).not.toHaveBeenCalled() + expect(props.onCancel).toHaveBeenCalledTimes(1) + expect(props.onError).not.toHaveBeenCalled() + }) - getElementByIdMock.mockRestore() + it('should not upload when conversion is cancelled while the draft is saving', async () => { + let resolveDraftSave: () => void = () => {} + const onBeforeTranscribe = vi.fn( + () => + new Promise((resolve) => { + resolveDraftSave = resolve + }), + ) + const { props } = renderVoiceInput({ onBeforeTranscribe }) + + fireEvent.click(await screen.findByRole('button', { name: 'common.voiceInput.stop' })) + await waitFor(() => expect(onBeforeTranscribe).toHaveBeenCalledTimes(1)) + fireEvent.click(screen.getByRole('button', { name: 'common.operation.cancel' })) + resolveDraftSave() + + await act(async () => {}) + expect(transcribeAudio).not.toHaveBeenCalled() + expect(props.onCancel).toHaveBeenCalledTimes(1) + expect(props.onError).not.toHaveBeenCalled() + }) }) - it('should handle getContext returning null gracefully during initialization', async () => { - const oldGetContext = HTMLCanvasElement.prototype.getContext - HTMLCanvasElement.prototype.getContext = vi.fn().mockReturnValue(null) + // The ten-minute limit is a local timer transition, not a render-time side effect. + describe('Duration limit', () => { + beforeEach(() => { + vi.useFakeTimers() + }) - const { unmount } = render() - await screen.findByTestId('voice-input-stop') + afterEach(() => { + vi.runOnlyPendingTimers() + vi.useRealTimers() + }) - unmount() + it('should update the timer while recording', async () => { + renderVoiceInput() + await act(async () => {}) - HTMLCanvasElement.prototype.getContext = oldGetContext + act(() => vi.advanceTimersByTime(1000)) + + expect(screen.getByTestId('voice-input-timer')).toHaveTextContent('00:01') + }) + + it('should stop automatically at ten minutes', async () => { + renderVoiceInput() + await act(async () => {}) + + act(() => vi.advanceTimersByTime(600_000)) + await act(async () => {}) + + expect(recorderStop).toHaveBeenCalledTimes(1) + expect(screen.getByTestId('voice-input-timer')).toHaveTextContent('10:00') + }) }) }) diff --git a/web/app/components/base/voice-input/__tests__/recorder.spec.ts b/web/app/components/base/voice-input/__tests__/recorder.spec.ts new file mode 100644 index 00000000000..40c3e473ab8 --- /dev/null +++ b/web/app/components/base/voice-input/__tests__/recorder.spec.ts @@ -0,0 +1,442 @@ +import { startVoiceRecorder } from '../recorder' + +const mediaMocks = vi.hoisted(() => ({ + audioSourceAdd: vi.fn<(buffer: AudioBuffer) => Promise>(), + audioSourceConfig: undefined as unknown, + outputCancel: vi.fn<() => Promise>(), + outputFinalize: vi.fn<() => Promise>(), + outputStart: vi.fn<() => Promise>(), + registerMp3Encoder: vi.fn(), + target: undefined as { buffer: ArrayBuffer | null } | undefined, +})) + +vi.mock('@mediabunny/mp3-encoder', () => ({ + registerMp3Encoder: mediaMocks.registerMp3Encoder, +})) + +vi.mock('mediabunny', () => ({ + AudioBufferSource: class MockAudioBufferSource { + constructor(config: unknown) { + mediaMocks.audioSourceConfig = config + } + + add(buffer: AudioBuffer) { + return mediaMocks.audioSourceAdd(buffer) + } + }, + BufferTarget: class MockBufferTarget { + buffer: ArrayBuffer | null = null + + constructor() { + mediaMocks.target = this + } + }, + Mp3OutputFormat: class MockMp3OutputFormat {}, + Output: class MockOutput { + addAudioTrack = vi.fn() + + start() { + return mediaMocks.outputStart() + } + + finalize() { + return mediaMocks.outputFinalize() + } + + cancel() { + return mediaMocks.outputCancel() + } + }, +})) + +const trackStop = vi.fn() +const stream = { + getAudioTracks: vi.fn(() => [{ stop: trackStop }]), + getTracks: vi.fn(() => [{ stop: trackStop }]), +} as unknown as MediaStream +const getUserMedia = vi.fn<() => Promise>() +const addModule = vi.fn<() => Promise>() +const audioContextClose = vi.fn<() => Promise>() +const audioContextResume = vi.fn<() => Promise>() +const analyserDisconnect = vi.fn() +const analyser = { + disconnect: analyserDisconnect, + fftSize: 0, +} as unknown as AnalyserNode +const streamSourceConnect = vi.fn() +const streamSourceDisconnect = vi.fn() +const streamSource = { + connect: streamSourceConnect, + disconnect: streamSourceDisconnect, +} as unknown as MediaStreamAudioSourceNode +const copyToChannel = vi.fn() +let audioContextState: AudioContextState = 'running' + +class MockAudioContext { + audioWorklet = { addModule } + destination = {} + sampleRate = 48_000 + + get state() { + return audioContextState + } + + createAnalyser() { + return analyser + } + + createMediaStreamSource() { + return streamSource + } + + createBuffer() { + return { copyToChannel } as unknown as AudioBuffer + } + + close() { + return audioContextClose() + } + + resume() { + audioContextState = 'running' + return audioContextResume() + } +} + +const workletConnect = vi.fn() +const workletDisconnect = vi.fn() +let workletRespondsToStop = true + +class MockAudioWorkletNode { + port: { + onmessage: + | ((event: MessageEvent<{ type: 'data'; buffer: ArrayBuffer } | { type: 'stopped' }>) => void) + | null + postMessage: (message: { type: string }) => void + } + + constructor() { + this.port = { + onmessage: null, + postMessage: vi.fn((message: { type: string }) => { + if (message.type !== 'stop' || !workletRespondsToStop) return + this.port.onmessage?.({ + data: { type: 'data', buffer: new Float32Array([0.25, -0.25]).buffer }, + } as MessageEvent<{ type: 'data'; buffer: ArrayBuffer }>) + this.port.onmessage?.({ data: { type: 'stopped' } } as MessageEvent<{ type: 'stopped' }>) + }), + } + } + + connect = workletConnect + disconnect = workletDisconnect +} + +describe('startVoiceRecorder', () => { + beforeEach(() => { + vi.clearAllMocks() + mediaMocks.target = undefined + mediaMocks.audioSourceConfig = undefined + getUserMedia.mockResolvedValue(stream) + addModule.mockResolvedValue() + audioContextClose.mockResolvedValue() + audioContextResume.mockResolvedValue() + mediaMocks.audioSourceAdd.mockResolvedValue() + mediaMocks.outputCancel.mockResolvedValue() + mediaMocks.outputStart.mockResolvedValue() + mediaMocks.outputFinalize.mockImplementation(async () => { + if (mediaMocks.target) mediaMocks.target.buffer = new Uint8Array([1, 2, 3]).buffer + }) + workletRespondsToStop = true + audioContextState = 'running' + vi.stubGlobal('AudioContext', MockAudioContext) + vi.stubGlobal('AudioWorkletNode', MockAudioWorkletNode) + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia }, + }) + vi.spyOn(URL, 'createObjectURL').mockReturnValue('blob:voice-worklet') + vi.spyOn(URL, 'revokeObjectURL').mockImplementation(() => {}) + }) + + // Recording setup uses one browser-native PCM path and one MP3 encoder configuration. + describe('Setup', () => { + it('should initialize mono microphone capture and the MP3 encoder', async () => { + await startVoiceRecorder() + + expect(getUserMedia).toHaveBeenCalledWith({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + }, + }) + expect(mediaMocks.registerMp3Encoder).toHaveBeenCalledTimes(1) + expect(mediaMocks.audioSourceConfig).toEqual({ + bitrate: 128_000, + codec: 'mp3', + transform: { + numberOfChannels: 1, + sampleRate: 16_000, + }, + }) + expect(addModule).toHaveBeenCalledWith('blob:voice-worklet') + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:voice-worklet') + }) + + it('should resume a suspended audio context before recording', async () => { + audioContextState = 'suspended' + + await startVoiceRecorder() + + expect(audioContextResume).toHaveBeenCalledTimes(1) + }) + + it('should release a microphone stream that resolves after setup is cancelled', async () => { + let resolveStream: (stream: MediaStream) => void = () => {} + getUserMedia.mockReturnValueOnce( + new Promise((resolve) => { + resolveStream = resolve + }), + ) + const abortController = new AbortController() + const recorderPromise = startVoiceRecorder(abortController.signal) + await vi.waitFor(() => expect(getUserMedia).toHaveBeenCalledTimes(1)) + + abortController.abort() + resolveStream(stream) + + await expect(recorderPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(trackStop).toHaveBeenCalledTimes(1) + expect(addModule).not.toHaveBeenCalled() + }) + + it('should abort and clean up while the worklet module is still loading', async () => { + addModule.mockReturnValueOnce(new Promise(() => {})) + const abortController = new AbortController() + const recorderPromise = startVoiceRecorder(abortController.signal) + await vi.waitFor(() => expect(addModule).toHaveBeenCalledTimes(1)) + + abortController.abort() + + await expect(recorderPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:voice-worklet') + }) + + it('should abort and clean up while the encoder is still starting', async () => { + mediaMocks.outputStart.mockReturnValueOnce(new Promise(() => {})) + const abortController = new AbortController() + const recorderPromise = startVoiceRecorder(abortController.signal) + await vi.waitFor(() => expect(mediaMocks.outputStart).toHaveBeenCalledTimes(1)) + + abortController.abort() + + await expect(recorderPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + + it('should abort and clean up while the audio context is still resuming', async () => { + audioContextState = 'suspended' + audioContextResume.mockReturnValueOnce(new Promise(() => {})) + const abortController = new AbortController() + const recorderPromise = startVoiceRecorder(abortController.signal) + await vi.waitFor(() => expect(audioContextResume).toHaveBeenCalledTimes(1)) + + abortController.abort() + + await expect(recorderPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + }) + + // Stopping flushes queued PCM before exposing the final MP3 blob. + describe('Stop', () => { + it('should encode queued PCM and release microphone resources', async () => { + const recorder = await startVoiceRecorder() + + const result = await recorder.stop() + + expect(copyToChannel).toHaveBeenCalledTimes(1) + expect(mediaMocks.audioSourceAdd).toHaveBeenCalledTimes(1) + expect(mediaMocks.outputFinalize).toHaveBeenCalledTimes(1) + expect(result.type).toBe('audio/mp3') + expect(result.size).toBe(3) + expect(trackStop).toHaveBeenCalled() + expect(audioContextClose).toHaveBeenCalledTimes(1) + }) + + it('should release microphone resources before MP3 finalization completes', async () => { + let resolveFinalize: () => void = () => {} + mediaMocks.outputFinalize.mockReturnValueOnce( + new Promise((resolve) => { + resolveFinalize = () => { + if (mediaMocks.target) mediaMocks.target.buffer = new Uint8Array([1, 2, 3]).buffer + resolve() + } + }), + ) + const recorder = await startVoiceRecorder() + + const stopPromise = recorder.stop() + await vi.waitFor(() => expect(mediaMocks.outputFinalize).toHaveBeenCalledTimes(1)) + const trackStopCallsBeforeFinalize = trackStop.mock.calls.length + const audioContextCloseCallsBeforeFinalize = audioContextClose.mock.calls.length + resolveFinalize() + await stopPromise + + expect(trackStopCallsBeforeFinalize).toBe(1) + expect(audioContextCloseCallsBeforeFinalize).toBe(1) + }) + + it('should reuse the in-flight stop operation', async () => { + const recorder = await startVoiceRecorder() + + const firstStop = recorder.stop() + const secondStop = recorder.stop() + + await expect(secondStop).resolves.toBe(await firstStop) + expect(mediaMocks.outputFinalize).toHaveBeenCalledTimes(1) + }) + + it('should reject when the encoder produces no MP3 bytes', async () => { + mediaMocks.outputFinalize.mockResolvedValueOnce() + const recorder = await startVoiceRecorder() + + await expect(recorder.stop()).rejects.toThrow('produced no audio data') + }) + + it('should cancel output when queued PCM encoding fails', async () => { + mediaMocks.audioSourceAdd.mockRejectedValueOnce(new Error('encode failed')) + const recorder = await startVoiceRecorder() + + await expect(recorder.stop()).rejects.toThrow('encode failed') + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + }) + + // Cancellation and setup failures must release the microphone without uploading data. + describe('Cleanup', () => { + it('should cancel encoding and release resources', async () => { + const recorder = await startVoiceRecorder() + + await recorder.cancel() + + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + expect(trackStop).toHaveBeenCalled() + expect(audioContextClose).toHaveBeenCalledTimes(1) + }) + + it('should release the stream when no audio track is available', async () => { + const emptyStream = { + getAudioTracks: () => [], + getTracks: () => [{ stop: trackStop }], + } as unknown as MediaStream + getUserMedia.mockResolvedValueOnce(emptyStream) + + await expect(startVoiceRecorder()).rejects.toThrow('No audio track') + expect(trackStop).toHaveBeenCalledTimes(1) + }) + + it('should cancel partial output when encoder startup fails', async () => { + mediaMocks.outputStart.mockRejectedValueOnce(new Error('encoder unavailable')) + + await expect(startVoiceRecorder()).rejects.toThrow('encoder unavailable') + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + expect(trackStop).toHaveBeenCalled() + expect(audioContextClose).toHaveBeenCalledTimes(1) + }) + + it('should release the microphone before canceling a failed encoder setup', async () => { + mediaMocks.outputStart.mockRejectedValueOnce(new Error('encoder unavailable')) + mediaMocks.outputCancel.mockRejectedValueOnce(new Error('encoder cleanup failed')) + + await expect(startVoiceRecorder()).rejects.toThrow() + + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + }) + + it('should release capture while an in-flight stop finishes encoding', async () => { + let resolveFinalize: () => void = () => {} + mediaMocks.outputFinalize.mockReturnValueOnce( + new Promise((resolve) => { + resolveFinalize = () => { + if (mediaMocks.target) mediaMocks.target.buffer = new Uint8Array([1, 2, 3]).buffer + resolve() + } + }), + ) + const recorder = await startVoiceRecorder() + const stopPromise = recorder.stop() + await vi.waitFor(() => expect(mediaMocks.outputFinalize).toHaveBeenCalledTimes(1)) + + const cancelPromise = recorder.cancel() + await vi.waitFor(() => expect(trackStop).toHaveBeenCalledTimes(1)) + resolveFinalize() + + await cancelPromise + await expect(stopPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + + it('should cancel capture when the worklet stop acknowledgement is pending', async () => { + workletRespondsToStop = false + const recorder = await startVoiceRecorder() + const stopPromise = recorder.stop() + + await recorder.cancel() + + await expect(stopPromise).rejects.toMatchObject({ name: 'AbortError' }) + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + + it('should complete runtime cleanup when a node disconnect fails', async () => { + streamSourceDisconnect.mockImplementationOnce(() => { + throw new Error('disconnect failed') + }) + const recorder = await startVoiceRecorder() + + await expect(recorder.cancel()).resolves.toBeUndefined() + + expect(trackStop).toHaveBeenCalledTimes(1) + expect(analyserDisconnect).toHaveBeenCalledTimes(1) + expect(workletDisconnect).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + expect(mediaMocks.outputCancel).toHaveBeenCalledTimes(1) + }) + + it('should resolve cancellation when encoder cleanup fails', async () => { + mediaMocks.outputCancel.mockRejectedValueOnce(new Error('encoder cleanup failed')) + const recorder = await startVoiceRecorder() + + await expect(recorder.cancel()).resolves.toBeUndefined() + + expect(trackStop).toHaveBeenCalledTimes(1) + expect(audioContextClose).toHaveBeenCalledTimes(1) + }) + + it('should not open the microphone when the encoder module fails to load', async () => { + vi.resetModules() + vi.doMock('../mp3-encoder', () => { + throw new Error('encoder chunk unavailable') + }) + + try { + const { startVoiceRecorder: startWithUnavailableEncoder } = await import('../recorder') + + await expect(startWithUnavailableEncoder()).rejects.toThrow() + expect(getUserMedia).not.toHaveBeenCalled() + } finally { + vi.doUnmock('../mp3-encoder') + } + }) + }) +}) diff --git a/web/app/components/base/voice-input/__tests__/utils.spec.ts b/web/app/components/base/voice-input/__tests__/utils.spec.ts deleted file mode 100644 index 390efaa046d..00000000000 --- a/web/app/components/base/voice-input/__tests__/utils.spec.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { convertToMp3 } from '../utils' - -// ── Hoisted mocks ── - -const mocks = vi.hoisted(() => { - const readHeader = vi.fn() - const encodeBuffer = vi.fn() - const flush = vi.fn() - - return { readHeader, encodeBuffer, flush } -}) - -vi.mock('lamejs', () => ({ - default: { - WavHeader: { - readHeader: mocks.readHeader, - }, - Mp3Encoder: class MockMp3Encoder { - encodeBuffer = mocks.encodeBuffer - flush = mocks.flush - }, - }, -})) - -vi.mock('lamejs/src/js/BitStream', () => ({ default: {} })) -vi.mock('lamejs/src/js/Lame', () => ({ default: {} })) -vi.mock('lamejs/src/js/MPEGMode', () => ({ default: {} })) - -// ── helpers ── - -/** Build a fake recorder whose getChannelData returns DataView-like objects with .buffer and .byteLength. */ -function createMockRecorder(opts: { - channels: number - sampleRate: number - leftSamples: number[] - rightSamples?: number[] -}) { - const toDataView = (samples: number[]) => { - const buf = new ArrayBuffer(samples.length * 2) - const view = new DataView(buf) - samples.forEach((v, i) => { - view.setInt16(i * 2, v, true) - }) - return view - } - - const leftView = toDataView(opts.leftSamples) - const rightView = opts.rightSamples ? toDataView(opts.rightSamples) : null - - mocks.readHeader.mockReturnValue({ - channels: opts.channels, - sampleRate: opts.sampleRate, - }) - - return { - getWAV: vi.fn(() => new ArrayBuffer(44)), - getChannelData: vi.fn(() => ({ - left: leftView, - right: rightView, - })), - } -} - -describe('convertToMp3', () => { - beforeEach(() => { - vi.clearAllMocks() - }) - - it('should convert mono WAV data to an MP3 blob', () => { - const recorder = createMockRecorder({ - channels: 1, - sampleRate: 44100, - leftSamples: [100, 200, 300, 400], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array([1, 2, 3])) - mocks.flush.mockReturnValue(new Int8Array([4, 5])) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - expect(result.type).toBe('audio/mp3') - expect(mocks.encodeBuffer).toHaveBeenCalled() - // Mono: encodeBuffer called with only left data - const firstCall = mocks.encodeBuffer.mock.calls[0] - expect(firstCall).toHaveLength(1) - expect(mocks.flush).toHaveBeenCalled() - }) - - it('should convert stereo WAV data to an MP3 blob', () => { - const recorder = createMockRecorder({ - channels: 2, - sampleRate: 48000, - leftSamples: [100, 200], - rightSamples: [300, 400], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array([10, 20])) - mocks.flush.mockReturnValue(new Int8Array([30])) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - expect(result.type).toBe('audio/mp3') - // Stereo: encodeBuffer called with left AND right - const firstCall = mocks.encodeBuffer.mock.calls[0] - expect(firstCall).toHaveLength(2) - }) - - it('should skip empty encoded buffers', () => { - const recorder = createMockRecorder({ - channels: 1, - sampleRate: 44100, - leftSamples: [100, 200], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array(0)) - mocks.flush.mockReturnValue(new Int8Array(0)) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - expect(result.type).toBe('audio/mp3') - expect(result.size).toBe(0) - }) - - it('should include flush data when flush returns non-empty buffer', () => { - const recorder = createMockRecorder({ - channels: 1, - sampleRate: 22050, - leftSamples: [1], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array(0)) - mocks.flush.mockReturnValue(new Int8Array([99, 98, 97])) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - expect(result.size).toBe(3) - }) - - it('should omit flush data when flush returns empty buffer', () => { - const recorder = createMockRecorder({ - channels: 1, - sampleRate: 44100, - leftSamples: [10, 20], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array([1, 2])) - mocks.flush.mockReturnValue(new Int8Array(0)) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - expect(result.size).toBe(2) - }) - - it('should process multiple chunks when sample count exceeds maxSamples (1152)', () => { - const samples = Array.from({ length: 2400 }, (_, i) => i % 32767) - const recorder = createMockRecorder({ - channels: 1, - sampleRate: 44100, - leftSamples: samples, - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array([1])) - mocks.flush.mockReturnValue(new Int8Array(0)) - - const result = convertToMp3(recorder) - - expect(mocks.encodeBuffer.mock.calls.length).toBeGreaterThan(1) - expect(result).toBeInstanceOf(Blob) - }) - - it('should encode stereo with right channel subarray', () => { - const recorder = createMockRecorder({ - channels: 2, - sampleRate: 44100, - leftSamples: [100, 200, 300], - rightSamples: [400, 500, 600], - }) - - mocks.encodeBuffer.mockReturnValue(new Int8Array([5, 6, 7])) - mocks.flush.mockReturnValue(new Int8Array([8])) - - const result = convertToMp3(recorder) - - expect(result).toBeInstanceOf(Blob) - for (const call of mocks.encodeBuffer.mock.calls) { - expect(call).toHaveLength(2) - expect(call[0]).toBeInstanceOf(Int16Array) - expect(call[1]).toBeInstanceOf(Int16Array) - } - }) -}) diff --git a/web/app/components/base/voice-input/api.ts b/web/app/components/base/voice-input/api.ts new file mode 100644 index 00000000000..13e31eefdba --- /dev/null +++ b/web/app/components/base/voice-input/api.ts @@ -0,0 +1,37 @@ +import type { SpeechToTextTarget } from './types' +import { audioToText } from '@/service/share' + +export async function transcribeAudio( + target: SpeechToTextTarget, + file: File, + signal?: AbortSignal, +): Promise<{ text: string }> { + if (target.type === 'agent' || target.type === 'consoleApp') { + const { consoleClient } = await import('@/service/client') + if (target.type === 'agent') { + return consoleClient.agent.byAgentId.audioToText.post( + { + body: { + draft_type: target.draftType, + file, + }, + params: { + agent_id: target.agentId, + }, + }, + { signal }, + ) + } + return consoleClient.apps.byAppId.audioToText.post( + { + body: { file }, + params: { app_id: target.appId }, + }, + { signal }, + ) + } + + const formData = new FormData() + formData.append('file', file) + return audioToText(target.appSourceType, target.appId, formData, signal) +} diff --git a/web/app/components/base/voice-input/index.stories.tsx b/web/app/components/base/voice-input/index.stories.tsx index 0273aad1ba3..a38aa6d8e31 100644 --- a/web/app/components/base/voice-input/index.stories.tsx +++ b/web/app/components/base/voice-input/index.stories.tsx @@ -88,7 +88,7 @@ const meta = { docs: { description: { component: - 'Voice input component for recording audio and converting speech to text. Features waveform visualization, recording timer (max 10 minutes), and audio-to-text conversion using js-audio-recorder.\n\n**Note:** This is a simplified mock for Storybook. The actual component requires microphone permissions and audio-to-text API.', + 'Voice input component for recording audio and converting speech to text. Features waveform visualization, recording timer (max 10 minutes), and browser-side MP3 encoding.\n\n**Note:** This is a simplified mock for Storybook. The actual component requires microphone permissions and audio-to-text API.', }, }, }, @@ -451,7 +451,7 @@ export const FeaturesShowcase: Story = {
🎤 Audio Recording
    -
  • • Uses js-audio-recorder for browser-based recording
    • +
  • • Uses AudioWorklet for browser-based recording
  • • 16kHz sample rate, 16-bit, mono channel
  • • Converts to MP3 format for transmission
diff --git a/web/app/components/base/voice-input/index.tsx b/web/app/components/base/voice-input/index.tsx index 663d9e0b4c4..a96e0f862da 100644 --- a/web/app/components/base/voice-input/index.tsx +++ b/web/app/components/base/voice-input/index.tsx @@ -1,200 +1,267 @@ +import type { Ref } from 'react' +import type { VoiceRecorder } from './recorder' +import type { SpeechToTextTarget } from './types' import { cn } from '@langgenius/dify-ui/cn' -import { useRafInterval } from 'ahooks' -import Recorder from 'js-audio-recorder' -import { useCallback, useEffect, useRef, useState } from 'react' +import { useCallback, useEffect, useEffectEvent, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' -import { isInstalledAppPath } from '@/app/components/explore/installed-app/routes' -import { useParams, usePathname } from '@/next/navigation' -import { AppSourceType, audioToText } from '@/service/share' +import { transcribeAudio } from './api' import s from './index.module.css' -import { convertToMp3 } from './utils' +import { startVoiceRecorder } from './recorder' -type VoiceInputTypes = { +const MAX_RECORDING_DURATION = 600 + +type VoiceInputStatus = 'starting' | 'recording' | 'converting' + +type VoiceInputProps = { + ref?: Ref onConverted: (text: string) => void onCancel: () => void - wordTimestamps?: string + onBeforeTranscribe?: () => Promise + onError?: () => void + onStartError?: (error: unknown) => void + target: SpeechToTextTarget } -const VoiceInput = ({ onCancel, onConverted, wordTimestamps }: VoiceInputTypes) => { +function VoiceInput({ + ref, + onCancel, + onBeforeTranscribe, + onConverted, + onError, + onStartError, + target, +}: VoiceInputProps) { const { t } = useTranslation() - const recorder = useRef( - new Recorder({ - sampleBits: 16, - sampleRate: 16000, - numChannels: 1, - compiling: false, - }), - ) + const recorderRef = useRef(null) const canvasRef = useRef(null) - const ctxRef = useRef(null) - const drawRecordId = useRef(null) - const [originDuration, setOriginDuration] = useState(0) - const [startRecord, setStartRecord] = useState(false) - const [startConvert, setStartConvert] = useState(false) - const pathname = usePathname() - const params = useParams() - const clearInterval = useRafInterval(() => { - setOriginDuration(originDuration + 1) - }, 1000) + const canvasContextRef = useRef(null) + const canvasSizeRef = useRef({ height: 0, width: 0 }) + const drawRecordIdRef = useRef(null) + const mountedRef = useRef(true) + const stopRequestedRef = useRef(false) + const cancelledRef = useRef(false) + const setupAbortControllerRef = useRef(null) + const transcriptionAbortControllerRef = useRef(null) + const [duration, setDuration] = useState(0) + const [status, setStatus] = useState('starting') + const handleRecorderStartError = useEffectEvent((error: unknown) => { + onStartError?.(error) + onCancel() + }) + + const stopDrawing = useCallback(() => { + if (drawRecordIdRef.current !== null) cancelAnimationFrame(drawRecordIdRef.current) + drawRecordIdRef.current = null + const { height, width } = canvasSizeRef.current + canvasContextRef.current?.clearRect(0, 0, width, height) + }, []) const drawRecord = useCallback(() => { - drawRecordId.current = requestAnimationFrame(drawRecord) - const canvas = canvasRef.current! - const ctx = ctxRef.current! - const dataUnit8Array = recorder.current.getRecordAnalyseData() - const dataArray = [].slice.call(dataUnit8Array) - const lineLength = Number.parseInt(`${canvas.width / 3}`) - const gap = Number.parseInt(`${1024 / lineLength}`) + const recorder = recorderRef.current + const context = canvasContextRef.current + const { height, width } = canvasSizeRef.current + if (!recorder || !context || !height || !width) return - ctx.clearRect(0, 0, canvas.width, canvas.height) - ctx.beginPath() - let x = 0 - for (let i = 0; i < lineLength; i++) { - let v = - dataArray.slice(i * gap, i * gap + gap).reduce((prev: number, next: number) => { - return prev + next - }, 0) / gap + const frequencyData = new Uint8Array(recorder.analyser.frequencyBinCount) + recorder.analyser.getByteFrequencyData(frequencyData) + const lineCount = Math.max(1, Math.floor(width / 3)) + const sampleStep = Math.max(1, Math.floor(frequencyData.length / lineCount)) - if (v < 128) v = 128 - if (v > 178) v = 178 - const y = ((v - 128) / 50) * canvas.height - - ctx.moveTo(x, 16) - if (ctx.roundRect) ctx.roundRect(x, 16 - y, 2, y, [1, 1, 0, 0]) - else ctx.rect(x, 16 - y, 2, y) - ctx.fill() - x += 3 + context.clearRect(0, 0, width, height) + context.beginPath() + for (let index = 0; index < lineCount; index++) { + const amplitude = frequencyData[index * sampleStep] ?? 0 + const lineHeight = Math.max(1, (amplitude / 255) * height) + const x = index * 3 + if (context.roundRect) context.roundRect(x, height - lineHeight, 2, lineHeight, [1, 1, 0, 0]) + else context.rect(x, height - lineHeight, 2, lineHeight) } - ctx.closePath() + context.fill() + context.closePath() + drawRecordIdRef.current = requestAnimationFrame(drawRecord) }, []) + const handleStopRecorder = useCallback(async () => { - clearInterval() - setStartRecord(false) - setStartConvert(true) - recorder.current.stop() - if (drawRecordId.current) cancelAnimationFrame(drawRecordId.current) - drawRecordId.current = null - const canvas = canvasRef.current! - const ctx = ctxRef.current! - ctx.clearRect(0, 0, canvas.width, canvas.height) - const mp3Blob = convertToMp3(recorder.current) - const mp3File = new File([mp3Blob], 'temp.mp3', { type: 'audio/mp3' }) - const formData = new FormData() - formData.append('file', mp3File) - formData.append('word_timestamps', wordTimestamps || 'disabled') - - let url = '' - let isPublic = false - - if (params.token) { - url = '/audio-to-text' - isPublic = true - } else if (params.appId) { - if (isInstalledAppPath(pathname)) url = `/installed-apps/${params.appId}/audio-to-text` - else url = `/apps/${params.appId}/audio-to-text` - } + const recorder = recorderRef.current + if (!recorder || status !== 'recording' || stopRequestedRef.current) return + stopRequestedRef.current = true + setStatus('converting') + stopDrawing() + let mp3Blob: Blob try { - const audioResponse = await audioToText( - url, - isPublic ? AppSourceType.webApp : AppSourceType.installedApp, - formData, - ) - onConverted(audioResponse.text) - onCancel() + mp3Blob = await recorder.stop() } catch { - onConverted('') - onCancel() - } - }, [clearInterval, onCancel, onConverted, params.appId, params.token, pathname, wordTimestamps]) - const handleStartRecord = useCallback(async () => { - try { - await recorder.current.start() - setStartRecord(true) - setStartConvert(false) - - if (canvasRef.current && ctxRef.current) drawRecord() - } catch { - onCancel() - } - }, [drawRecord, onCancel, setStartRecord, setStartConvert]) - const initCanvas = useCallback(() => { - const dpr = window.devicePixelRatio || 1 - const canvas = document.getElementById('voice-input-record') as HTMLCanvasElement - - if (canvas) { - const { width: cssWidth, height: cssHeight } = canvas.getBoundingClientRect() - - canvas.width = dpr * cssWidth - canvas.height = dpr * cssHeight - canvasRef.current = canvas - - const ctx = canvas.getContext('2d') - if (ctx) { - ctx.scale(dpr, dpr) - ctx.fillStyle = 'rgba(209, 224, 255, 1)' - ctxRef.current = ctx + if (mountedRef.current && !cancelledRef.current) { + onError?.() + onCancel() } + return } - }, []) - if (originDuration >= 600 && startRecord) handleStopRecorder() + if (cancelledRef.current) return + + try { + await onBeforeTranscribe?.() + } catch { + if (mountedRef.current && !cancelledRef.current) onCancel() + return + } + if (cancelledRef.current) return + + const file = new File([mp3Blob], 'temp.mp3', { type: 'audio/mp3' }) + const abortController = new AbortController() + transcriptionAbortControllerRef.current = abortController + try { + const audioResponse = await transcribeAudio(target, file, abortController.signal) + if (mountedRef.current && !cancelledRef.current) onConverted(audioResponse.text) + } catch { + if (mountedRef.current && !cancelledRef.current) onError?.() + } finally { + transcriptionAbortControllerRef.current = null + if (mountedRef.current && !cancelledRef.current) onCancel() + } + }, [onBeforeTranscribe, onCancel, onConverted, onError, status, stopDrawing, target]) + + const handleCancel = useCallback(() => { + cancelledRef.current = true + setupAbortControllerRef.current?.abort() + transcriptionAbortControllerRef.current?.abort() + void recorderRef.current?.cancel() + onCancel() + }, [onCancel]) useEffect(() => { - initCanvas() - handleStartRecord() - const recorderRef = recorder?.current - return () => { - recorderRef?.stop() - } - }, [handleStartRecord, initCanvas]) + const canvas = canvasRef.current + if (!canvas) return + const devicePixelRatio = window.devicePixelRatio || 1 + const { height, width } = canvas.getBoundingClientRect() + canvas.width = devicePixelRatio * width + canvas.height = devicePixelRatio * height + canvasSizeRef.current = { height, width } + const context = canvas.getContext('2d') + if (!context) return + context.scale(devicePixelRatio, devicePixelRatio) + context.fillStyle = 'rgba(209, 224, 255, 1)' + canvasContextRef.current = context + }, []) - const minutes = Number.parseInt(`${Number.parseInt(`${originDuration}`) / 60}`) - const seconds = Number.parseInt(`${originDuration}`) % 60 + useEffect(() => { + mountedRef.current = true + cancelledRef.current = false + let effectCancelled = false + const abortController = new AbortController() + setupAbortControllerRef.current = abortController + void startVoiceRecorder(abortController.signal) + .then((recorder) => { + if (setupAbortControllerRef.current === abortController) + setupAbortControllerRef.current = null + if (effectCancelled) { + void recorder.cancel() + return + } + recorderRef.current = recorder + setStatus('recording') + drawRecord() + }) + .catch((error: unknown) => { + if (setupAbortControllerRef.current === abortController) + setupAbortControllerRef.current = null + if (effectCancelled || abortController.signal.aborted) return + handleRecorderStartError(error) + }) + + return () => { + effectCancelled = true + mountedRef.current = false + cancelledRef.current = true + abortController.abort() + transcriptionAbortControllerRef.current?.abort() + stopDrawing() + void recorderRef.current?.cancel() + } + }, [drawRecord, stopDrawing]) + + useEffect(() => { + if (status !== 'recording') return + const intervalId = window.setInterval(() => { + setDuration((currentDuration) => currentDuration + 1) + }, 1000) + return () => window.clearInterval(intervalId) + }, [status]) + + useEffect(() => { + if (duration >= MAX_RECORDING_DURATION && status === 'recording') void handleStopRecorder() + }, [duration, handleStopRecorder, status]) + + const minutes = Math.floor(duration / 60) + .toString() + .padStart(2, '0') + const seconds = (duration % 60).toString().padStart(2, '0') + const isStarting = status === 'starting' + const isRecording = status === 'recording' + const isConverting = status === 'converting' return ( -
+
- - {startConvert && ( + + {(isStarting || isConverting) && (