diff --git a/.env.local.example b/.env.local.example deleted file mode 100644 index 6604913..0000000 --- a/.env.local.example +++ /dev/null @@ -1,32 +0,0 @@ -# Supabase -NEXT_PUBLIC_SUPABASE_URL= -NEXT_PUBLIC_SUPABASE_ANON_KEY= -SUPABASE_SERVICE_ROLE_KEY= - -# AI Model (optional — app works in mock mode without this) -DEEPSEEK_API_KEY= -DEEPSEEK_BASE_URL=https://api.deepseek.com - -# Optional: Production STT/TTS providers -# STT_PROVIDER=mock|browser|whisper -# TTS_PROVIDER=mock|xunfei|volcengine|tencent - -# Xunfei TTS (recommended first provider in China) -XUNFEI_APP_ID= -XUNFEI_API_KEY= -XUNFEI_API_SECRET= -# Required when TTS_PROVIDER=xunfei. Use a V3-compatible vcn from the Xunfei console. -XUNFEI_TTS_VOICE= -# Specific Xunfei coach voices are selected in Settings from the app voice catalog. - -# Volcengine TTS -VOLCENGINE_TTS_APP_ID= -VOLCENGINE_TTS_ACCESS_TOKEN= -VOLCENGINE_TTS_CLUSTER=volcano_tts -VOLCENGINE_TTS_VOICE=BV001_streaming - -# Tencent Cloud TTS -TENCENT_SECRET_ID= -TENCENT_SECRET_KEY= -TENCENT_TTS_REGION=ap-guangzhou -TENCENT_TTS_VOICE=101001 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49c15f3..cca62fd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,8 @@ name: CI +permissions: + contents: read + on: pull_request: push: diff --git a/.github/workflows/deploy-tencent.yml b/.github/workflows/deploy-tencent.yml new file mode 100644 index 0000000..9b426b4 --- /dev/null +++ b/.github/workflows/deploy-tencent.yml @@ -0,0 +1,72 @@ +name: Deploy Tencent + +permissions: + contents: read + +on: + push: + branches: + - main + - release + workflow_dispatch: + +concurrency: + group: deploy-tencent-${{ github.ref }} + cancel-in-progress: false + +jobs: + deploy: + name: Deploy Web/API + runs-on: [self-hosted, linux, x64, tencent, meteorvoice] + environment: tencent + steps: + - name: Resolve target + id: target + shell: bash + run: | + if [ "${GITHUB_REF_NAME}" = "release" ]; then + echo "app_dir=/srv/meteorvoice-release" >> "$GITHUB_OUTPUT" + echo "pm2_name=meteorvoice-release" >> "$GITHUB_OUTPUT" + echo "port=3100" >> "$GITHUB_OUTPUT" + else + echo "app_dir=/srv/meteorvoice" >> "$GITHUB_OUTPUT" + echo "pm2_name=meteorvoice" >> "$GITHUB_OUTPUT" + echo "port=3101" >> "$GITHUB_OUTPUT" + fi + + - name: Deploy + shell: bash + run: | + retry() { + local attempts="$1" + local delay="$2" + shift 2 + + local attempt=1 + until "$@"; do + if [ "$attempt" -ge "$attempts" ]; then + return 1 + fi + + echo "Command failed. Retrying in ${delay}s (${attempt}/${attempts})..." + sleep "$delay" + attempt=$((attempt + 1)) + done + } + + cd '${{ steps.target.outputs.app_dir }}' + retry 5 15 timeout 180s git fetch origin "$GITHUB_REF_NAME" + git checkout "$GITHUB_REF_NAME" + git reset --hard "origin/$GITHUB_REF_NAME" + retry 3 15 npm ci + set -a + . /etc/meteorvoice/meteorvoice.env + set +a + retry 2 15 npm run build + unset RUNNER_TRACKING_ID + PORT='${{ steps.target.outputs.port }}' HOSTNAME=127.0.0.1 \ + pm2 restart '${{ steps.target.outputs.pm2_name }}' --update-env || \ + PORT='${{ steps.target.outputs.port }}' HOSTNAME=127.0.0.1 \ + pm2 start npm --name '${{ steps.target.outputs.pm2_name }}' -- run start --workspace @meteorvoice/web -- --hostname 127.0.0.1 --port '${{ steps.target.outputs.port }}' + pm2 save + retry 12 5 curl -fsS 'http://127.0.0.1:${{ steps.target.outputs.port }}/' >/dev/null diff --git a/AGENTS.md b/AGENTS.md index 9ca14fa..64cc5f1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -46,6 +46,8 @@ All spec, plan, and implementation handoff files live in `docs/`: ## Environment -- `.env.local` — local dev credentials (never committed) -- `.env.local.example` — template -- Vercel env vars: `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL` +- `apps/web/.env.local` — Web/API local development credentials (never committed) +- `apps/web/.env.local.example` — Web/API template +- `apps/mobile/.env` — mobile build-time public variables only, such as `EXPO_PUBLIC_SUPABASE_URL` and `EXPO_PUBLIC_SUPABASE_ANON_KEY` +- `/etc/meteorvoice/meteorvoice.env` — Tencent/server runtime environment for the deployed Web/API process +- Vercel env vars mirror the Web/API server environment: `NEXT_PUBLIC_SUPABASE_URL`, `NEXT_PUBLIC_SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL` diff --git a/README.md b/README.md index a1aef60..63bfe75 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

- Next.js + Next.js Expo Supabase AI @@ -72,8 +72,12 @@ sequenceDiagram - Accent adaptation across sessions (American, British, Indian, Australian, and more) - Non-blocking correction panel — corrections appear without interrupting the conversation - Bilingual UI (English and Chinese) outside the conversation area +- Response language routing — AI replies and correction explanations follow the selected UI language - Theme switching with CSS custom properties - Login, session history, and preference sync through Supabase +- Shared ASR provider layer with Xunfei bootstrap, native iOS PCM capture, and provider diagnostics +- Unified runtime feedback for loading, blocking operations, network errors, and 401 sign-out +- Authenticated API guard for high-cost AI, TTS, ASR, and session routes - Mock AI/STT/TTS providers for local development without API keys - Native mobile client (iOS/Android) via Expo React Native @@ -85,16 +89,18 @@ flowchart TB Mobile[MeteorVoice Mobile
Expo React Native] subgraph Platform[Backend] - API[Next.js API Routes
TTS / Chat / Session / Turns] + API[Next.js API Routes
TTS / ASR / Chat / Session / Turns] Supabase[Supabase
Auth / DB / Preferences] AI[AI Provider
DeepSeek via Vercel AI SDK] TTS[TTS Providers
Xunfei / Volcengine / Tencent / Azure] + ASR[ASR Providers
Native / Xunfei / Azure-ready] end subgraph Shared[packages/] SharedPkg[shared
types / i18n / provider interfaces] SessionCore[session-core
turn lifecycle / workflow] APIClient[api-client
typed API calls] + Runtime[shared runtime
feedback / operation groups] end Web --> API @@ -103,17 +109,26 @@ flowchart TB API --> Supabase API --> AI API --> TTS + API --> ASR Web --> Shared Mobile --> Shared + Web --> Runtime + Mobile --> Runtime ``` Responsibility boundaries: - `apps/web`: Next.js full-stack app — UI, API routes, server-side TTS/AI orchestration. -- `apps/mobile`: Expo React Native native client — voice session UI, native audio, background keep-alive. -- `packages/shared`: cross-client types, i18n strings, provider interfaces, TTS capabilities. +- `apps/mobile`: Expo React Native native client — voice session UI, native audio, native PCM capture, background keep-alive. +- `packages/shared`: cross-client types, i18n strings, provider interfaces, ASR/TTS capabilities, app feedback state, and grouped async operation helpers. - `packages/session-core`: platform-neutral turn lifecycle and workflow state machine. -- `packages/api-client`: typed API calls shared by Web and Mobile. +- `packages/api-client`: typed API calls shared by Web and Mobile, including request timeout and formatted error handling. + +Settings synchronization now separates full refresh and targeted updates: + +- Page entry, login, foreground resume, and manual refresh use grouped operations so multiple API requests share one loading state. +- A single setting save uses the returned `/api/preferences` payload and applies only the affected settings domain. +- AI response language is passed as `responseLocale`; ASR recognition language is configured separately. ## Project Structure @@ -171,12 +186,15 @@ SUPABASE_SERVICE_ROLE_KEY=your-service-role-key DEEPSEEK_API_KEY=your-deepseek-api-key # optional — mock AI works without it ``` -TTS provider keys (all optional — mock TTS works without them): +ASR/TTS provider keys (all optional — mock/native fallback works without them): ```text +ASR_PROVIDER=native +TTS_PROVIDER=mock XUNFEI_APP_ID= XUNFEI_API_KEY= XUNFEI_API_SECRET= +XUNFEI_ASR_PRODUCT=zh_iat XUNFEI_TTS_VOICE= # default fallback vcn; coach voice is selectable in Settings VOLCENGINE_ACCESS_KEY= VOLCENGINE_SECRET_KEY= @@ -202,7 +220,7 @@ npm run dev Open `http://127.0.0.1:3001` -The app runs fully in mock mode without any API keys. Real AI replies require `DEEPSEEK_API_KEY`. Real voice output requires at least one TTS provider key. +The app runs fully in mock mode without any API keys. Real AI replies require `DEEPSEEK_API_KEY`. Real voice output requires at least one TTS provider key. Remote ASR requires provider credentials and `ASR_PROVIDER`; otherwise mobile can use native speech recognition. ## Mobile @@ -233,7 +251,7 @@ Set the API base URL in `apps/mobile/app.json`: ```json "extra": { "apiBaseUrl": "https://meteorvoice.jcmeteor.com", - "apiBaseUrlPreview": "https://meteorvoice-pre.jcmeteor.com" + "apiBaseUrlPreview": "https://mv-pre.jcmeteor.com" } ``` @@ -253,6 +271,20 @@ Each user's selected provider is stored in Supabase. Provider credentials stay i See `docs/tts-integration.md` for provider setup details. +## ASR Providers + +MeteorVoice now exposes a shared ASR provider layer: + +| Provider | Key | Status | +|----------|-----|--------| +| Native mobile speech | `native` | Default fallback | +| Xunfei `zh_iat` | `xunfei` | Signed WebSocket bootstrap and iOS PCM streaming path | +| Azure Speech | `azure` | Contract-ready, pending runtime adapter | + +The ASR layer is split from AI response language routing: `ASR languageMode` controls recognition, while `responseLocale` controls how the coach replies. + +See `docs/asr-provider-layer.md` for contracts, rollout steps, diagnostics, and test guidance. + ## Validation ```bash @@ -280,6 +312,7 @@ flowchart LR | **Dual-platform architecture** | `apps/web` + `apps/mobile` + `packages/*` monorepo | ✅ Delivered | | **Immersive UI** | Voice waveform, desktop/mobile layouts, real-time subtitles | ✅ Delivered | | **Productization** | History with expansion/pagination, cross-device preferences sync, CI parallel jobs, mobile audio hardening | ✅ Delivered | -| **Semantic endpointing** | L1 local rules + L2 LLM-based end-of-turn detection + L3 safety net, replacing fixed silence timeout | 🚧 In implementation | -| **Accent capabilities** | TTS provider voice catalog, multi-accent speakers (US/UK/Indian/Australian) | 📋 Planned | +| **Semantic endpointing** | L1 local rules + L2 LLM-based end-of-turn detection + L3 safety net, replacing fixed silence timeout | ✅ Delivered | +| **ASR provider layer** | Shared ASR contracts, Xunfei bootstrap, native PCM diagnostics, mobile session integration | ✅ Delivered | +| **Accent capabilities** | TTS provider voice catalog, multi-accent speakers (US/UK/Indian/Australian) | 🚧 In implementation | | **Distribution** | TestFlight beta, EAS production build | 📋 Planned | diff --git a/README.zh-CN.md b/README.zh-CN.md index 0539a32..0ca61ef 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -5,7 +5,7 @@

- Next.js + Next.js Expo Supabase AI @@ -72,8 +72,12 @@ sequenceDiagram - 会话级口音适配（美式、英式、印度、澳洲等） - 非阻塞纠错面板——纠错建议不打断对话 - 对话区外支持中英双语 UI +- AI 回复语言路由和 ASR 识别语言分离，避免中英文设置互相覆盖 - 基于 CSS 自定义属性的全局主题切换 - 通过 Supabase 管理登录、历史记录和偏好同步 +- 统一 ASR provider 层，支持讯飞启动会话、iOS 原生 PCM 采集和诊断 +- 统一运行时反馈能力，覆盖加载、阻断操作、网络错误和 401 强制登出 +- 高成本 AI、TTS、ASR、会话接口增加登录态保护 - 提供 mock AI/STT/TTS，无需 API Key 即可本地开发 - 基于 Expo React Native 的原生移动端（iOS/Android） @@ -85,16 +89,18 @@ flowchart TB Mobile[MeteorVoice Mobile
Expo React Native] subgraph Platform[后端] - API[Next.js API Routes
TTS / Chat / Session / Turns] + API[Next.js API Routes
TTS / ASR / Chat / Session / Turns] Supabase[Supabase
Auth / DB / 偏好] AI[AI Provider
DeepSeek via Vercel AI SDK] TTS[TTS 服务商
讯飞 / 火山 / 腾讯 / Azure] + ASR[ASR 服务商
Native / 讯飞 / Azure-ready] end subgraph Shared[packages/] SharedPkg[shared
类型 / i18n / provider 接口] SessionCore[session-core
轮次生命周期 / 工作流] APIClient[api-client
类型化 API 调用] + Runtime[shared runtime
反馈 / 操作组] end Web --> API @@ -103,17 +109,26 @@ flowchart TB API --> Supabase API --> AI API --> TTS + API --> ASR Web --> Shared Mobile --> Shared + Web --> Runtime + Mobile --> Runtime ``` 职责边界： - `apps/web`：Next.js 全栈应用——UI、API 路由、服务端 TTS/AI 编排。 -- `apps/mobile`：Expo React Native 原生客户端——语音会话 UI、原生音频、后台保活。 -- `packages/shared`：跨端类型、i18n 文案、provider 接口、TTS 能力表。 +- `apps/mobile`：Expo React Native 原生客户端——语音会话 UI、原生音频、原生 PCM 采集、后台保活。 +- `packages/shared`：跨端类型、i18n 文案、provider 接口、ASR/TTS 能力表、反馈状态和操作组。 - `packages/session-core`：平台无关的轮次生命周期和工作流状态机。 -- `packages/api-client`：Web 和 Mobile 共用的类型化 API 客户端。 +- `packages/api-client`：Web 和 Mobile 共用的类型化 API 客户端，包含请求超时和统一错误格式化。 + +设置同步现在区分全量刷新和局部更新： + +- 进入页面、登录、前后台恢复和手动刷新使用 grouped operation，多个接口共享一个 loading 状态。 +- 单项设置保存成功后，以 `/api/preferences` 返回结果为准，只应用受影响的设置域。 +- AI 回复语言通过 `responseLocale` 传递；ASR 识别语言单独通过 `languageMode` 配置。 ## 项目结构 @@ -171,12 +186,15 @@ SUPABASE_SERVICE_ROLE_KEY=your-service-role-key DEEPSEEK_API_KEY=your-deepseek-api-key # 可选，不填则使用 mock AI ``` -TTS 服务商密钥（均可选，不填则使用 mock TTS）： +ASR/TTS 服务商密钥（均可选，不填则使用 mock/native 兜底）： ```text +ASR_PROVIDER=native +TTS_PROVIDER=mock XUNFEI_APP_ID= XUNFEI_API_KEY= XUNFEI_API_SECRET= +XUNFEI_ASR_PRODUCT=zh_iat XUNFEI_TTS_VOICE= # 默认兜底 vcn；具体教练声音在设置页选择 VOLCENGINE_ACCESS_KEY= VOLCENGINE_SECRET_KEY= @@ -202,7 +220,7 @@ npm run dev 打开 `http://127.0.0.1:3001` -不配置任何 API Key 也能以 mock 模式完整运行。真实 AI 回复需要 `DEEPSEEK_API_KEY`，真实语音输出需要至少一个 TTS 服务商密钥。 +不配置任何 API Key 也能以 mock 模式完整运行。真实 AI 回复需要 `DEEPSEEK_API_KEY`，真实语音输出需要至少一个 TTS 服务商密钥。远端 ASR 需要配置服务商密钥和 `ASR_PROVIDER`；否则移动端可以使用原生语音识别。 ## 移动端 @@ -233,7 +251,7 @@ eas build --platform ios --profile preview ```json "extra": { "apiBaseUrl": "https://meteorvoice.jcmeteor.com", - "apiBaseUrlPreview": "https://meteorvoice-pre.jcmeteor.com" + "apiBaseUrlPreview": "https://mv-pre.jcmeteor.com" } ``` @@ -253,6 +271,20 @@ eas build --platform ios --profile preview 详见 `docs/tts-integration.md`。 +## ASR 服务商 + +MeteorVoice 现在提供统一 ASR provider 层： + +| 服务商 | Key | 状态 | +|--------|-----|------| +| 原生移动端语音识别 | `native` | 默认兜底 | +| 讯飞 `zh_iat` | `xunfei` | 已支持签名 WebSocket 启动和 iOS PCM streaming 路径 | +| Azure Speech | `azure` | 合约已准备，运行时 adapter 待接入 | + +ASR 层和 AI 回复语言路由是分离的：`ASR languageMode` 只控制识别语言，`responseLocale` 控制教练回复语言。 + +合约、落地路径、诊断和测试说明详见 `docs/asr-provider-layer.md`。 + ## 验证 ```bash @@ -280,6 +312,7 @@ flowchart LR | **双端架构** | `apps/web` + `apps/mobile` + `packages/*` monorepo | ✅ 已交付 | | **沉浸式 UI** | 语音波形、桌面/移动双布局、实时字幕 | ✅ 已交付 | | **产品化** | 历史页详展/分页、Preferences 跨设备同步、CI 并行化、Mobile 音频硬化 | ✅ 已交付 | -| **三合一层级判停** | L1 本地判断 + L2 LLM 语义判停 + L3 安全网，替代固定静默等待 | 🚧 实现中 | -| **口音能力专项** | TTS provider voice catalog，多口音发音人（美式/英式/印度/澳式） | 📋 待排期 | +| **三合一层级判停** | L1 本地判断 + L2 LLM 语义判停 + L3 安全网，替代固定静默等待 | ✅ 已交付 | +| **ASR provider 层** | 统一 ASR 合约、讯飞启动会话、原生 PCM 诊断、移动端会话接入 | ✅ 已交付 | +| **口音能力专项** | TTS provider voice catalog，多口音发音人（美式/英式/印度/澳式） | 🚧 实现中 | | **分发** | TestFlight 内测、EAS 正式构建 | 📋 待排期 | diff --git a/apps/mobile/.env.example b/apps/mobile/.env.example new file mode 100644 index 0000000..f22d777 --- /dev/null +++ b/apps/mobile/.env.example @@ -0,0 +1,2 @@ +EXPO_PUBLIC_SUPABASE_URL= +EXPO_PUBLIC_SUPABASE_ANON_KEY= diff --git a/apps/mobile/app.json b/apps/mobile/app.json index 6250956..4c3f677 100644 --- a/apps/mobile/app.json +++ b/apps/mobile/app.json @@ -3,7 +3,7 @@ "name": "MeteorVoice", "slug": "meteorvoice-mobile", "owner": "junchenmeteors-team", - "version": "1.2.0", + "version": "1.3.0", "orientation": "portrait", "scheme": "meteorvoice", "userInterfaceStyle": "automatic", @@ -11,19 +11,22 @@ "ios": { "supportsTablet": true, "bundleIdentifier": "com.tzcpa.meteorvoice", - "buildNumber": "2026053107", + "buildNumber": "2026060601", "deploymentTarget": "18.0", "infoPlist": { "NSMicrophoneUsageDescription": "MeteorVoice uses the microphone for voice practice sessions.", "UIBackgroundModes": [ "audio" ], - "NSSpeechRecognitionUsageDescription": "MeteorVoice uses speech recognition to turn your practice speech into text." + "NSSpeechRecognitionUsageDescription": "MeteorVoice uses speech recognition to turn your practice speech into text.", + "NSAppTransportSecurity": { + "NSAllowsArbitraryLoads": false + } } }, "android": { "package": "com.jcmeteor.meteorvoice", - "versionCode": 2026053107, + "versionCode": 2026060601, "permissions": [ "android.permission.RECORD_AUDIO", "android.permission.MODIFY_AUDIO_SETTINGS", @@ -57,7 +60,7 @@ ], "extra": { "apiBaseUrl": "https://meteorvoice.jcmeteor.com", - "apiBaseUrlPreview": "https://meteorvoice-pre.jcmeteor.com", + "apiBaseUrlPreview": "https://mv-pre.jcmeteor.com", "eas": { "projectId": "9590b05e-34a0-4e1c-8c41-3841b1765c77" } diff --git a/apps/mobile/assets/icon.png b/apps/mobile/assets/icon.png index cd17172..b918023 100644 Binary files a/apps/mobile/assets/icon.png and b/apps/mobile/assets/icon.png differ diff --git a/apps/mobile/eas.json b/apps/mobile/eas.json index e96d030..064e41d 100644 --- a/apps/mobile/eas.json +++ b/apps/mobile/eas.json @@ -7,7 +7,7 @@ "developmentClient": true, "distribution": "internal", "env": { - "EXPO_PUBLIC_API_BASE_URL": "https://meteorvoice-pre.jcmeteor.com" + "EXPO_PUBLIC_API_BASE_URL": "https://mv-pre.jcmeteor.com" }, "ios": { "simulator": true @@ -16,7 +16,7 @@ "preview": { "distribution": "internal", "env": { - "EXPO_PUBLIC_API_BASE_URL": "https://meteorvoice-pre.jcmeteor.com" + "EXPO_PUBLIC_API_BASE_URL": "https://mv-pre.jcmeteor.com" }, "ios": { "simulator": false diff --git a/apps/mobile/ios/MeteorVoice.xcodeproj/project.pbxproj b/apps/mobile/ios/MeteorVoice.xcodeproj/project.pbxproj index b5c3261..3ae5a9a 100644 --- a/apps/mobile/ios/MeteorVoice.xcodeproj/project.pbxproj +++ b/apps/mobile/ios/MeteorVoice.xcodeproj/project.pbxproj @@ -338,7 +338,7 @@ CODE_SIGN_ENTITLEMENTS = MeteorVoice/MeteorVoice.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 2026053107; + CURRENT_PROJECT_VERSION = 2026060601; DEVELOPMENT_TEAM = LGGMBJG4VB; ENABLE_BITCODE = NO; GCC_PREPROCESSOR_DEFINITIONS = ( @@ -351,7 +351,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.2.0; + MARKETING_VERSION = 1.3.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", @@ -376,7 +376,7 @@ CODE_SIGN_ENTITLEMENTS = MeteorVoice/MeteorVoice.entitlements; CODE_SIGN_IDENTITY = "Apple Development"; CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 2026053107; + CURRENT_PROJECT_VERSION = 2026060601; DEVELOPMENT_TEAM = LGGMBJG4VB; INFOPLIST_FILE = MeteorVoice/Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 18.0; @@ -384,7 +384,7 @@ "$(inherited)", "@executable_path/Frameworks", ); - MARKETING_VERSION = 1.2.0; + MARKETING_VERSION = 1.3.0; OTHER_LDFLAGS = ( "$(inherited)", "-ObjC", diff --git a/apps/mobile/ios/MeteorVoice/Images.xcassets/AppIcon.appiconset/App-Icon-1024x1024@1x.png b/apps/mobile/ios/MeteorVoice/Images.xcassets/AppIcon.appiconset/App-Icon-1024x1024@1x.png index b43044e..3c1e163 100644 Binary files a/apps/mobile/ios/MeteorVoice/Images.xcassets/AppIcon.appiconset/App-Icon-1024x1024@1x.png and b/apps/mobile/ios/MeteorVoice/Images.xcassets/AppIcon.appiconset/App-Icon-1024x1024@1x.png differ diff --git a/apps/mobile/ios/MeteorVoice/Info.plist b/apps/mobile/ios/MeteorVoice/Info.plist index 6cc1f69..fe9f4b3 100644 --- a/apps/mobile/ios/MeteorVoice/Info.plist +++ b/apps/mobile/ios/MeteorVoice/Info.plist @@ -38,13 +38,11 @@ 12.0 LSRequiresIPhoneOS - NSAppTransportSecurity - - NSAllowsArbitraryLoads - - NSAllowsLocalNetworking - - + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + NSFaceIDUsageDescription Allow $(PRODUCT_NAME) to access your Face ID biometric data. NSMicrophoneUsageDescription @@ -84,4 +82,4 @@ UIViewControllerBasedStatusBarAppearance - \ No newline at end of file + diff --git a/apps/mobile/ios/Podfile.lock b/apps/mobile/ios/Podfile.lock index 9b0f9a2..bafa939 100644 --- a/apps/mobile/ios/Podfile.lock +++ b/apps/mobile/ios/Podfile.lock @@ -1856,6 +1856,8 @@ PODS: - ReactNativeDependencies (0.83.0) - VoiceAudioSession (0.1.0): - ExpoModulesCore + - VoicePcmCapture (0.1.0): + - ExpoModulesCore - Yoga (0.0.0) DEPENDENCIES: @@ -1946,6 +1948,7 @@ DEPENDENCIES: - ReactCommon/turbomodule/core (from `../../../node_modules/react-native/ReactCommon`) - ReactNativeDependencies (from `../../../node_modules/react-native/third-party-podspecs/ReactNativeDependencies.podspec`) - VoiceAudioSession (from `../modules/voice-audio-session/ios`) + - VoicePcmCapture (from `../modules/voice-pcm-capture/ios`) - Yoga (from `../../../node_modules/react-native/ReactCommon/yoga`) EXTERNAL SOURCES: @@ -2122,6 +2125,8 @@ EXTERNAL SOURCES: :podspec: "../../../node_modules/react-native/third-party-podspecs/ReactNativeDependencies.podspec" VoiceAudioSession: :path: "../modules/voice-audio-session/ios" + VoicePcmCapture: + :path: "../modules/voice-pcm-capture/ios" Yoga: :path: "../../../node_modules/react-native/ReactCommon/yoga" @@ -2212,6 +2217,7 @@ SPEC CHECKSUMS: ReactCommon: e22f6e537e566de4c61121da2139a1980caff948 ReactNativeDependencies: d4bf16fb4f11558ffe141c42146791892f58666f VoiceAudioSession: 480d1183a4f27f9f0253116e067343f1a5067af0 + VoicePcmCapture: 341b0ed696cea24b337066c8ce56738c9a89de61 Yoga: 90687fcd1ebd927341caed917696d692813c0b24 PODFILE CHECKSUM: 09905f1f2c1d6ab15e8eda814d07fdc5f2141069 diff --git a/apps/mobile/modules/voice-pcm-capture/expo-module.config.json b/apps/mobile/modules/voice-pcm-capture/expo-module.config.json new file mode 100644 index 0000000..73bac84 --- /dev/null +++ b/apps/mobile/modules/voice-pcm-capture/expo-module.config.json @@ -0,0 +1,6 @@ +{ + "platforms": ["ios"], + "ios": { + "modules": ["VoicePcmCaptureModule"] + } +} diff --git a/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCapture.podspec b/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCapture.podspec new file mode 100644 index 0000000..936953c --- /dev/null +++ b/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCapture.podspec @@ -0,0 +1,26 @@ +require 'json' + +package = JSON.parse(File.read(File.join(__dir__, '..', 'package.json'))) + +Pod::Spec.new do |s| + s.name = 'VoicePcmCapture' + s.version = package['version'] + s.summary = package['description'] + s.description = package['description'] + s.license = package['license'] + s.author = 'MeteorVoice' + s.homepage = 'https://meteorvoice.jcmeteor.com' + s.platforms = { :ios => '18.0' } + s.swift_version = '5.9' + s.source = { :path => '.' } + s.static_framework = true + + s.dependency 'ExpoModulesCore' + + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'SWIFT_COMPILATION_MODE' => 'wholemodule' + } + + s.source_files = '**/*.{swift}' +end diff --git a/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCaptureModule.swift b/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCaptureModule.swift new file mode 100644 index 0000000..5974f11 --- /dev/null +++ b/apps/mobile/modules/voice-pcm-capture/ios/VoicePcmCaptureModule.swift @@ -0,0 +1,353 @@ +import AVFoundation +import ExpoModulesCore + +public class VoicePcmCaptureModule: Module { + private let captureQueue = DispatchQueue(label: "com.meteorvoice.voicePcmCapture") + private var engine: AVAudioEngine? + private var converter: AVAudioConverter? + private var targetFormat: AVAudioFormat? + private var pendingPcm = Data() + private var isCapturing = false + private var sequence = 0 + private var totalBytes = 0 + private var startedAtMs: Int64 = 0 + private var frameSizeBytes = 1280 + private var frameDurationMs = 40 + private var sampleRate = 16000.0 + private var routeChangeObserver: NSObjectProtocol? + private var interruptionObserver: NSObjectProtocol? + private var mediaServicesResetObserver: NSObjectProtocol? + + public func definition() -> ModuleDefinition { + Name("VoicePcmCapture") + + Events("onPcmCaptureFrame", "onPcmCaptureState") + + AsyncFunction("start") { (options: [String: Any]) -> [String: Any] in + try self.startCapture(options: options) + } + + AsyncFunction("stop") { (reason: String?) -> [String: Any] in + self.stopCapture(reason: reason ?? "manual") + } + + AsyncFunction("getStatus") { () -> [String: Any] in + self.currentStatus() + } + } + + private func startCapture(options: [String: Any]) throws -> [String: Any] { + if isCapturing { + return currentStatus() + } + + sampleRate = options["sampleRate"] as? Double ?? 16000.0 + frameDurationMs = options["frameDurationMs"] as? Int ?? 40 + frameSizeBytes = Int(sampleRate * Double(frameDurationMs) / 1000.0) * 2 + + try configureAudioSession() + + let nextEngine = AVAudioEngine() + let inputFormat = nextEngine.inputNode.outputFormat(forBus: 0) + guard let nextTargetFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: false + ) else { + throw NSError( + domain: "VoicePcmCapture", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Failed to create target PCM format."] + ) + } + guard let nextConverter = AVAudioConverter(from: inputFormat, to: nextTargetFormat) else { + throw NSError( + domain: "VoicePcmCapture", + code: 2, + userInfo: [NSLocalizedDescriptionKey: "Failed to create PCM converter."] + ) + } + + pendingPcm.removeAll(keepingCapacity: false) + sequence = 0 + totalBytes = 0 + startedAtMs = nowMs() + targetFormat = nextTargetFormat + converter = nextConverter + isCapturing = true + + do { + try installAndStartEngine(nextEngine, inputFormat: inputFormat) + } catch { + isCapturing = false + nextEngine.inputNode.removeTap(onBus: 0) + engine = nil + converter = nil + targetFormat = nil + throw error + } + + installAudioSessionObservers() + sendState("started", message: nil) + return currentStatus() + } + + private func stopCapture(reason: String) -> [String: Any] { + if !isCapturing { + return currentStatus(reason: reason) + } + + isCapturing = false + engine?.inputNode.removeTap(onBus: 0) + engine?.stop() + engine = nil + converter = nil + targetFormat = nil + pendingPcm.removeAll(keepingCapacity: false) + removeAudioSessionObservers() + + let status = currentStatus(reason: reason) + sendState("stopped", message: reason) + return status + } + + private func configureAudioSession() throws { + let session = AVAudioSession.sharedInstance() + try session.setCategory( + .playAndRecord, + mode: .voiceChat, + options: [.allowBluetoothHFP, .defaultToSpeaker] + ) + try session.setActive(true) + try session.overrideOutputAudioPort(.speaker) + } + + private func installAndStartEngine(_ nextEngine: AVAudioEngine, inputFormat: AVAudioFormat) throws { + let inputNode = nextEngine.inputNode + let tapBufferSize = AVAudioFrameCount(max(1024, Int(inputFormat.sampleRate * 0.04))) + inputNode.installTap(onBus: 0, bufferSize: tapBufferSize, format: inputFormat) { [weak self] buffer, _ in + self?.captureQueue.async { + self?.handleInputBuffer(buffer) + } + } + nextEngine.prepare() + try nextEngine.start() + engine = nextEngine + } + + private func restartCaptureEngine(reason: String) { + guard isCapturing else { + return + } + + sendState("restarting", message: reason) + engine?.inputNode.removeTap(onBus: 0) + engine?.stop() + engine = nil + converter = nil + targetFormat = nil + pendingPcm.removeAll(keepingCapacity: false) + + do { + try configureAudioSession() + let nextEngine = AVAudioEngine() + let inputFormat = nextEngine.inputNode.outputFormat(forBus: 0) + guard let nextTargetFormat = AVAudioFormat( + commonFormat: .pcmFormatInt16, + sampleRate: sampleRate, + channels: 1, + interleaved: false + ) else { + throw NSError( + domain: "VoicePcmCapture", + code: 3, + userInfo: [NSLocalizedDescriptionKey: "Failed to recreate target PCM format."] + ) + } + guard let nextConverter = AVAudioConverter(from: inputFormat, to: nextTargetFormat) else { + throw NSError( + domain: "VoicePcmCapture", + code: 4, + userInfo: [NSLocalizedDescriptionKey: "Failed to recreate PCM converter."] + ) + } + + targetFormat = nextTargetFormat + converter = nextConverter + try installAndStartEngine(nextEngine, inputFormat: inputFormat) + sendState("restarted", message: reason) + } catch { + isCapturing = false + engine?.inputNode.removeTap(onBus: 0) + engine?.stop() + engine = nil + converter = nil + targetFormat = nil + pendingPcm.removeAll(keepingCapacity: false) + removeAudioSessionObservers() + sendState("error", message: error.localizedDescription) + } + } + + private func installAudioSessionObservers() { + removeAudioSessionObservers() + let center = NotificationCenter.default + routeChangeObserver = center.addObserver( + forName: AVAudioSession.routeChangeNotification, + object: AVAudioSession.sharedInstance(), + queue: nil + ) { [weak self] notification in + let reasonValue = notification.userInfo?[AVAudioSessionRouteChangeReasonKey] as? UInt + let reason = AVAudioSession.RouteChangeReason(rawValue: reasonValue ?? 0) + self?.captureQueue.async { + if reason == .categoryChange { + self?.sendState("route_change_ignored", message: "route_change:\(reasonValue ?? 0)") + return + } + self?.restartCaptureEngine(reason: "route_change:\(reasonValue ?? 0)") + } + } + interruptionObserver = center.addObserver( + forName: AVAudioSession.interruptionNotification, + object: AVAudioSession.sharedInstance(), + queue: nil + ) { [weak self] notification in + let typeValue = notification.userInfo?[AVAudioSessionInterruptionTypeKey] as? UInt + let type = AVAudioSession.InterruptionType(rawValue: typeValue ?? 0) + guard type == .ended else { + self?.sendState("interrupted", message: "audio_session_interruption") + return + } + self?.captureQueue.async { + self?.restartCaptureEngine(reason: "interruption_ended") + } + } + mediaServicesResetObserver = center.addObserver( + forName: AVAudioSession.mediaServicesWereResetNotification, + object: AVAudioSession.sharedInstance(), + queue: nil + ) { [weak self] _ in + self?.captureQueue.async { + self?.restartCaptureEngine(reason: "media_services_reset") + } + } + } + + private func removeAudioSessionObservers() { + let center = NotificationCenter.default + if let routeChangeObserver { + center.removeObserver(routeChangeObserver) + self.routeChangeObserver = nil + } + if let interruptionObserver { + center.removeObserver(interruptionObserver) + self.interruptionObserver = nil + } + if let mediaServicesResetObserver { + center.removeObserver(mediaServicesResetObserver) + self.mediaServicesResetObserver = nil + } + } + + private func handleInputBuffer(_ buffer: AVAudioPCMBuffer) { + guard isCapturing, let converter, let targetFormat else { + return + } + + let ratio = targetFormat.sampleRate / buffer.format.sampleRate + let outputCapacity = AVAudioFrameCount(max(1, Double(buffer.frameLength) * ratio + 8)) + guard let outputBuffer = AVAudioPCMBuffer(pcmFormat: targetFormat, frameCapacity: outputCapacity) else { + sendState("error", message: "Failed to allocate PCM output buffer.") + return + } + + var didProvideInput = false + var convertError: NSError? + converter.convert(to: outputBuffer, error: &convertError) { _, outStatus in + if didProvideInput { + outStatus.pointee = .noDataNow + return nil + } + didProvideInput = true + outStatus.pointee = .haveData + return buffer + } + + if let convertError { + sendState("error", message: convertError.localizedDescription) + return + } + + guard let channelData = outputBuffer.int16ChannelData else { + sendState("error", message: "Converted PCM buffer has no int16 channel data.") + return + } + + let byteCount = Int(outputBuffer.frameLength) * 2 + if byteCount <= 0 { + return + } + + pendingPcm.append(Data(bytes: channelData[0], count: byteCount)) + emitCompleteFrames() + } + + private func emitCompleteFrames() { + while pendingPcm.count >= frameSizeBytes { + let frame = pendingPcm.prefix(frameSizeBytes) + pendingPcm.removeFirst(frameSizeBytes) + sequence += 1 + totalBytes += frameSizeBytes + + let body: [String: Any] = [ + "sequence": sequence, + "timestampMs": nowMs(), + "elapsedMs": max(0, nowMs() - startedAtMs), + "audioBase64": Data(frame).base64EncodedString(), + "byteCount": frameSizeBytes, + "sampleRate": Int(sampleRate), + "channels": 1, + "bitDepth": 16, + "durationMs": frameDurationMs, + ] + + DispatchQueue.main.async { + self.sendEvent("onPcmCaptureFrame", body) + } + } + } + + private func currentStatus(reason: String? = nil) -> [String: Any] { + var status: [String: Any] = [ + "isCapturing": isCapturing, + "sampleRate": Int(sampleRate), + "channels": 1, + "bitDepth": 16, + "frameDurationMs": frameDurationMs, + "frameSizeBytes": frameSizeBytes, + "frameCount": sequence, + "totalBytes": totalBytes, + "elapsedMs": startedAtMs > 0 ? max(0, nowMs() - startedAtMs) : 0, + ] + if let reason { + status["reason"] = reason + } + return status + } + + private func sendState(_ state: String, message: String?) { + var body = currentStatus() + body["state"] = state + if let message { + body["message"] = message + } + DispatchQueue.main.async { + self.sendEvent("onPcmCaptureState", body) + } + } + + private func nowMs() -> Int64 { + Int64(Date().timeIntervalSince1970 * 1000) + } +} diff --git a/apps/mobile/modules/voice-pcm-capture/package.json b/apps/mobile/modules/voice-pcm-capture/package.json new file mode 100644 index 0000000..6973922 --- /dev/null +++ b/apps/mobile/modules/voice-pcm-capture/package.json @@ -0,0 +1,8 @@ +{ + "name": "voice-pcm-capture", + "version": "0.1.0", + "description": "MeteorVoice iOS PCM microphone capture module.", + "main": "src/index.ts", + "license": "UNLICENSED", + "private": true +} diff --git a/apps/mobile/package.json b/apps/mobile/package.json index edb5737..dd98185 100644 --- a/apps/mobile/package.json +++ b/apps/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@meteorvoice/mobile", - "version": "1.2.0", + "version": "1.3.0", "private": true, "main": "index.ts", "scripts": { diff --git a/apps/mobile/src/App.tsx b/apps/mobile/src/App.tsx index 8593a57..73728c9 100644 --- a/apps/mobile/src/App.tsx +++ b/apps/mobile/src/App.tsx @@ -11,7 +11,9 @@ import { } from 'react-native' import { createMeteorVoiceApiClient, - MeteorVoiceApiError, + fetchWithTimeout, + formatApiRequestError, + type CreateASRSessionResponse, type HistorySession, type PreferencesResponse, type SessionTurnDto, @@ -46,8 +48,12 @@ import { getScenarioDescription, getScenarioLabel, getTTSSpeedRouting, + runAppOperationGroup, scenarios, t, + appFeedback, + displayErrorFeedback, + type AppFeedbackState, type ConversationMessage, type ConversationResponse, type Locale, @@ -58,24 +64,47 @@ import * as SecureStore from 'expo-secure-store' import { useMobileAuth } from './mobileAuth' import { useNativeSessionAudio } from './nativeAudio' import { useNativeSpeech } from './nativeSpeech' -import { pullMobilePreferences, syncMobilePreferences, type XunfeiVoice } from './mobilePreferences' +import { syncMobilePreferences, type XunfeiVoice } from './mobilePreferences' import { getDefaultApiBaseUrl, getDisplayAppVersion } from './mobileConfig' +import { + addPcmFrameListener, + addPcmStateListener, + isPcmCaptureAvailable, + startPcmCapture, + stopPcmCapture, + type PcmCaptureFrameEvent, +} from './voicePcmCapture' import { ThemeProvider, useTheme } from './ThemeProvider' import { SessionScreen } from './screens/SessionScreen' import { HomeScreen } from './screens/HomeScreen' import { HistoryScreen } from './screens/HistoryScreen' import { SettingsScreen } from './screens/SettingsScreen' +import { AppFeedbackOverlay } from './components/AppFeedbackOverlay' const defaultApiBaseUrl = getDefaultApiBaseUrl() const appVersion = getDisplayAppVersion() const apiBaseUrlStorageKey = 'api_base_url' +const sessionSttProviderStorageKey = 'session_stt_provider' type Tab = 'session' | 'home' | 'history' | 'settings' +type ApiBaseUrlSource = 'default' | 'user' +type SessionSttProvider = 'native' | 'xunfei' type VoiceMetricEntry = { ts: number stage: string data: Record } +type ASREvaluationRun = { + startedAt?: number + firstPartialMs?: number | null + finalMs?: number | null + chars?: number + source?: string + frameCount?: number + totalBytes?: number + error?: string +} + const TAB_LABELS: Record = { home: 'nav.home', session: 'nav.practice', @@ -114,6 +143,211 @@ function TabIcon({ tab, color }: { tab: Tab; color: string }) { ) } +function createASREvaluationReport(entries: VoiceMetricEntry[]) { + const nativeRuns: ASREvaluationRun[] = [] + const remoteRuns: ASREvaluationRun[] = [] + let currentNative: ASREvaluationRun | null = null + let currentRemote: ASREvaluationRun | null = null + + for (const entry of entries) { + if (entry.stage === 'stt_start') { + currentNative = { startedAt: entry.ts } + nativeRuns.push(currentNative) + } else if (entry.stage === 'stt_first_partial' && currentNative) { + currentNative.firstPartialMs = readMetricNumber(entry.data.elapsedMs) + currentNative.chars = readMetricNumber(entry.data.chars) ?? currentNative.chars + } else if (entry.stage === 'stt_submit' && currentNative) { + currentNative.finalMs = readMetricNumber(entry.data.elapsedMs) + currentNative.chars = readMetricNumber(entry.data.chars) ?? currentNative.chars + currentNative.source = typeof entry.data.source === 'string' ? entry.data.source : undefined + } else if (entry.stage === 'stt_end' && currentNative && currentNative.finalMs == null) { + currentNative.finalMs = readMetricNumber(entry.data.elapsedMs) + } + + if (entry.stage === 'asr_stream_start') { + currentRemote = { startedAt: entry.ts } + remoteRuns.push(currentRemote) + } else if (entry.stage === 'asr_first_partial' && currentRemote) { + currentRemote.firstPartialMs = readMetricNumber(entry.data.elapsedMs) + currentRemote.chars = readMetricNumber(entry.data.chars) ?? currentRemote.chars + } else if (entry.stage === 'asr_stream_done' && currentRemote) { + currentRemote.finalMs = readMetricNumber(entry.data.streamElapsedMs) ?? readMetricNumber(entry.data.elapsedMs) + currentRemote.chars = readMetricNumber(entry.data.transcriptChars) ?? currentRemote.chars + currentRemote.frameCount = readMetricNumber(entry.data.frameCount) ?? undefined + currentRemote.totalBytes = readMetricNumber(entry.data.totalBytes) ?? undefined + } else if (entry.stage === 'asr_stream_provider_error' && currentRemote) { + currentRemote.error = typeof entry.data.message === 'string' ? entry.data.message : 'Provider error' + } else if (entry.stage === 'asr_diagnostic_error' && currentRemote) { + currentRemote.error = typeof entry.data.message === 'string' ? entry.data.message : 'Diagnostic error' + } + } + + const latestNative = nativeRuns.at(-1) + const latestRemote = remoteRuns.at(-1) + return [ + 'ASR P4 evaluation report', + `Generated: ${new Date().toISOString()}`, + '', + `Native runs: ${nativeRuns.length}`, + formatASRRun('Latest native', latestNative), + '', + `Remote Xunfei runs: ${remoteRuns.length}`, + formatASRRun('Latest remote', latestRemote), + '', + 'Acceptance checks:', + '- Compare first partial latency between native and remote.', + '- Compare final latency between native and remote.', + '- Compare transcript chars and exported raw metrics against the spoken script.', + '- Do not switch production STT until remote accuracy and latency are better on device.', + ].join('\n') +} + +function formatASRRun(label: string, run: ASREvaluationRun | undefined) { + if (!run) return `${label}: no run captured` + return [ + `${label}:`, + ` startedAt: ${run.startedAt ? new Date(run.startedAt).toLocaleString() : 'unknown'}`, + ` firstPartialMs: ${formatMetricValue(run.firstPartialMs)}`, + ` finalMs: ${formatMetricValue(run.finalMs)}`, + ` chars: ${formatMetricValue(run.chars)}`, + run.source ? ` source: ${run.source}` : null, + run.frameCount != null ? ` frameCount: ${run.frameCount}` : null, + run.totalBytes != null ? ` totalBytes: ${run.totalBytes}` : null, + run.error ? ` error: ${run.error}` : null, + ].filter(Boolean).join('\n') +} + +function readMetricNumber(value: unknown) { + return typeof value === 'number' && Number.isFinite(value) ? value : null +} + +function formatMetricValue(value: number | null | undefined) { + return value == null ? 'n/a' : String(value) +} + +function withTimeout(promise: Promise, timeoutMs: number, message: string) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(message)), timeoutMs) + promise + .then(resolve, reject) + .finally(() => clearTimeout(timer)) + }) +} + +function createXunfeiASRFrame(session: CreateASRSessionResponse, status: 0 | 1 | 2, audioBase64: string, sequence: number) { + const providerConfig = session.providerConfig + const header = { + app_id: providerConfig?.appId, + status, + } + const audio = { + encoding: providerConfig?.audioEncoding ?? 'raw', + sample_rate: providerConfig?.sampleRate ?? 16000, + channels: providerConfig?.channels ?? 1, + bit_depth: providerConfig?.bitDepth ?? 16, + seq: sequence, + status, + audio: audioBase64, + } + if (status !== 0) { + return { + header, + payload: { audio }, + } + } + + return { + header, + parameter: { + iat: { + domain: providerConfig?.domain ?? 'slm', + language: providerConfig?.language ?? 'zh_cn', + accent: providerConfig?.accent ?? 'mandarin', + eos: providerConfig?.eosMs ?? 900, + dwa: 'wpgs', + result: { + encoding: 'utf8', + compress: 'raw', + format: 'json', + }, + }, + }, + payload: { audio }, + } +} + +function parseJsonObject(value: unknown): Record | null { + if (typeof value !== 'string') return null + try { + const parsed = JSON.parse(value) + return parsed && typeof parsed === 'object' ? parsed : null + } catch { + return null + } +} + +function getObject(value: unknown): Record | null { + return value && typeof value === 'object' && !Array.isArray(value) ? value as Record : null +} + +function extractXunfeiRecognitionResult(payload: Record | null) { + const payloadObject = getObject(payload?.payload) + const payloadResult = getObject(payloadObject?.result) + const encodedText = typeof payloadResult?.text === 'string' ? payloadResult.text : null + if (encodedText) { + const decoded = decodeBase64Utf8(encodedText) + const decodedPayload = parseJsonObject(decoded) + const decodedWords = extractXunfeiWords(decodedPayload?.ws) + if (decodedWords) { + const rg = Array.isArray(decodedPayload?.rg) && + typeof decodedPayload.rg[0] === 'number' && + typeof decodedPayload.rg[1] === 'number' + ? [decodedPayload.rg[0], decodedPayload.rg[1]] as [number, number] + : null + return { + text: decodedWords, + sn: typeof decodedPayload?.sn === 'number' ? decodedPayload.sn : null, + pgs: typeof decodedPayload?.pgs === 'string' ? decodedPayload.pgs : null, + rg, + } + } + } + + const data = getObject(payload?.data) + const result = getObject(data?.result) + const fallbackWords = extractXunfeiWords(result?.ws) + return fallbackWords + ? { text: fallbackWords, sn: null, pgs: null, rg: null } + : null +} + +function extractXunfeiWords(words: unknown) { + if (!Array.isArray(words)) return '' + return words.map(item => { + const word = getObject(item) + const candidates = word?.cw + if (!Array.isArray(candidates)) return '' + return candidates.map(candidate => { + const candidateObject = getObject(candidate) + return typeof candidateObject?.w === 'string' ? candidateObject.w : '' + }).join('') + }).join('') +} + +function decodeBase64Utf8(value: string) { + try { + const decoder = globalThis.atob + if (!decoder) return '' + const binary = decoder(value) + const escaped = Array.from(binary) + .map(char => `%${char.charCodeAt(0).toString(16).padStart(2, '0')}`) + .join('') + return decodeURIComponent(escaped) + } catch { + return '' + } +} + export default function App() { return ( @@ -126,6 +360,7 @@ function AppInner() { const { C, setTheme: setThemeLocal } = useTheme() const [activeTab, setActiveTab] = useState('session') const [apiBaseUrl, setApiBaseUrl] = useState(defaultApiBaseUrl) + const [apiBaseUrlSource, setApiBaseUrlSource] = useState('default') const [messages, setMessages] = useState([]) const [correctionHistory, setCorrectionHistory] = useState([]) const [audioUrl, setAudioUrl] = useState(null) @@ -138,7 +373,13 @@ function AppInner() { useEffect(() => { SecureStore.getItemAsync(apiBaseUrlStorageKey).then(value => { const stored = value?.trim() - if (stored) setApiBaseUrl(stored) + if (stored) { + setApiBaseUrl(stored) + setApiBaseUrlSource('user') + } else { + setApiBaseUrl(defaultApiBaseUrl) + setApiBaseUrlSource('default') + } }) }, []) const setLocale = useCallback((l: Locale) => { @@ -155,8 +396,13 @@ function AppInner() { const [selectedHistoryTurns, setSelectedHistoryTurns] = useState([]) const [settingsLoading, setSettingsLoading] = useState(false) const [settingsMessage, setSettingsMessage] = useState(null) + const [authSubmitting, setAuthSubmitting] = useState(false) + const [scenarioSwitching, setScenarioSwitching] = useState(false) + const [activeFeedback, setActiveFeedback] = useState(() => appFeedback.getFeedback()) const [ttsProvider, setTtsProvider] = useState('mock') const [availableProviders, setAvailableProviders] = useState(['mock']) + const [sessionSttProvider, setSessionSttProviderState] = useState('native') + const [availableSessionSttProviders, setAvailableSessionSttProviders] = useState(['native']) const [ttsVoiceId, setTtsVoiceId] = useState(null) const [voiceProfiles, setVoiceProfiles] = useState([]) const [selectedVoiceProfileId, setSelectedVoiceProfileId] = useState(null) @@ -174,21 +420,48 @@ function AppInner() { const audio = useNativeSessionAudio(audioUrl, ttsSpeedRouting.playbackRate) const auth = useMobileAuth() const getAuthHeaders = auth.getAuthHeaders + const signOut = auth.signOut + const snapshotRef = useRef(snapshot) + const messagesRef = useRef(messages) + const localeRef = useRef(locale) + const sessionSttProviderRef = useRef(sessionSttProvider) + const sessionSttProviderHydratedRef = useRef(false) + const sessionSttProvidersLoadedRef = useRef(false) + const startListeningWithProviderRef = useRef<(provider: SessionSttProvider, lang?: string) => Promise>(() => Promise.resolve(false)) const prefSyncTimerRef = useRef | null>(null) const themeInitializedRef = useRef(false) const listeningStartMsRef = useRef(0) const speechStartListeningRef = useRef<(lang?: string) => Promise>(() => Promise.resolve(false)) const speechCancelListeningRef = useRef<() => void | Promise>(() => undefined) + const nativeSpeechStartListeningRef = useRef<(lang?: string) => Promise>(() => Promise.resolve(false)) + const nativeSpeechCancelListeningRef = useRef<() => void | Promise>(() => undefined) + const xunfeiSessionSttRef = useRef<{ + socket: WebSocket | null + frameSubscription: { remove: () => void } | null + stateSubscription: { remove: () => void } | null + finalizeTimer: ReturnType | null + hardTimer: ReturnType | null + noFrameTimer: ReturnType | null + settled: boolean + } | null>(null) const endpointRequestRef = useRef(0) + const turnRequestRef = useRef(0) const sessionActiveRef = useRef(false) const canListenOnRouteRef = useRef(true) const playbackActiveRef = useRef(false) const audioPlayingRef = useRef(false) + const busyRef = useRef(false) const playbackStartedRef = useRef(false) const playbackEndedAtMsRef = useRef(null) const pendingNativeTranscriptRef = useRef('') const isCorrectionPlayingRef = useRef(false) const resumeListeningTimerRef = useRef | null>(null) + const historyAutoLoadRef = useRef(false) + const settingsAutoLoadRef = useRef(false) + const settingsRequestRef = useRef(0) + const settingsLoadingRef = useRef(false) + const activeTabRef = useRef(activeTab) + const listeningTeardownRef = useRef | null>(null) const scenario = scenarios.find(item => item.key === selectedScenarioKey) ?? scenarios[0] const accent = accentProfiles.find(item => item.key === selectedAccentKey) ?? accentProfiles[0] @@ -198,10 +471,60 @@ function AppInner() { ?? providerVoiceProfiles.find(profile => profile.status === 'active') const sessionAccentName = selectedVoiceProfile?.accentLabel ?? getAccentLabel(accent, locale) const sessionAccentRegion = selectedVoiceProfile?.accentRegion ?? getAccentRegion(accent, locale) + const tr = useCallback((key: string) => t[locale]?.[key] ?? t.en[key] ?? key, [locale]) + + useEffect(() => { + SecureStore.getItemAsync(sessionSttProviderStorageKey).then(value => { + if (value === 'xunfei' || value === 'native') { + sessionSttProviderRef.current = value + setSessionSttProviderState(value) + } + sessionSttProviderHydratedRef.current = true + }) + }, []) + + useEffect(() => { + snapshotRef.current = snapshot + }, [snapshot]) + + useEffect(() => { + messagesRef.current = messages + }, [messages]) + + useEffect(() => { + localeRef.current = locale + }, [locale]) + + useEffect(() => { + sessionSttProviderRef.current = sessionSttProvider + }, [sessionSttProvider]) + + useEffect(() => { + activeTabRef.current = activeTab + }, [activeTab]) + + useEffect(() => { + busyRef.current = busy + }, [busy]) + + const listeningStartupStatus = useCallback((provider = sessionSttProviderRef.current) => ( + provider === 'xunfei' + ? 'session.status.preparing_listening' + : 'session.status.listening' + ), []) + const handleUnauthorized = useCallback(() => { + if (auth.state !== 'signed-in') return signOut(null) + return signOut(tr('settings.auth_expired')) + }, [auth.state, signOut, tr]) const api = useMemo(() => createMeteorVoiceApiClient({ baseUrl: apiBaseUrl.trim(), headers: getAuthHeaders, - }), [apiBaseUrl, getAuthHeaders]) + onUnauthorized: handleUnauthorized, + }), [apiBaseUrl, getAuthHeaders, handleUnauthorized]) + const setSettingsLoadingFlag = useCallback((loading: boolean) => { + settingsLoadingRef.current = loading + setSettingsLoading(loading) + }, []) const applyThemeLocal = useCallback((k: Parameters[0]) => { setThemeLocal(k) }, [setThemeLocal]) @@ -212,8 +535,6 @@ function AppInner() { void SecureStore.setItemAsync('theme_set_at', now) void api.updatePreferences({ ui_theme: k }).catch(() => {}) }, [setThemeLocal, api]) - const tr = useCallback((key: string) => t[locale]?.[key] ?? t.en[key] ?? key, [locale]) - const clearResumeListeningTimer = useCallback(() => { if (!resumeListeningTimerRef.current) return clearTimeout(resumeListeningTimerRef.current) @@ -229,14 +550,79 @@ function AppInner() { ) const entry = { ts: Date.now(), stage, data: sanitizedData } console.info('[voice-metrics]', JSON.stringify(entry)) - setVoiceMetrics(previous => [...previous.slice(-79), entry]) + setVoiceMetrics(previous => [...previous.slice(-239), entry]) }, []) + const logUserAction = useCallback((action: string, data: Record = {}) => { + logVoiceMetric('user_action', { + action, + activeTab: activeTabRef.current, + scenario: selectedScenarioKey, + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + busy, + playbackActive: playbackActiveRef.current, + audioPlaying: audioPlayingRef.current, + sttProvider: sessionSttProviderRef.current, + pendingTeardown: Boolean(listeningTeardownRef.current), + ...data, + }) + }, [busy, logVoiceMetric, selectedScenarioKey]) + + const runListeningTeardown = useCallback((reason: string, action: () => void | Promise) => { + const startedAt = Date.now() + logVoiceMetric('listening_teardown_start', { + reason, + provider: sessionSttProviderRef.current, + activeTab: activeTabRef.current, + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + }) + const task = Promise.resolve() + .then(action) + .catch(error => { + logVoiceMetric('listening_teardown_error', { + reason, + message: error instanceof Error ? error.message : 'Listening teardown failed', + }) + }) + .finally(() => { + if (listeningTeardownRef.current === task) { + listeningTeardownRef.current = null + } + logVoiceMetric('listening_teardown_done', { + reason, + elapsedMs: Date.now() - startedAt, + }) + }) + listeningTeardownRef.current = task + return task + }, [logVoiceMetric]) + + const waitForListeningTeardown = useCallback(async (context: string) => { + const pending = listeningTeardownRef.current + if (!pending) return + const startedAt = Date.now() + logVoiceMetric('listening_teardown_wait', { context }) + await pending + logVoiceMetric('listening_teardown_wait_done', { + context, + elapsedMs: Date.now() - startedAt, + }) + }, [logVoiceMetric]) + + const cancelListeningForReason = useCallback((reason: string) => ( + runListeningTeardown(reason, async () => { + await speechCancelListeningRef.current() + }) + ), [runListeningTeardown]) + const voiceMetricsText = useMemo(() => { return voiceMetrics .map(entry => `${new Date(entry.ts).toLocaleTimeString()} ${entry.stage} ${JSON.stringify(entry.data)}`) .join('\n') }, [voiceMetrics]) + const asrEvaluationText = useMemo(() => createASREvaluationReport(voiceMetrics), [voiceMetrics]) const scheduleResumeListening = useCallback((delayMs = DEFAULT_PLAYBACK_COOLDOWN_MS, updateStatus = true) => { clearResumeListeningTimer() @@ -250,10 +636,10 @@ function AppInner() { return } listeningStartMsRef.current = Date.now() - if (updateStatus) setStatus('session.status.listening') + if (updateStatus) setStatus(listeningStartupStatus()) void speechStartListeningRef.current('en-US') }, delayMs) - }, [clearResumeListeningTimer, logVoiceMetric]) + }, [clearResumeListeningTimer, listeningStartupStatus, logVoiceMetric]) useEffect(() => clearResumeListeningTimer, [clearResumeListeningTimer]) @@ -261,14 +647,94 @@ function AppInner() { setApiBaseUrl(value) const normalized = value.trim() if (!normalized || normalized === defaultApiBaseUrl) { + setApiBaseUrlSource('default') void SecureStore.deleteItemAsync(apiBaseUrlStorageKey) return } + setApiBaseUrlSource('user') void SecureStore.setItemAsync(apiBaseUrlStorageKey, normalized) }, []) - function startSession() { - logVoiceMetric('session_start', { scenario: scenario.key, accent: accent.key, provider: ttsProvider }) + const resetApiBaseUrl = useCallback(() => { + setApiBaseUrl(defaultApiBaseUrl) + setApiBaseUrlSource('default') + void SecureStore.deleteItemAsync(apiBaseUrlStorageKey) + }, []) + + const setSessionSttProvider = useCallback((provider: SessionSttProvider) => { + sessionSttProviderRef.current = provider + setSessionSttProviderState(provider) + void SecureStore.setItemAsync(sessionSttProviderStorageKey, provider) + logVoiceMetric('stt_provider_selected', { provider }) + }, [logVoiceMetric, setSessionSttProviderState]) + + async function ensureSessionSttProviderForStart() { + let provider = sessionSttProviderRef.current + + if (!sessionSttProviderHydratedRef.current) { + const stored = await SecureStore.getItemAsync(sessionSttProviderStorageKey) + if (stored === 'xunfei' || stored === 'native') { + provider = stored + sessionSttProviderRef.current = stored + setSessionSttProviderState(stored) + } + sessionSttProviderHydratedRef.current = true + } + + if (auth.state === 'signed-in' && !sessionSttProvidersLoadedRef.current) { + try { + const result = await api.listASRProviders() + const providers: SessionSttProvider[] = ['native'] + if (result.providers.some(item => item.key === 'xunfei' && item.enabled)) { + providers.push('xunfei') + } + setAvailableSessionSttProviders(providers) + sessionSttProvidersLoadedRef.current = true + if (!providers.includes(provider)) { + provider = 'native' + sessionSttProviderRef.current = 'native' + setSessionSttProviderState('native') + void SecureStore.setItemAsync(sessionSttProviderStorageKey, 'native') + } + } catch (error) { + const requestError = formatApiRequestError(error, { + context: 'mobile_asr_providers_load', + presentation: 'silent', + }) + logVoiceMetric('mobile_silent_request_error', requestError.logData) + } + } + + return provider + } + + async function startSession() { + logUserAction('session_start_tap', { scenario: scenario.key }) + if (scenarioSwitching) { + logVoiceMetric('session_start_blocked', { reason: 'scenario_switching' }) + return + } + if (auth.state !== 'signed-in') { + setActiveTab('settings') + setStatus('login.signin') + return + } + + setStatus('session.status.preparing_listening') + logVoiceMetric('session_start_requested', { + scenario: scenario.key, + activeTab: activeTabRef.current, + pendingTeardown: Boolean(listeningTeardownRef.current), + }) + await waitForListeningTeardown('session_start') + await cancelListeningForReason('session_start_reset') + const listeningProvider = await ensureSessionSttProviderForStart() + logVoiceMetric('session_start', { + scenario: scenario.key, + accent: accent.key, + provider: ttsProvider, + sttProvider: listeningProvider, + }) endpointRequestRef.current += 1 clearResumeListeningTimer() playbackActiveRef.current = false @@ -280,6 +746,8 @@ function AppInner() { canListenOnRouteRef.current = true const nextSessionId = apiSessionId ?? `mobile-${Date.now()}` const nextSnapshot = startListeningSession(nextSessionId) + snapshotRef.current = nextSnapshot + messagesRef.current = [] setSnapshot(nextSnapshot) setMessages([]) setCorrectionHistory([]) @@ -288,8 +756,13 @@ function AppInner() { setPlaybackQueue(createPlaybackQueueSnapshot()) setSummary(null) setIsSessionActive(true) - setStatus('session.status.listening') - void speechStartListeningRef.current('en-US') + setStatus(listeningStartupStatus(listeningProvider)) + logVoiceMetric('session_listening_start_requested', { + sttProvider: listeningProvider, + sessionId: nextSessionId, + canListenOnRoute: canListenOnRouteRef.current, + }) + void startListeningWithProviderRef.current(listeningProvider, 'en-US') } const synthesizeCoachSpeech = useCallback(async (text: string) => { @@ -307,9 +780,9 @@ function AppInner() { if (audio.isPlaying && audioUrl && playbackActiveRef.current && !playbackStartedRef.current) { playbackStartedRef.current = true logVoiceMetric('playback_started', { audioUrl }) - void speechCancelListeningRef.current() + void cancelListeningForReason('playback_started') } - }, [audio.isPlaying, audioUrl, logVoiceMetric]) + }, [audio.isPlaying, audioUrl, cancelListeningForReason, logVoiceMetric]) useEffect(() => { if (!audioUrl || !audio.didJustFinish || audio.isPlaying) return @@ -352,7 +825,7 @@ function AppInner() { playbackStartedRef.current = false playbackEndedAtMsRef.current = null clearResumeListeningTimer() - void speechCancelListeningRef.current() + void cancelListeningForReason('play_next_audio') setStatus('session.status.playing_reply') setAudioUrl(nextQueue.currentAudioUrl) return @@ -373,11 +846,16 @@ function AppInner() { cancelled = true clearTimeout(timeout) } - }, [audio.didJustFinish, audio.isPlaying, audioUrl, playbackQueue, clearResumeListeningTimer, logVoiceMetric, scheduleResumeListening]) + }, [ + audio.didJustFinish, audio.isPlaying, audioUrl, playbackQueue, cancelListeningForReason, + clearResumeListeningTimer, logVoiceMetric, scheduleResumeListening, + ]) const submitTurn = useCallback(async (sourceTranscript: string) => { const submitStartedAt = Date.now() const transcript = sourceTranscript.trim() + const currentSnapshot = snapshotRef.current + const currentMessages = messagesRef.current if ( busy || audio.isRecording || @@ -385,14 +863,16 @@ function AppInner() { !canAcceptUserTranscript({ activeSession: isSessionActive, canListenOnRoute: canListenOnRouteRef.current, - workflowState: snapshot.state, + workflowState: currentSnapshot.state, transcript, }) ) return - const acceptedTurn = acceptTranscriptTurn({ snapshot, transcript, messages }) + const acceptedTurn = acceptTranscriptTurn({ snapshot: currentSnapshot, transcript, messages: currentMessages }) const nextMessages = acceptedTurn.messages let nextSnapshot = acceptedTurn.snapshot + snapshotRef.current = nextSnapshot + messagesRef.current = nextMessages setSnapshot(nextSnapshot) setMessages(nextMessages) setAudioUrl(null) @@ -401,22 +881,29 @@ function AppInner() { playbackEndedAtMsRef.current = null pendingNativeTranscriptRef.current = '' clearResumeListeningTimer() - await speechCancelListeningRef.current() + await cancelListeningForReason('submit_turn') setBusy(true) + const turnRequestId = ++turnRequestRef.current try { setStatus('session.status.requesting_reply') nextSnapshot = requestCoachReply(nextSnapshot) + snapshotRef.current = nextSnapshot setSnapshot(nextSnapshot) - const coachReply = await api.generateCoachReply({ + const coachReply = await withTimeout(api.generateCoachReply({ messages: nextMessages, context: { scenario: { name: scenario.name, description: scenario.description }, accentProfile: { name: accent.name, region: accent.region }, sessionId: nextSnapshot.sessionId, turnNumber: nextMessages.filter(message => message.role === 'user').length, + responseLocale: localeRef.current, }, - }) + }), 20_000, 'Coach reply request timed out.') + if (turnRequestRef.current !== turnRequestId || !sessionActiveRef.current) { + logVoiceMetric('coach_reply_ignored', { reason: 'session_inactive' }) + return + } logVoiceMetric('coach_reply_ready', { elapsedMs: Date.now() - submitStartedAt, chars: coachReply.text.length, @@ -429,15 +916,24 @@ function AppInner() { corrections: coachReply.corrections, }) nextSnapshot = coachTurn.snapshot + snapshotRef.current = nextSnapshot + messagesRef.current = coachTurn.messages setMessages(coachTurn.messages) setSnapshot(nextSnapshot) setStatus('session.status.requesting_voice') if (!coachReply.text.trim()) { setStatus('session.status.reply_without_text') + if (sessionActiveRef.current && canListenOnRouteRef.current) { + scheduleResumeListening(500) + } + return + } + const voice = await withTimeout(synthesizeCoachSpeech(coachReply.text), 20_000, 'Coach voice request timed out.') + if (turnRequestRef.current !== turnRequestId || !sessionActiveRef.current) { + logVoiceMetric('tts_ignored', { reason: 'session_inactive' }) return } - const voice = await synthesizeCoachSpeech(coachReply.text) logVoiceMetric('tts_ready', { elapsedMs: Date.now() - submitStartedAt, hasAudio: Boolean(voice.audioUrl), @@ -448,7 +944,7 @@ function AppInner() { playbackStartedRef.current = false playbackEndedAtMsRef.current = null clearResumeListeningTimer() - await speechCancelListeningRef.current() + await cancelListeningForReason('playback_enqueue') setStatus('session.status.playing_reply') setPlaybackQueue(startPlaybackQueue(voice.audioUrl)) setAudioUrl(voice.audioUrl) @@ -456,11 +952,15 @@ function AppInner() { } else { playbackActiveRef.current = false setStatus('session.status.reply_without_audio') + if (sessionActiveRef.current && canListenOnRouteRef.current) { + scheduleResumeListening(500) + } } const completedTurn = completeCoachPlayback({ snapshot: nextSnapshot, corrections: coachReply.corrections, }) + snapshotRef.current = completedTurn.snapshot setSnapshot(completedTurn.snapshot) } catch (error) { const recovery = recoverSessionError({ @@ -469,19 +969,25 @@ function AppInner() { activeSession: isSessionActive, canListenOnRoute: canListenOnRouteRef.current, }) + snapshotRef.current = recovery.snapshot setSnapshot(recovery.snapshot) - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error - ? error.message - : 'Request failed' - setStatus(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_session_submit', + presentation: 'banner', + }) + logVoiceMetric('mobile_session_request_error', requestError.logData) + displayErrorFeedback(requestError, 'mobile_session_submit') + setStatus(requestError.displayMessage) + if (sessionActiveRef.current && canListenOnRouteRef.current) { + scheduleResumeListening(900) + } } finally { - setBusy(false) + if (turnRequestRef.current === turnRequestId) setBusy(false) } }, [ - accent.name, accent.region, api, audio.isRecording, busy, clearResumeListeningTimer, isSessionActive, - logVoiceMetric, messages, scenario.description, scenario.name, snapshot, synthesizeCoachSpeech, + accent.name, accent.region, api, audio.isRecording, busy, cancelListeningForReason, + clearResumeListeningTimer, isSessionActive, logVoiceMetric, scenario.description, scenario.name, + scheduleResumeListening, synthesizeCoachSpeech, ]) const handleNativeFinalTranscript = useCallback(async (finalTranscript: string) => { @@ -493,16 +999,18 @@ function AppInner() { .filter(Boolean) .join(' ') - if (!isSessionActive) { + if (!sessionActiveRef.current) { logVoiceMetric('transcript_ignored_inactive', { chars: transcript.length }) setStatus('session.status.speech_captured') return } + const currentSnapshot = snapshotRef.current + const currentMessages = messagesRef.current const transcriptGate = gateUserTranscript({ activeSession: sessionActiveRef.current, canListenOnRoute: canListenOnRouteRef.current, - workflowState: snapshot.state, + workflowState: currentSnapshot.state, transcript: endpointTranscript, playbackActive: playbackActiveRef.current, audioPlaying: audio.isPlaying, @@ -516,7 +1024,7 @@ function AppInner() { chars: endpointTranscript.length, }) if (transcriptGate.reason === 'playback_active') { - void speechCancelListeningRef.current() + void cancelListeningForReason('transcript_gate_playback_active') } pendingNativeTranscriptRef.current = '' return @@ -524,7 +1032,7 @@ function AppInner() { const echoGuard = shouldIgnoreLikelyPlaybackEcho({ transcript: endpointTranscript, - lastAssistantResponse: snapshot.lastResponse, + lastAssistantResponse: currentSnapshot.lastResponse, playbackEndedAtMs: playbackEndedAtMsRef.current, nowMs: Date.now(), }) @@ -534,7 +1042,7 @@ function AppInner() { chars: endpointTranscript.length, }) pendingNativeTranscriptRef.current = '' - setStatus('session.status.listening') + setStatus(listeningStartupStatus()) if (!playbackActiveRef.current && !audioPlayingRef.current) { void speechStartListeningRef.current('en-US') } @@ -547,14 +1055,16 @@ function AppInner() { const endpointResult = await judgeEndpoint({ transcript: endpointTranscript, listeningDurationMs: Date.now() - listeningStartMsRef.current, - messages, + messages: currentMessages, scenario: scenario.key, semanticCheck: auth.state === 'signed-in' ? async (t, ctx) => { - const res = await fetch(`${baseUrl}/api/semantic-endpoint`, { + const authHeaders = await getAuthHeaders() + const res = await fetchWithTimeout(fetch, `${baseUrl}/api/semantic-endpoint`, { method: 'POST', - headers: { 'Content-Type': 'application/json', ...getAuthHeaders() }, + headers: { 'Content-Type': 'application/json', 'X-MeteorVoice-Client': 'meteorvoice-mobile', ...authHeaders }, body: JSON.stringify({ transcript: t, messages: ctx.messages, scenario: ctx.scenario }), }) + if (res.status === 401) await handleUnauthorized() if (!res.ok) throw new Error('Semantic check failed') const data = await res.json() as { judgment: 'done' | 'thinking' } return data.judgment @@ -569,7 +1079,7 @@ function AppInner() { if (endpointResult.judgment === 'continue') { pendingNativeTranscriptRef.current = endpointTranscript - setStatus('session.status.listening') + setStatus(listeningStartupStatus()) if (!playbackActiveRef.current && !audioPlayingRef.current) { void speechStartListeningRef.current('en-US') } @@ -579,14 +1089,21 @@ function AppInner() { pendingNativeTranscriptRef.current = '' logVoiceMetric('submit_turn_start', { chars: endpointTranscript.length }) void submitTurn(endpointTranscript) - }, [apiBaseUrl, audio.isPlaying, auth.state, getAuthHeaders, isSessionActive, logVoiceMetric, messages, scenario.key, snapshot.lastResponse, snapshot.state, submitTurn]) + }, [ + apiBaseUrl, audio.isPlaying, auth.state, cancelListeningForReason, getAuthHeaders, handleUnauthorized, + listeningStartupStatus, logVoiceMetric, scenario.key, submitTurn, + ]) const handleListeningEndedWithoutTranscript = useCallback(() => { if (!sessionActiveRef.current || !canListenOnRouteRef.current || busy || playbackActiveRef.current || audioPlayingRef.current) { logVoiceMetric('stt_end_restart_skipped', { + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + activeTab: activeTabRef.current, busy, playbackActive: playbackActiveRef.current, audioPlaying: audioPlayingRef.current, + provider: sessionSttProviderRef.current, }) return } @@ -594,6 +1111,359 @@ function AppInner() { scheduleResumeListening(250, false) }, [busy, logVoiceMetric, scheduleResumeListening]) + const cancelXunfeiSessionListening = useCallback(async (reason = 'cancel') => { + const current = xunfeiSessionSttRef.current + if (!current || current.settled) return + current.settled = true + if (current.finalizeTimer) clearTimeout(current.finalizeTimer) + if (current.hardTimer) clearTimeout(current.hardTimer) + if (current.noFrameTimer) clearTimeout(current.noFrameTimer) + current.frameSubscription?.remove() + current.stateSubscription?.remove() + if (current.socket && current.socket.readyState === WebSocket.OPEN) { + current.socket.close() + } + await stopPcmCapture(`session_${reason}`).catch(() => undefined) + xunfeiSessionSttRef.current = null + logVoiceMetric('stt_end', { provider: 'xunfei', cancelled: true, reason }) + }, [logVoiceMetric]) + + const startXunfeiSessionListening = useCallback(async () => { + const canUseXunfeiRoute = (context: string) => { + const allowed = sessionActiveRef.current && + canListenOnRouteRef.current && + !busyRef.current && + !playbackActiveRef.current && + !audioPlayingRef.current && + activeTabRef.current === 'session' + if (!allowed) { + logVoiceMetric('stt_start_aborted', { + provider: 'xunfei', + context, + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + activeTab: activeTabRef.current, + busy: busyRef.current, + playbackActive: playbackActiveRef.current, + audioPlaying: audioPlayingRef.current, + }) + } + return allowed + } + + if (!canUseXunfeiRoute('entry')) return false + if (xunfeiSessionSttRef.current && !xunfeiSessionSttRef.current.settled) return true + if (!availableSessionSttProviders.includes('xunfei') || !isPcmCaptureAvailable()) { + logVoiceMetric('stt_provider_fallback', { + requested: 'xunfei', + reason: !isPcmCaptureAvailable() ? 'pcm_unavailable' : 'provider_unavailable', + }) + return nativeSpeechStartListeningRef.current('en-US') + } + if (auth.state !== 'signed-in') return nativeSpeechStartListeningRef.current('en-US') + + const streamStartedAt = Date.now() + let socket: WebSocket | null = null + let frameSubscription: { remove: () => void } | null = null + let stateSubscription: { remove: () => void } | null = null + let finalizeTimer: ReturnType | null = null + let hardTimer: ReturnType | null = null + let noFrameTimer: ReturnType | null = null + let settled = false + let firstFrame = true + let finalFrameSent = false + let finalReceived = false + let audioSequence = 0 + let frameCount = 0 + let totalBytes = 0 + let transcript = '' + let firstPartialAt: number | null = null + const transcriptSegments: string[] = [] + + const updateCurrent = () => { + xunfeiSessionSttRef.current = { + socket, + frameSubscription, + stateSubscription, + finalizeTimer, + hardTimer, + noFrameTimer, + settled, + } + } + + const clearNoFrameTimer = () => { + if (!noFrameTimer) return + clearTimeout(noFrameTimer) + noFrameTimer = null + updateCurrent() + } + + const settle = (reason: string, submitted: boolean) => { + if (settled) return + settled = true + updateCurrent() + if (finalizeTimer) clearTimeout(finalizeTimer) + if (hardTimer) clearTimeout(hardTimer) + if (noFrameTimer) clearTimeout(noFrameTimer) + frameSubscription?.remove() + void stopPcmCapture(`session_${reason}`).catch(() => undefined).finally(() => stateSubscription?.remove()) + if (socket && socket.readyState === WebSocket.OPEN) socket.close() + xunfeiSessionSttRef.current = null + logVoiceMetric('stt_end', { + provider: 'xunfei', + cancelled: reason !== 'final', + reason, + hadTranscript: Boolean(transcript.trim()), + submitted, + elapsedMs: Date.now() - streamStartedAt, + frameCount, + totalBytes, + }) + } + + const sendAudioFrame = (status: 0 | 1 | 2, audioBase64: string, session: CreateASRSessionResponse) => { + if (!socket || socket.readyState !== WebSocket.OPEN || finalFrameSent) return + if (status === 2) finalFrameSent = true + audioSequence += 1 + socket.send(JSON.stringify(createXunfeiASRFrame(session, status, audioBase64, audioSequence))) + } + + try { + const authReady = await auth.refreshSession() + if (!canUseXunfeiRoute('after_auth_refresh')) return false + if (!authReady) return nativeSpeechStartListeningRef.current('en-US') + + const session = await api.createASRSession({ + provider: 'xunfei', + mode: 'streaming', + languageMode: 'mixed_zh_en', + scenarioKey: scenario.key, + sessionId: snapshotRef.current.sessionId, + endpointSilenceMs: 900, + clientTraceId: `mobile-session-${Date.now()}`, + }) + if (!canUseXunfeiRoute('after_session_create')) { + logVoiceMetric('stt_start_stale_after_bootstrap', { provider: 'xunfei' }) + return false + } + if (session.provider !== 'xunfei' || session.status !== 'created' || session.transport !== 'websocket' || !session.endpointUrl) { + logVoiceMetric('stt_provider_fallback', { requested: 'xunfei', reason: 'session_not_ready' }) + return nativeSpeechStartListeningRef.current('en-US') + } + + logVoiceMetric('stt_bootstrap_start', { provider: 'xunfei' }) + setStatus('session.status.preparing_listening') + await nativeSpeechCancelListeningRef.current() + + socket = new WebSocket(session.endpointUrl) + updateCurrent() + + const finishAudio = () => { + if (!canUseXunfeiRoute('finish_audio')) { + settle('route_inactive', false) + return + } + if (finalFrameSent) return + sendAudioFrame(2, '', session) + void stopPcmCapture('session_endpoint').catch(() => undefined) + } + + const scheduleFinalize = () => { + if (finalizeTimer) clearTimeout(finalizeTimer) + finalizeTimer = setTimeout(finishAudio, session.providerConfig?.eosMs ?? 900) + updateCurrent() + } + + stateSubscription = addPcmStateListener(event => { + logVoiceMetric('stt_pcm_state', { + provider: 'xunfei', + state: event.state, + frameCount: event.frameCount, + totalBytes: event.totalBytes, + message: event.message, + }) + }) + + frameSubscription = addPcmFrameListener((event: PcmCaptureFrameEvent) => { + if (!canUseXunfeiRoute('pcm_frame')) { + settle('route_inactive', false) + return + } + if (!socket || socket.readyState !== WebSocket.OPEN || finalFrameSent) return + frameCount += 1 + totalBytes += event.byteCount + if (frameCount === 1) clearNoFrameTimer() + sendAudioFrame(firstFrame ? 0 : 1, event.audioBase64, session) + firstFrame = false + if (frameCount === 1 || frameCount % 50 === 0) { + logVoiceMetric('stt_pcm_frame', { + provider: 'xunfei', + frameCount, + totalBytes, + elapsedMs: event.elapsedMs, + }) + } + }) + + socket.onopen = () => { + if (!canUseXunfeiRoute('socket_open')) { + settle('route_inactive', false) + return + } + void startPcmCapture({ + sampleRate: session.providerConfig?.sampleRate ?? 16000, + frameDurationMs: session.providerConfig?.frameIntervalMs ?? 40, + }).then(status => { + if (!canUseXunfeiRoute('after_pcm_start')) { + void stopPcmCapture('session_route_inactive').catch(() => undefined) + settle('route_inactive', false) + return + } + listeningStartMsRef.current = Date.now() + logVoiceMetric('stt_start', { provider: 'xunfei' }) + logVoiceMetric('stt_ready', { + provider: 'xunfei', + elapsedMs: Date.now() - streamStartedAt, + sampleRate: status.sampleRate, + frameSizeBytes: status.frameSizeBytes, + }) + setStatus('session.status.listening') + noFrameTimer = setTimeout(() => { + if (settled || frameCount > 0) return + logVoiceMetric('stt_pcm_no_frame', { + provider: 'xunfei', + elapsedMs: Date.now() - streamStartedAt, + pcmStatusFrameCount: status.frameCount, + pcmStatusTotalBytes: status.totalBytes, + }) + settle('pcm_no_frame', false) + handleListeningEndedWithoutTranscript() + }, 1800) + updateCurrent() + }).catch(error => { + logVoiceMetric('stt_provider_error', { + provider: 'xunfei', + message: error instanceof Error ? error.message : 'PCM capture failed', + }) + settle('pcm_error', false) + handleListeningEndedWithoutTranscript() + }) + hardTimer = setTimeout(finishAudio, 15_000) + updateCurrent() + } + + socket.onmessage = event => { + if (!canUseXunfeiRoute('socket_message')) { + settle('route_inactive', false) + return + } + const payload = parseJsonObject(event.data) + const header = getObject(payload?.header) + const code = typeof header?.code === 'number' + ? header.code + : typeof payload?.code === 'number' + ? payload.code + : 0 + if (code !== 0) { + const message = typeof header?.message === 'string' + ? header.message + : typeof payload?.message === 'string' + ? payload.message + : `Xunfei ASR error ${code}` + logVoiceMetric('stt_provider_error', { provider: 'xunfei', code, message }) + settle('provider_error', false) + handleListeningEndedWithoutTranscript() + return + } + + const recognitionResult = extractXunfeiRecognitionResult(payload) + if (recognitionResult?.text) { + if (recognitionResult.pgs === 'rpl' && recognitionResult.rg) { + const [start, end] = recognitionResult.rg + for (let index = start; index <= end; index += 1) transcriptSegments[index] = '' + } + if (recognitionResult.sn != null) { + transcriptSegments[recognitionResult.sn] = recognitionResult.text + } else { + transcriptSegments.push(recognitionResult.text) + } + transcript = transcriptSegments.filter(Boolean).join('').trim() + if (!firstPartialAt) { + firstPartialAt = Date.now() + logVoiceMetric('stt_first_partial', { + provider: 'xunfei', + elapsedMs: firstPartialAt - streamStartedAt, + chars: transcript.length, + }) + } + logVoiceMetric('stt_partial', { provider: 'xunfei', chars: transcript.length }) + scheduleFinalize() + } + + const data = getObject(payload?.data) + const status = typeof header?.status === 'number' + ? header.status + : typeof data?.status === 'number' + ? data.status + : undefined + if (status === 2) { + finalReceived = true + const normalized = transcript.trim() + if (normalized) { + logVoiceMetric('stt_submit', { + provider: 'xunfei', + source: 'xunfei_final', + chars: normalized.length, + elapsedMs: Date.now() - streamStartedAt, + }) + void handleNativeFinalTranscript(normalized) + settle('final', true) + } else { + settle('final', false) + handleListeningEndedWithoutTranscript() + } + } + } + + socket.onerror = () => { + logVoiceMetric('stt_provider_error', { provider: 'xunfei', message: 'WebSocket error' }) + settle('socket_error', false) + handleListeningEndedWithoutTranscript() + } + + socket.onclose = event => { + logVoiceMetric('stt_socket_close', { + provider: 'xunfei', + code: typeof event?.code === 'number' ? event.code : null, + reason: typeof event?.reason === 'string' ? event.reason : '', + wasClean: Boolean(event?.wasClean), + finalReceived, + finalFrameSent, + frameCount, + totalBytes, + }) + if (!finalReceived && !settled) { + settle('socket_closed', false) + handleListeningEndedWithoutTranscript() + } + } + + updateCurrent() + return true + } catch (error) { + logVoiceMetric('stt_provider_error', { + provider: 'xunfei', + message: error instanceof Error ? error.message : 'Xunfei STT failed to start', + }) + settle('start_error', false) + return nativeSpeechStartListeningRef.current('en-US') + } + }, [ + api, auth, availableSessionSttProviders, handleListeningEndedWithoutTranscript, + handleNativeFinalTranscript, logVoiceMetric, scenario.key, + ]) + const speech = useNativeSpeech({ onFinalTranscript: handleNativeFinalTranscript, onListeningEndedWithoutTranscript: handleListeningEndedWithoutTranscript, @@ -601,15 +1471,42 @@ function AppInner() { }) useEffect(() => { - speechStartListeningRef.current = speech.startListening - speechCancelListeningRef.current = speech.cancelListening + nativeSpeechStartListeningRef.current = speech.startListening + nativeSpeechCancelListeningRef.current = speech.cancelListening }, [speech.cancelListening, speech.startListening]) + useEffect(() => { + speechStartListeningRef.current = sessionSttProvider === 'xunfei' + ? startXunfeiSessionListening + : speech.startListening + speechCancelListeningRef.current = sessionSttProvider === 'xunfei' + ? cancelXunfeiSessionListening + : speech.cancelListening + startListeningWithProviderRef.current = (provider, lang) => ( + provider === 'xunfei' + ? startXunfeiSessionListening() + : speech.startListening(lang) + ) + }, [ + cancelXunfeiSessionListening, sessionSttProvider, speech, speech.cancelListening, speech.startListening, + startXunfeiSessionListening, + ]) + useEffect(() => { sessionActiveRef.current = isSessionActive }, [isSessionActive]) const selectTab = useCallback((tab: Tab) => { + logUserAction('tab_tap', { to: tab, from: activeTabRef.current }) + logVoiceMetric('tab_change', { + from: activeTabRef.current, + to: tab, + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + busy, + playbackActive: playbackActiveRef.current, + audioPlaying: audioPlayingRef.current, + }) setActiveTab(tab) if (tab !== 'session') { canListenOnRouteRef.current = false @@ -617,7 +1514,7 @@ function AppInner() { clearResumeListeningTimer() endpointRequestRef.current += 1 pendingNativeTranscriptRef.current = '' - void speechCancelListeningRef.current() + void cancelListeningForReason(`tab:${tab}`) if (sessionActiveRef.current) setStatus('session.paused') return } @@ -625,29 +1522,43 @@ function AppInner() { canListenOnRouteRef.current = true if (sessionActiveRef.current && !busy && !playbackActiveRef.current && !audioPlayingRef.current) { listeningStartMsRef.current = Date.now() - setStatus('session.status.listening') + setStatus(listeningStartupStatus()) void speechStartListeningRef.current('en-US') } - }, [busy, clearResumeListeningTimer]) + }, [busy, cancelListeningForReason, clearResumeListeningTimer, listeningStartupStatus, logUserAction, logVoiceMetric]) async function endSession() { - if (!canEndSession({ activeSession: isSessionActive, workflowState: snapshot.state }) || busy) return + logUserAction('session_stop_tap') + if (!canEndSession({ activeSession: isSessionActive, workflowState: snapshot.state })) return + logVoiceMetric('session_end_requested', { + sessionId: snapshot.sessionId, + state: snapshot.state, + messages: messages.length, + activeTab: activeTabRef.current, + pendingTeardown: Boolean(listeningTeardownRef.current), + }) // 立即结束 session，不等 API + turnRequestRef.current += 1 sessionActiveRef.current = false canListenOnRouteRef.current = false playbackActiveRef.current = false + audioPlayingRef.current = false + playbackStartedRef.current = false playbackEndedAtMsRef.current = null clearResumeListeningTimer() endpointRequestRef.current += 1 pendingNativeTranscriptRef.current = '' - void speechCancelListeningRef.current() + audio.stopPlayback() + setAudioUrl(null) + setPlaybackQueue(createPlaybackQueueSnapshot()) + void cancelListeningForReason('session_end') const endedSnapshot = endActiveSession(snapshot).snapshot setSnapshot(endedSnapshot) setIsSessionActive(false) setStatus('session.ended') + setBusy(false) - setBusy(true) const userTurns = messages.filter(m => m.role === 'user').length try { const result = await api.generateSummary({ @@ -667,25 +1578,60 @@ function AppInner() { }).catch(() => undefined) } catch { // summary 失败不影响 session 已结束的状态 - } finally { - setBusy(false) } } - function selectScenario(key: string) { - setSelectedScenarioKey(key) - setMessages([]) - setCorrectionHistory([]) - setAudioUrl(null) - setPlaybackQueue(createPlaybackQueueSnapshot()) - setSummary(null) - setSnapshot(createInitialSnapshot('mobile-session')) - setIsSessionActive(false) - setStatus('session.status.scenario_selected') + async function selectScenario(key: string) { + if (scenarioSwitching) { + logUserAction('scenario_tap_ignored_switching', { to: key }) + return + } + logUserAction('scenario_tap', { to: key, from: selectedScenarioKey }) + logVoiceMetric('scenario_select_requested', { + from: selectedScenarioKey, + to: key, + activeTab: activeTabRef.current, + sessionActive: sessionActiveRef.current, + canListenOnRoute: canListenOnRouteRef.current, + pendingTeardown: Boolean(listeningTeardownRef.current), + }) + setScenarioSwitching(true) + setStatus('session.status.switching_session') + try { + turnRequestRef.current += 1 + endpointRequestRef.current += 1 + sessionActiveRef.current = false + canListenOnRouteRef.current = false + playbackActiveRef.current = false + audioPlayingRef.current = false + playbackStartedRef.current = false + playbackEndedAtMsRef.current = null + clearResumeListeningTimer() + pendingNativeTranscriptRef.current = '' + audio.stopPlayback() + setBusy(false) + await cancelListeningForReason('scenario_change') + setSelectedScenarioKey(key) + setMessages([]) + setCorrectionHistory([]) + setAudioUrl(null) + setPlaybackQueue(createPlaybackQueueSnapshot()) + setSummary(null) + setSnapshot(createInitialSnapshot('mobile-session')) + setIsSessionActive(false) + setStatus('session.status.scenario_selected') + logVoiceMetric('scenario_selected', { key }) + } finally { + setScenarioSwitching(false) + } } - async function loadHistory() { + const loadHistory = useCallback(async () => { if (historyLoading) return + if (auth.state !== 'signed-in') { + setHistoryError(tr('history.auth_required')) + return + } setHistoryLoading(true) setHistoryError(null) try { @@ -694,21 +1640,37 @@ function AppInner() { setSelectedHistory(result.sessions[0] ?? null) setSelectedHistoryTurns([]) } catch (error) { - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error ? error.message : 'History request failed' - setHistoryError(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_history_list', + presentation: 'inline', + }) + setHistoryError(requestError.displayMessage) } finally { setHistoryLoading(false) } - } + }, [api, auth.state, historyLoading, tr]) + + useEffect(() => { + if (activeTab !== 'history') return + if (auth.state !== 'signed-in') { + historyAutoLoadRef.current = false + return + } + if (historyAutoLoadRef.current) return + historyAutoLoadRef.current = true + void loadHistory() + }, [activeTab, auth.state, loadHistory]) async function deleteSession(id: string) { setHistorySessions(prev => prev.map(s => s.id === id ? { ...s, status: 'deleted' } : s)) try { - await fetch(`${apiBaseUrl.trim()}/api/session?id=${encodeURIComponent(id)}`, { + const authHeaders = await getAuthHeaders() + await fetchWithTimeout(fetch, `${apiBaseUrl.trim()}/api/session?id=${encodeURIComponent(id)}`, { method: 'DELETE', - headers: getAuthHeaders() as Record, + headers: authHeaders as Record, + }).then(res => { + if (res.status === 401) return handleUnauthorized() + return undefined }) } catch { // 静默失败 @@ -722,103 +1684,290 @@ function AppInner() { const result = await api.listSessionTurns(item.id) setSelectedHistoryTurns(result.turns) } catch (error) { - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error ? error.message : 'Turn detail request failed' - setHistoryError(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_history_turns', + presentation: 'inline', + }) + setHistoryError(requestError.displayMessage) } } - async function loadPreferences() { - if (settingsLoading) return - setSettingsLoading(true) + const applyPreferences = useCallback((preferences: PreferencesResponse, successMessage?: string) => { + setLocale(preferences.locale === 'zh' ? 'zh' : 'en') + setTtsProvider(preferences.tts_provider ?? 'mock') + setAvailableProviders(preferences.available_providers?.length ? preferences.available_providers : ['mock']) + setTtsSpeed(preferences.tts_speed ?? 1) + if (preferences.tts_voice_id !== undefined) setTtsVoiceId(preferences.tts_voice_id) + if (preferences.voice_profiles) setVoiceProfiles(preferences.voice_profiles) + if (preferences.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(preferences.selected_voice_profile_id) + if (preferences.xunfei_voices?.configured) setXunfeiVoices(preferences.xunfei_voices.configured) + if (preferences.default_scenario_key) setSelectedScenarioKey(preferences.default_scenario_key) + const profile = preferences.voice_profiles?.find(item => item.id === preferences.selected_voice_profile_id) + if (profile) setSelectedAccentKey(profile.accentKey) + if (preferences.ui_theme && !themeInitializedRef.current) { + themeInitializedRef.current = true + void SecureStore.getItemAsync('theme_set_at').then(localSetAt => { + const serverTs = new Date(preferences.ui_theme_updated_at ?? new Date(0).toISOString()).getTime() + const localTs = localSetAt ? new Date(localSetAt).getTime() : 0 + if (serverTs >= localTs) { + applyThemeLocal(preferences.ui_theme as Parameters[0]) + } + }) + } + setSettingsMessage(successMessage ?? tr('session.status.preferences_loaded')) + }, [applyThemeLocal, setLocale, tr]) + + const applyTtsPreferences = useCallback((preferences: PreferencesResponse, successMessage = tr('session.status.preferences_saved')) => { + setTtsProvider(preferences.tts_provider ?? 'mock') + setTtsSpeed(preferences.tts_speed ?? 1) + if (preferences.tts_voice_id !== undefined) setTtsVoiceId(preferences.tts_voice_id) + if (preferences.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(preferences.selected_voice_profile_id) + const profiles = preferences.voice_profiles ?? voiceProfiles + const profile = profiles.find(item => item.id === preferences.selected_voice_profile_id) + if (profile) setSelectedAccentKey(profile.accentKey) + setSettingsMessage(successMessage) + }, [tr, voiceProfiles]) + + const applyPracticePreferences = useCallback((preferences: PreferencesResponse, successMessage = tr('session.status.practice_defaults_saved')) => { + setTtsProvider(preferences.tts_provider ?? 'mock') + setTtsSpeed(preferences.tts_speed ?? 1) + if (preferences.default_scenario_key) setSelectedScenarioKey(preferences.default_scenario_key) + setSettingsMessage(successMessage) + }, [tr]) + + const applyVoiceProfilePreferences = useCallback((preferences: PreferencesResponse, successMessage = tr('session.status.preferences_saved')) => { + setTtsProvider(preferences.tts_provider ?? 'mock') + if (preferences.tts_voice_id !== undefined) setTtsVoiceId(preferences.tts_voice_id) + if (preferences.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(preferences.selected_voice_profile_id) + const profiles = preferences.voice_profiles ?? voiceProfiles + const profile = profiles.find(item => item.id === preferences.selected_voice_profile_id) + if (profile) setSelectedAccentKey(profile.accentKey) + setSettingsMessage(successMessage) + }, [tr, voiceProfiles]) + + const applyLocalePreferences = useCallback((preferences: PreferencesResponse, successMessage = tr('session.status.preferences_saved')) => { + setLocale(preferences.locale === 'zh' ? 'zh' : 'en') + setSettingsMessage(successMessage) + }, [setLocale, tr]) + + const fetchSessionSttProviders = useCallback(async () => { + const result = await api.listASRProviders() + const providers: SessionSttProvider[] = ['native'] + if (result.providers.some(provider => provider.key === 'xunfei' && provider.enabled)) { + providers.push('xunfei') + } + return providers + }, [api]) + + const applySessionSttProviders = useCallback((providers: SessionSttProvider[]) => { + setAvailableSessionSttProviders(providers) + if (!providers.includes(sessionSttProvider)) { + sessionSttProviderRef.current = 'native' + setSessionSttProviderState('native') + void SecureStore.setItemAsync(sessionSttProviderStorageKey, 'native') + } + }, [sessionSttProvider]) + + const loadSessionSttProviders = useCallback(async () => { + try { + const providers = await fetchSessionSttProviders() + applySessionSttProviders(providers) + sessionSttProvidersLoadedRef.current = true + } catch (error) { + const requestError = formatApiRequestError(error, { + context: 'mobile_asr_providers_load', + presentation: 'silent', + }) + logVoiceMetric('mobile_silent_request_error', requestError.logData) + } + }, [applySessionSttProviders, fetchSessionSttProviders, logVoiceMetric]) + + useEffect(() => appFeedback.subscribe(setActiveFeedback), []) + + useEffect(() => { + if (settingsLoading && activeTab === 'settings') { + appFeedback.show({ + message: tr('settings.syncing'), + variant: 'hud', + source: 'settings', + }) + return + } + appFeedback.hide('settings') + }, [activeTab, settingsLoading, tr]) + + useEffect(() => { + if (authSubmitting && activeTab === 'settings') { + appFeedback.show({ + message: tr('login.loading'), + variant: 'hud', + source: 'auth', + }) + return + } + appFeedback.hide('auth') + }, [activeTab, authSubmitting, tr]) + + useEffect(() => { + if (scenarioSwitching) { + appFeedback.show({ + message: tr('session.status.switching_session'), + variant: 'hud', + source: 'session-transition', + }) + return + } + appFeedback.hide('session-transition') + }, [scenarioSwitching, tr]) + + const loadPreferences = useCallback(async (options: { force?: boolean; successMessage?: string } = {}) => { + if (settingsLoadingRef.current && !options.force) return + if (auth.state !== 'signed-in') { + setSettingsMessage(tr('settings.auth_required')) + return + } + const requestId = ++settingsRequestRef.current + setSettingsLoadingFlag(true) setSettingsMessage(null) try { const preferences = await api.getPreferences() - setLocale(preferences.locale === 'zh' ? 'zh' : 'en') - setTtsProvider(preferences.tts_provider ?? 'mock') - setAvailableProviders(preferences.available_providers?.length ? preferences.available_providers : ['mock']) - setTtsSpeed(preferences.tts_speed ?? 1) - if (preferences.tts_voice_id !== undefined) setTtsVoiceId(preferences.tts_voice_id) - if (preferences.voice_profiles) setVoiceProfiles(preferences.voice_profiles) - if (preferences.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(preferences.selected_voice_profile_id) - if (preferences.xunfei_voices?.configured) setXunfeiVoices(preferences.xunfei_voices.configured) - if (preferences.default_scenario_key) setSelectedScenarioKey(preferences.default_scenario_key) - const profile = preferences.voice_profiles?.find(item => item.id === preferences.selected_voice_profile_id) - if (profile) setSelectedAccentKey(profile.accentKey) - setSettingsMessage(tr('session.status.preferences_loaded')) + if (requestId !== settingsRequestRef.current) return + applyPreferences(preferences, options.successMessage) } catch (error) { - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error ? error.message : 'Preferences request failed' - setSettingsMessage(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_preferences_load', + presentation: 'inline', + }) + setSettingsMessage(requestError.displayMessage) } finally { - setSettingsLoading(false) + if (requestId === settingsRequestRef.current) { + setSettingsLoadingFlag(false) + } } - } + }, [api, applyPreferences, auth.state, setSettingsLoadingFlag, tr]) + + const loadSettingsDataGroup = useCallback(() => { + if (settingsLoadingRef.current) return () => undefined + if (auth.state !== 'signed-in') { + setSettingsMessage(tr('settings.auth_required')) + return () => undefined + } + + let cancelled = false + const requestId = ++settingsRequestRef.current + setSettingsLoadingFlag(true) + setSettingsMessage(null) + + void runAppOperationGroup({ + source: 'mobile_settings_data', + tasks: { + preferences: () => api.getPreferences(), + providers: fetchSessionSttProviders, + }, + }).then(({ preferences: preferencesResult, providers: providersResult }) => { + if (cancelled || requestId !== settingsRequestRef.current) return + + if (preferencesResult.status === 'fulfilled') { + applyPreferences(preferencesResult.value) + } else { + const requestError = formatApiRequestError(preferencesResult.reason, { + context: 'mobile_preferences_load', + presentation: 'inline', + }) + setSettingsMessage(requestError.displayMessage) + } + + if (providersResult.status === 'fulfilled') { + applySessionSttProviders(providersResult.value) + } else { + const requestError = formatApiRequestError(providersResult.reason, { + context: 'mobile_asr_providers_load', + presentation: 'silent', + }) + logVoiceMetric('mobile_silent_request_error', requestError.logData) + } + }).finally(() => { + if (!cancelled && requestId === settingsRequestRef.current) { + setSettingsLoadingFlag(false) + } + }) + + return () => { + cancelled = true + } + }, [ + api, applyPreferences, applySessionSttProviders, auth.state, + fetchSessionSttProviders, logVoiceMetric, setSettingsLoadingFlag, tr, + ]) + + const reloadSettingsData = useCallback(() => loadSettingsDataGroup(), [loadSettingsDataGroup]) + + useEffect(() => { + if (auth.state !== 'signed-in') { + settingsAutoLoadRef.current = false + return + } + if (activeTab !== 'settings' || settingsAutoLoadRef.current) return + settingsAutoLoadRef.current = true + return reloadSettingsData() + }, [activeTab, auth.state, reloadSettingsData]) async function saveProvider(provider: string) { + settingsRequestRef.current += 1 setTtsProvider(provider) setAudioUrl(null) playbackEndedAtMsRef.current = null - setSettingsLoading(true) + setSettingsLoadingFlag(true) setSettingsMessage(null) if (auth.state !== 'signed-in') { setSettingsMessage(tr('session.status.preferences_saved')) - setSettingsLoading(false) + setSettingsLoadingFlag(false) return } try { - const result = await api.updatePreferences({ + const preferences = await api.updatePreferences({ tts_provider: provider, default_scenario_key: selectedScenarioKey, tts_speed: ttsSpeed, }) - setTtsProvider(result.tts_provider) - setTtsSpeed(result.tts_speed) - if (result.voice_profiles) setVoiceProfiles(result.voice_profiles) - if (result.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(result.selected_voice_profile_id) - const profile = result.voice_profiles?.find(item => item.id === result.selected_voice_profile_id) - if (profile) setSelectedAccentKey(profile.accentKey) - setSettingsMessage(tr('session.status.preferences_saved')) + applyTtsPreferences(preferences) } catch (error) { - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error ? error.message : 'Preferences save failed' - setSettingsMessage(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_preferences_save_provider', + presentation: 'inline', + }) + setSettingsMessage(requestError.displayMessage) } finally { - setSettingsLoading(false) + setSettingsLoadingFlag(false) } } async function savePracticePreferences() { - setSettingsLoading(true) + settingsRequestRef.current += 1 + setSettingsLoadingFlag(true) setSettingsMessage(null) if (auth.state !== 'signed-in') { setSettingsMessage(tr('session.status.practice_defaults_saved')) - setSettingsLoading(false) + setSettingsLoadingFlag(false) return } try { - const result = await api.updatePreferences({ + const preferences = await api.updatePreferences({ tts_provider: ttsProvider, default_scenario_key: selectedScenarioKey, tts_speed: ttsSpeed, }) - setTtsProvider(result.tts_provider) - setTtsSpeed(result.tts_speed) - if (result.voice_profiles) setVoiceProfiles(result.voice_profiles) - if (result.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(result.selected_voice_profile_id) - setSettingsMessage(tr('session.status.practice_defaults_saved')) + applyPracticePreferences(preferences) } catch (error) { - const message = error instanceof MeteorVoiceApiError - ? `${error.message} (${error.status})` - : error instanceof Error ? error.message : 'Preferences save failed' - setSettingsMessage(message) + const requestError = formatApiRequestError(error, { + context: 'mobile_preferences_save_practice', + presentation: 'inline', + }) + setSettingsMessage(requestError.displayMessage) } finally { - setSettingsLoading(false) + setSettingsLoadingFlag(false) } } @@ -830,9 +1979,14 @@ function AppInner() { void syncMobilePreferences({ apiBaseUrl: apiBaseUrl.trim(), getAuthHeaders: auth.getAuthHeaders, + onUnauthorized: handleUnauthorized, ttsSpeed: next, ttsProvider, defaultScenarioKey: selectedScenarioKey, + }).then(preferences => { + if (preferences) { + applyTtsPreferences(preferences) + } }) }, 600) return next @@ -841,6 +1995,7 @@ function AppInner() { async function selectVoiceProfile(profile: VoiceProfile) { if (profile.status !== 'active') return + settingsRequestRef.current += 1 setAudioUrl(null) playbackEndedAtMsRef.current = null setSelectedVoiceProfileId(profile.id) @@ -851,106 +2006,129 @@ function AppInner() { if (auth.state !== 'signed-in') return try { - const result = await api.updatePreferences({ selected_voice_profile_id: profile.id }) - setTtsProvider(result.tts_provider) - setTtsVoiceId(result.tts_voice_id) - setSelectedVoiceProfileId(result.selected_voice_profile_id) - if (result.voice_profiles) setVoiceProfiles(result.voice_profiles) - } catch { - // 静默失败 + const preferences = await api.updatePreferences({ selected_voice_profile_id: profile.id }) + applyVoiceProfilePreferences(preferences) + } catch (error) { + const requestError = formatApiRequestError(error, { + context: 'mobile_preferences_select_voice_profile', + presentation: 'silent', + }) + logVoiceMetric('mobile_silent_request_error', requestError.logData) } } - async function submitAuth() { - const normalizedEmail = email.trim() - if (!normalizedEmail || !password || auth.state === 'loading') return - const success = await auth.submit(authMode, normalizedEmail, password) - if (success) setPassword('') - } + async function saveLocalePreference(nextLocale: Locale) { + if (nextLocale === locale) return + setLocale(nextLocale) - const applyPrefs = useCallback((prefs: Awaited>) => { - if (!prefs) return - setTtsProvider(prefs.ttsProvider) - setTtsSpeed(prefs.ttsSpeed) - setAvailableProviders(prefs.availableProviders) - setTtsVoiceId(prefs.ttsVoiceId) - setVoiceProfiles(prefs.voiceProfiles) - setSelectedVoiceProfileId(prefs.selectedVoiceProfileId) - if (prefs.xunfeiVoices.length > 0) setXunfeiVoices(prefs.xunfeiVoices) - if (prefs.defaultScenarioKey) setSelectedScenarioKey(prefs.defaultScenarioKey) - const profile = prefs.voiceProfiles.find(item => item.id === prefs.selectedVoiceProfileId) - if (profile) setSelectedAccentKey(profile.accentKey) - if (prefs.locale === 'zh' || prefs.locale === 'en') setLocale(prefs.locale) - if (prefs.uiTheme && !themeInitializedRef.current) { - themeInitializedRef.current = true - void SecureStore.getItemAsync('theme_set_at').then(localSetAt => { - const serverTs = new Date(prefs.uiThemeUpdatedAt).getTime() - const localTs = localSetAt ? new Date(localSetAt).getTime() : 0 - if (serverTs >= localTs) { - applyThemeLocal(prefs.uiTheme as Parameters[0]) - } + if (auth.state !== 'signed-in') { + setSettingsMessage(tr('settings.auth_required')) + return + } + + const requestId = ++settingsRequestRef.current + setSettingsLoadingFlag(true) + setSettingsMessage(null) + try { + const preferences = await api.updatePreferences({ locale: nextLocale }) + if (requestId !== settingsRequestRef.current) return + applyLocalePreferences(preferences) + } catch (error) { + const requestError = formatApiRequestError(error, { + context: 'mobile_preferences_save_locale', + presentation: 'banner', }) + setSettingsMessage(requestError.displayMessage) + displayErrorFeedback(requestError, 'mobile_preferences_save_locale') + } finally { + if (requestId === settingsRequestRef.current) { + setSettingsLoadingFlag(false) + } } - }, [applyThemeLocal, setLocale]) + } - const applyServerPreferences = useCallback((preferences: PreferencesResponse) => { - setTtsProvider(preferences.tts_provider ?? 'mock') - setAvailableProviders(preferences.available_providers?.length ? preferences.available_providers : ['mock']) - setTtsSpeed(preferences.tts_speed ?? 1) - if (preferences.tts_voice_id !== undefined) setTtsVoiceId(preferences.tts_voice_id) - if (preferences.voice_profiles) setVoiceProfiles(preferences.voice_profiles) - if (preferences.selected_voice_profile_id !== undefined) setSelectedVoiceProfileId(preferences.selected_voice_profile_id) - if (preferences.xunfei_voices?.configured) setXunfeiVoices(preferences.xunfei_voices.configured) - if (preferences.default_scenario_key) setSelectedScenarioKey(preferences.default_scenario_key) - const profile = preferences.voice_profiles?.find(item => item.id === preferences.selected_voice_profile_id) - if (profile) setSelectedAccentKey(profile.accentKey) - if (preferences.locale === 'zh' || preferences.locale === 'en') setLocale(preferences.locale) - }, [setLocale]) + async function submitAuth() { + const normalizedEmail = email.trim() + if (!normalizedEmail || !password || auth.state === 'loading' || authSubmitting) return + logUserAction('auth_submit_tap', { + mode: authMode, + hasEmail: Boolean(normalizedEmail), + passwordLength: password.length, + }) + setAuthSubmitting(true) + try { + const success = await auth.submit(authMode, normalizedEmail, password) + logVoiceMetric('auth_submit_done', { mode: authMode, success }) + if (success) setPassword('') + } catch (error) { + logVoiceMetric('auth_submit_error', { + mode: authMode, + message: error instanceof Error ? error.message : 'Auth submit failed', + }) + } finally { + setAuthSubmitting(false) + } + } useEffect(() => { - void api.getPreferences() - .then(applyServerPreferences) - .catch(() => undefined) - }, [api, applyServerPreferences]) + const timer = setTimeout(() => { + void loadSessionSttProviders() + }, 0) + return () => clearTimeout(timer) + }, [loadSessionSttProviders]) // 登录后自动拉取偏好 useEffect(() => { if (auth.state !== 'signed-in') return - void pullMobilePreferences(apiBaseUrl.trim(), auth.getAuthHeaders).then(applyPrefs) - }, [auth.state, apiBaseUrl, auth.getAuthHeaders, applyPrefs]) + const timer = setTimeout(() => { + loadSettingsDataGroup() + }, 0) + return () => clearTimeout(timer) + }, [auth.state, loadSettingsDataGroup]) useEffect(() => { const subscription = AppState.addEventListener('change', (nextState: AppStateStatus) => { if (nextState !== 'active') { + logVoiceMetric('app_state_inactive', { + nextState, + sessionActive: sessionActiveRef.current, + activeTab: activeTabRef.current, + }) canListenOnRouteRef.current = false playbackEndedAtMsRef.current = null clearResumeListeningTimer() endpointRequestRef.current += 1 pendingNativeTranscriptRef.current = '' - void speechCancelListeningRef.current() + void cancelListeningForReason(`app_state:${nextState}`) if (sessionActiveRef.current) setStatus('session.paused') return } canListenOnRouteRef.current = true + logVoiceMetric('app_state_active', { + sessionActive: sessionActiveRef.current, + activeTab: activeTabRef.current, + busy, + playbackActive: playbackActiveRef.current, + audioPlaying: audioPlayingRef.current, + }) if (auth.state === 'signed-in') { - void pullMobilePreferences(apiBaseUrl.trim(), auth.getAuthHeaders).then(applyPrefs) - } else { - void api.getPreferences().then(applyServerPreferences).catch(() => undefined) + void loadPreferences() } if (sessionActiveRef.current && !busy && !playbackActiveRef.current && !audioPlayingRef.current) { listeningStartMsRef.current = Date.now() - setStatus('session.status.listening') + setStatus(listeningStartupStatus()) void speechStartListeningRef.current('en-US') } }) return () => subscription.remove() - }, [api, apiBaseUrl, applyPrefs, applyServerPreferences, auth.getAuthHeaders, auth.state, busy, clearResumeListeningTimer]) + }, [auth.state, busy, cancelListeningForReason, clearResumeListeningTimer, listeningStartupStatus, loadPreferences, logVoiceMetric]) function playCorrection(text: string) { + logUserAction('play_correction_tap', { chars: text.length }) clearResumeListeningTimer() - void speechCancelListeningRef.current() + void cancelListeningForReason('play_correction') void synthesizeCoachSpeech(text).then(voice => { if (voice.audioUrl) { isCorrectionPlayingRef.current = true @@ -985,7 +2163,10 @@ function AppInner() { onStart={startSession} onEnd={() => void endSession()} onPlayCorrection={playCorrection} - onSubmitText={text => void submitTurn(text)} + onSubmitText={text => { + logUserAction('manual_text_submit', { chars: text.trim().length }) + void submitTurn(text) + }} /> ) case 'home': @@ -996,6 +2177,7 @@ function AppInner() { scenarios={scenarios} selectedScenarioKey={selectedScenarioKey} isSessionActive={isSessionActive} + scenarioSwitching={scenarioSwitching} onSelectScenario={selectScenario} onGoToSession={() => selectTab('session')} /> @@ -1022,40 +2204,95 @@ function AppInner() { locale={locale} ttsProvider={ttsProvider} availableProviders={availableProviders} + sessionSttProvider={sessionSttProvider} + availableSessionSttProviders={availableSessionSttProviders} ttsSpeed={ttsSpeed} ttsVoiceId={ttsVoiceId} voiceProfiles={voiceProfiles} selectedVoiceProfileId={selectedVoiceProfileId} xunfeiVoices={xunfeiVoices} settingsLoading={settingsLoading} + authSubmitting={authSubmitting} settingsMessage={settingsMessage} auth={auth} email={email} password={password} authMode={authMode} apiBaseUrl={apiBaseUrl} + apiBaseUrlSource={apiBaseUrlSource} + defaultApiBaseUrl={defaultApiBaseUrl} appVersion={appVersion} voiceMetricsText={voiceMetricsText} - onSetLocale={l => setLocale(l as Locale)} - onSetTheme={setTheme} - onSaveProvider={p => void saveProvider(p)} - onAdjustSpeed={adjustSpeed} - onSavePracticePreferences={() => void savePracticePreferences()} - onLoadPreferences={() => void loadPreferences()} - onSelectVoiceProfile={profile => void selectVoiceProfile(profile)} + asrEvaluationText={asrEvaluationText} + onSetLocale={localeValue => { + logUserAction('settings_locale_tap', { locale: localeValue }) + void saveLocalePreference(localeValue as Locale) + }} + onSetTheme={key => { + logUserAction('settings_theme_tap', { theme: key }) + setTheme(key) + }} + onSaveProvider={provider => { + logUserAction('settings_tts_provider_tap', { provider }) + void saveProvider(provider) + }} + onSetSessionSttProvider={provider => { + logUserAction('settings_stt_provider_tap', { provider }) + setSessionSttProvider(provider) + }} + onAdjustSpeed={delta => { + logUserAction('settings_tts_speed_tap', { delta }) + adjustSpeed(delta) + }} + onSavePracticePreferences={() => { + logUserAction('settings_save_preferences_tap') + void savePracticePreferences() + }} + onLoadPreferences={() => { + logUserAction('settings_reload_tap') + reloadSettingsData() + }} + onSelectVoiceProfile={profile => { + logUserAction('settings_voice_profile_tap', { profileId: profile.id, provider: profile.provider }) + void selectVoiceProfile(profile) + }} onSetEmail={setEmail} onSetPassword={setPassword} - onSetAuthMode={setAuthMode} + onSetAuthMode={mode => { + logUserAction('auth_mode_tap', { mode }) + setAuthMode(mode) + }} onSubmitAuth={() => void submitAuth()} - onSignOut={() => void auth.signOut()} - onSetApiBaseUrl={updateApiBaseUrl} - onClearVoiceMetrics={() => setVoiceMetrics([])} + onSignOut={() => { + logUserAction('auth_sign_out_tap') + void auth.signOut() + }} + onSetApiBaseUrl={value => { + logUserAction('settings_api_base_url_edit', { hasValue: Boolean(value.trim()) }) + updateApiBaseUrl(value) + }} + onResetApiBaseUrl={() => { + logUserAction('settings_api_base_url_reset_tap') + resetApiBaseUrl() + }} + onClearVoiceMetrics={() => { + logUserAction('diagnostics_clear_tap') + setVoiceMetrics([]) + }} onShareVoiceMetrics={() => { + logUserAction('diagnostics_share_tap', { entries: voiceMetrics.length }) void Share.share({ title: 'MeteorVoice voice diagnostics', message: voiceMetricsText || 'No voice metrics yet.', }) }} + onShareASREvaluation={() => { + logUserAction('diagnostics_asr_share_tap', { entries: voiceMetrics.length }) + void Share.share({ + title: 'MeteorVoice ASR P4 evaluation', + message: asrEvaluationText, + }) + }} /> ) } @@ -1065,7 +2302,10 @@ function AppInner() { return ( - {renderScreen()} + + {renderScreen()} + + {(['home', 'session', 'history', 'settings'] as Tab[]).map(tab => ( diff --git a/apps/mobile/src/components/AppFeedbackOverlay.tsx b/apps/mobile/src/components/AppFeedbackOverlay.tsx new file mode 100644 index 0000000..f7b9769 --- /dev/null +++ b/apps/mobile/src/components/AppFeedbackOverlay.tsx @@ -0,0 +1,74 @@ +import { useEffect } from 'react' +import { ActivityIndicator, StyleSheet, Text, View } from 'react-native' +import { hideAppFeedback, type AppFeedbackState } from '@meteorvoice/shared' +import { useTheme } from '../ThemeProvider' + +type Props = { + feedback: AppFeedbackState | null +} + +export function AppFeedbackOverlay({ feedback }: Props) { + const { C } = useTheme() + useEffect(() => { + if (!feedback?.active || !feedback.autoDismissMs) return + const timer = setTimeout(() => { + hideAppFeedback(feedback.source) + }, feedback.autoDismissMs) + return () => clearTimeout(timer) + }, [feedback?.active, feedback?.autoDismissMs, feedback?.source]) + + if (!feedback?.active) return null + + const variant = feedback.variant ?? 'hud' + const blocksInteraction = feedback.blocksInteraction ?? true + const styles = StyleSheet.create({ + overlay: { + ...StyleSheet.absoluteFillObject, + backgroundColor: variant === 'bar' ? 'transparent' : 'rgba(0,0,0,0.28)', + alignItems: 'center', + justifyContent: variant === 'bar' ? 'flex-start' : 'center', + paddingHorizontal: 24, + paddingTop: 18, + }, + surface: { + minWidth: variant === 'bar' ? '100%' : undefined, + backgroundColor: C.surface, + borderRadius: variant === 'bar' ? 8 : 10, + borderWidth: 1, + borderColor: C.border, + alignItems: 'center', + flexDirection: variant === 'panel' ? 'column' : 'row', + gap: 10, + paddingHorizontal: 18, + paddingVertical: 14, + }, + message: { + color: C.textPrimary, + fontSize: 14, + fontWeight: '700', + textAlign: 'center', + }, + title: { + color: C.textPrimary, + fontSize: 15, + fontWeight: '800', + textAlign: 'center', + }, + textStack: { + gap: 3, + flexShrink: 1, + }, + }) + + return ( + + + + + {feedback.title ? {feedback.title} : null} + {feedback.message} + + + + ) +} diff --git a/apps/mobile/src/mobileAuth.ts b/apps/mobile/src/mobileAuth.ts index 7f92b3c..819b9d0 100644 --- a/apps/mobile/src/mobileAuth.ts +++ b/apps/mobile/src/mobileAuth.ts @@ -1,4 +1,5 @@ -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { Platform, Settings } from 'react-native' import { createClient, type Session, type SupabaseClient, type User } from '@supabase/supabase-js' import * as SecureStore from 'expo-secure-store' @@ -15,9 +16,25 @@ export type MobileAuthState = ReturnType const supabaseUrl = process.env.EXPO_PUBLIC_SUPABASE_URL const supabaseAnonKey = process.env.EXPO_PUBLIC_SUPABASE_ANON_KEY +const authStorageKey = 'meteorvoice-mobile-supabase-auth' +const installMarkerKey = 'meteorvoice.installation_marker.v1' +const installMarkerValue = 'installed' + +let shouldClearPersistedAuth = false +if (Platform.OS === 'ios' && Settings.get(installMarkerKey) !== installMarkerValue) { + shouldClearPersistedAuth = true + Settings.set({ [installMarkerKey]: installMarkerValue }) +} const secureStorage = { - getItem: (key: string) => SecureStore.getItemAsync(key), + getItem: async (key: string) => { + if (shouldClearPersistedAuth && key === authStorageKey) { + shouldClearPersistedAuth = false + await SecureStore.deleteItemAsync(key) + return null + } + return SecureStore.getItemAsync(key) + }, setItem: (key: string, value: string) => SecureStore.setItemAsync(key, value), removeItem: (key: string) => SecureStore.deleteItemAsync(key), } @@ -27,6 +44,7 @@ export function useMobileAuth() { const [user, setUser] = useState(null) const [state, setState] = useState(supabaseUrl && supabaseAnonKey ? 'loading' : 'unconfigured') const [message, setMessage] = useState(null) + const sessionRef = useRef(null) const client = useMemo(() => { if (!supabaseUrl || !supabaseAnonKey) return null @@ -36,6 +54,7 @@ export function useMobileAuth() { autoRefreshToken: true, detectSessionInUrl: false, persistSession: true, + storageKey: authStorageKey, storage: secureStorage, }, }) @@ -55,12 +74,14 @@ export function useMobileAuth() { return } + sessionRef.current = data.session setSession(data.session) setUser(data.session?.user ?? null) setState(data.session ? 'signed-in' : 'signed-out') }) const { data: { subscription } } = client.auth.onAuthStateChange((_event, nextSession) => { + sessionRef.current = nextSession setSession(nextSession) setUser(nextSession?.user ?? null) setState(nextSession ? 'signed-in' : 'signed-out') @@ -94,6 +115,7 @@ export function useMobileAuth() { return false } + sessionRef.current = result.data.session setSession(result.data.session) setUser(result.data.user) setState(result.data.session ? 'signed-in' : 'signed-out') @@ -107,22 +129,71 @@ export function useMobileAuth() { } }, [client]) - const signOut = useCallback(async () => { + const signOut = useCallback(async (nextMessage: string | null = null) => { if (!client) return setState('loading') await client.auth.signOut() + sessionRef.current = null setSession(null) setUser(null) setState('signed-out') + setMessage(nextMessage) + }, [client]) + + const refreshSession = useCallback(async () => { + if (!client) { + setState('unconfigured') + setMessage('Set EXPO_PUBLIC_SUPABASE_URL and EXPO_PUBLIC_SUPABASE_ANON_KEY to enable mobile auth.') + return false + } + + const currentSession = sessionRef.current + if (!currentSession?.access_token) { + const { data, error } = await client.auth.getSession() + if (error) { + setMessage(error.message) + setState('error') + return false + } + sessionRef.current = data.session + setSession(data.session) + setUser(data.session?.user ?? null) + setState(data.session ? 'signed-in' : 'signed-out') + return Boolean(data.session?.access_token) + } + + const expiresAtMs = currentSession.expires_at ? currentSession.expires_at * 1000 : 0 + if (!expiresAtMs || expiresAtMs - Date.now() > 60_000) { + return true + } + + const { data, error } = await client.auth.refreshSession(currentSession) + if (error) { + sessionRef.current = null + setSession(null) + setUser(null) + setState('signed-out') + setMessage(error.message) + return false + } + + sessionRef.current = data.session + setSession(data.session) + setUser(data.session?.user ?? null) + setState(data.session ? 'signed-in' : 'signed-out') + setMessage(null) + return Boolean(data.session?.access_token) }, [client]) - const getAuthHeaders = useCallback((): HeadersInit => ( - session?.access_token ? { Authorization: `Bearer ${session.access_token}` } : {} - ), [session]) + const getAuthHeaders = useCallback(async (): Promise => { + await refreshSession() + return sessionRef.current?.access_token ? { Authorization: `Bearer ${sessionRef.current.access_token}` } : {} + }, [refreshSession]) return { getAuthHeaders, message, + refreshSession, session, signOut, state, diff --git a/apps/mobile/src/mobilePreferences.ts b/apps/mobile/src/mobilePreferences.ts index f00b268..8e17470 100644 --- a/apps/mobile/src/mobilePreferences.ts +++ b/apps/mobile/src/mobilePreferences.ts @@ -1,4 +1,4 @@ -import { createMeteorVoiceApiClient } from '@meteorvoice/api-client' +import { createMeteorVoiceApiClient, type PreferencesResponse } from '@meteorvoice/api-client' import type { VoiceProfile } from '@meteorvoice/shared' export type XunfeiVoice = { @@ -13,7 +13,8 @@ export type XunfeiVoice = { type PrefInput = { apiBaseUrl: string - getAuthHeaders: () => HeadersInit + getAuthHeaders: () => HeadersInit | Promise + onUnauthorized?: () => void | Promise ttsProvider?: string ttsSpeed?: number defaultScenarioKey?: string @@ -25,16 +26,16 @@ type PrefInput = { // 内存中记录同步失败的 key，下次成功 sync 时重试 const pendingSyncKeys = new Set() -async function hasAuth(getAuthHeaders: () => HeadersInit) { +async function hasAuth(getAuthHeaders: () => HeadersInit | Promise) { const headers = await getAuthHeaders() return !!(headers as Record).Authorization } -export async function syncMobilePreferences(input: PrefInput) { - if (!input.apiBaseUrl) return +export async function syncMobilePreferences(input: PrefInput): Promise { + if (!input.apiBaseUrl) return null const authed = await hasAuth(input.getAuthHeaders) - if (!authed) return + if (!authed) return null const body: Record = {} if (input.ttsProvider !== undefined) body.tts_provider = input.ttsProvider @@ -44,24 +45,31 @@ export async function syncMobilePreferences(input: PrefInput) { if (input.selectedVoiceProfileId !== undefined) body.selected_voice_profile_id = input.selectedVoiceProfileId if (input.uiTheme !== undefined) body.ui_theme = input.uiTheme - if (Object.keys(body).length === 0) return + if (Object.keys(body).length === 0) return null try { - const api = createMeteorVoiceApiClient({ baseUrl: input.apiBaseUrl, headers: input.getAuthHeaders }) - await api.updatePreferences(body) + const api = createMeteorVoiceApiClient({ + baseUrl: input.apiBaseUrl, + headers: input.getAuthHeaders, + onUnauthorized: input.onUnauthorized, + }) + const preferences = await api.updatePreferences(body) // 成功后清除 pending pendingSyncKeys.clear() + return preferences } catch { // 静默失败，记录待同步项（应用重启后下次 API 拉取会自动覆盖） for (const key of Object.keys(body)) { pendingSyncKeys.add(key) } + return null } } export async function pullMobilePreferences( apiBaseUrl: string, - getAuthHeaders: () => HeadersInit, + getAuthHeaders: () => HeadersInit | Promise, + onUnauthorized?: () => void | Promise, ): Promise<{ ttsProvider: string ttsSpeed: number @@ -82,7 +90,7 @@ export async function pullMobilePreferences( if (!authed) return null try { - const api = createMeteorVoiceApiClient({ baseUrl: apiBaseUrl, headers: getAuthHeaders }) + const api = createMeteorVoiceApiClient({ baseUrl: apiBaseUrl, headers: getAuthHeaders, onUnauthorized }) const raw = await api.getPreferences() pendingSyncKeys.clear() return { diff --git a/apps/mobile/src/nativeAudio.ts b/apps/mobile/src/nativeAudio.ts index e0092cd..b28da05 100644 --- a/apps/mobile/src/nativeAudio.ts +++ b/apps/mobile/src/nativeAudio.ts @@ -218,6 +218,18 @@ export function useNativeSessionAudio(audioUrl: string | null, playbackRateValue }) }, [applyPlaybackRate, audioUrl, configurePlayback, player, recorder, recorderState.isRecording, runExclusive]) + const stopPlayback = useCallback(() => { + try { + player.pause() + player.seekTo(0) + if (phase === 'playing') setPhase('idle') + } catch (error) { + const message = error instanceof Error ? error.message : 'Coach voice failed to stop' + setErrorMessage(message) + setPhase('error') + } + }, [phase, player]) + // 中断后恢复：清除中断标记，允许继续操作 const resumeAfterInterruption = useCallback(async () => { if (!interrupted) return false @@ -310,6 +322,7 @@ export function useNativeSessionAudio(audioUrl: string | null, playbackRateValue playReply, resumeAfterInterruption, startRecording, + stopPlayback, stopRecording, }), [ errorMessage, @@ -324,6 +337,7 @@ export function useNativeSessionAudio(audioUrl: string | null, playbackRateValue resumeAfterInterruption, recorderState.durationMillis, startRecording, + stopPlayback, stopRecording, ]) } diff --git a/apps/mobile/src/screens/HistoryScreen.tsx b/apps/mobile/src/screens/HistoryScreen.tsx index 7ea2458..c4b39d2 100644 --- a/apps/mobile/src/screens/HistoryScreen.tsx +++ b/apps/mobile/src/screens/HistoryScreen.tsx @@ -32,12 +32,6 @@ export function HistoryScreen({ tr, locale, sessions, loading, error, selectedHi const { C } = useTheme() const [expandedId, setExpandedId] = useState(null) const [filterScenario, setFilterScenario] = useState(null) - const [localSessions, setLocalSessions] = useState(sessions) - - // 同步外部 sessions 变化 - if (sessions !== localSessions && sessions.length !== localSessions.length) { - setLocalSessions(sessions) - } function toggle(id: string | number) { if (expandedId === id) { @@ -49,13 +43,12 @@ export function HistoryScreen({ tr, locale, sessions, loading, error, selectedHi } function handleDelete(id: string) { - setLocalSessions(prev => prev.map(s => s.id === id ? { ...s, status: 'deleted' } : s)) onDelete(id) } const filtered = filterScenario - ? localSessions.filter(s => s.scenario_key === filterScenario || s.scenario === filterScenario) - : localSessions + ? sessions.filter(s => s.scenario_key === filterScenario || s.scenario === filterScenario) + : sessions const styles = useMemo(() => StyleSheet.create({ @@ -227,4 +220,3 @@ export function HistoryScreen({ tr, locale, sessions, loading, error, selectedHi ) } - diff --git a/apps/mobile/src/screens/HomeScreen.tsx b/apps/mobile/src/screens/HomeScreen.tsx index a4c9320..5ffe5c1 100644 --- a/apps/mobile/src/screens/HomeScreen.tsx +++ b/apps/mobile/src/screens/HomeScreen.tsx @@ -12,18 +12,19 @@ interface Props { scenarios: Scenario[] selectedScenarioKey: string isSessionActive: boolean - onSelectScenario: (key: string) => void + scenarioSwitching: boolean + onSelectScenario: (key: string) => void | Promise onGoToSession: () => void } export function HomeScreen({ tr, locale, scenarios, - selectedScenarioKey, isSessionActive, + selectedScenarioKey, isSessionActive, scenarioSwitching, onSelectScenario, onGoToSession, }: Props) { const { C } = useTheme() - function handleScenario(key: string) { - onSelectScenario(key) + async function handleScenario(key: string) { + await onSelectScenario(key) onGoToSession() } @@ -40,6 +41,7 @@ export function HomeScreen({ borderWidth: 1, borderColor: C.border, padding: 14, gap: 6, }, cardActive: { borderColor: C.accent, backgroundColor: 'rgba(49,95,72,0.2)' }, + cardDisabled: { opacity: 0.5 }, cardIcon: { fontSize: 22 }, cardName: { color: C.textPrimary, fontSize: 14, fontWeight: '700' }, cardNameActive: { color: C.cream }, @@ -75,8 +77,9 @@ export function HomeScreen({ const active = item.key === selectedScenarioKey return ( handleScenario(item.key)} + style={[styles.card, active && styles.cardActive, scenarioSwitching && styles.cardDisabled]} + disabled={scenarioSwitching} + onPress={() => { void handleScenario(item.key) }} > {item.icon} @@ -99,4 +102,3 @@ export function HomeScreen({ ) } - diff --git a/apps/mobile/src/screens/SettingsScreen.tsx b/apps/mobile/src/screens/SettingsScreen.tsx index 8be44a1..d64bfff 100644 --- a/apps/mobile/src/screens/SettingsScreen.tsx +++ b/apps/mobile/src/screens/SettingsScreen.tsx @@ -11,23 +11,30 @@ interface Props { locale: Locale ttsProvider: string availableProviders: string[] + sessionSttProvider: 'native' | 'xunfei' + availableSessionSttProviders: Array<'native' | 'xunfei'> ttsSpeed: number ttsVoiceId: string | null voiceProfiles: VoiceProfile[] selectedVoiceProfileId: string | null xunfeiVoices: XunfeiVoice[] settingsLoading: boolean + authSubmitting: boolean settingsMessage: string | null auth: MobileAuthState email: string password: string authMode: 'sign-in' | 'sign-up' apiBaseUrl: string + apiBaseUrlSource: 'default' | 'user' + defaultApiBaseUrl: string appVersion: string voiceMetricsText: string + asrEvaluationText: string onSetLocale: (l: string) => void onSetTheme: (k: ThemeKey) => void onSaveProvider: (p: string) => void + onSetSessionSttProvider: (p: 'native' | 'xunfei') => void onAdjustSpeed: (delta: number) => void onSavePracticePreferences: () => void onLoadPreferences: () => void @@ -38,18 +45,21 @@ interface Props { onSubmitAuth: () => void onSignOut: () => void onSetApiBaseUrl: (v: string) => void + onResetApiBaseUrl: () => void onClearVoiceMetrics: () => void onShareVoiceMetrics: () => void + onShareASREvaluation: () => void } export function SettingsScreen({ - tr, locale, ttsProvider, availableProviders, ttsSpeed, + tr, locale, ttsProvider, availableProviders, sessionSttProvider, availableSessionSttProviders, ttsSpeed, ttsVoiceId, voiceProfiles, selectedVoiceProfileId, xunfeiVoices, - settingsLoading, settingsMessage, - auth, email, password, authMode, apiBaseUrl, appVersion, voiceMetricsText, - onSetLocale, onSetTheme, onSaveProvider, onAdjustSpeed, onSavePracticePreferences, + settingsLoading, authSubmitting, settingsMessage, + auth, email, password, authMode, apiBaseUrl, apiBaseUrlSource, defaultApiBaseUrl, appVersion, voiceMetricsText, asrEvaluationText, + onSetLocale, onSetTheme, onSaveProvider, onSetSessionSttProvider, onAdjustSpeed, onSavePracticePreferences, onLoadPreferences, onSelectVoiceProfile, - onSetEmail, onSetPassword, onSetAuthMode, onSubmitAuth, onSignOut, onSetApiBaseUrl, onClearVoiceMetrics, onShareVoiceMetrics, + onSetEmail, onSetPassword, onSetAuthMode, onSubmitAuth, onSignOut, onSetApiBaseUrl, + onResetApiBaseUrl, onClearVoiceMetrics, onShareVoiceMetrics, onShareASREvaluation, }: Props) { const { C, themeKey } = useTheme() const speedFill = Math.max(0, Math.min(1, (ttsSpeed - 0.7) / 0.6)) @@ -167,7 +177,12 @@ export function SettingsScreen({ {tr('settings.language')} {(['en', 'zh'] as const).map(l => ( - onSetLocale(l)} style={[styles.chip, locale === l && styles.chipActive]}> + onSetLocale(l)} + disabled={settingsLoading} + style={[styles.chip, locale === l && styles.chipActive, settingsLoading && styles.chipDisabled]} + > {l === 'en' ? tr('settings.language_en') : tr('settings.language_zh')} ))} @@ -198,7 +213,12 @@ export function SettingsScreen({ {availableProviders.map(p => ( - onSaveProvider(p)} style={[styles.chip, ttsProvider === p && styles.chipActive]}> + onSaveProvider(p)} + disabled={settingsLoading} + style={[styles.chip, ttsProvider === p && styles.chipActive, settingsLoading && styles.chipDisabled]} + > {tr(`settings.tts_provider_${p}`) !== `settings.tts_provider_${p}` ? tr(`settings.tts_provider_${p}`) : p} @@ -208,6 +228,26 @@ export function SettingsScreen({ {settingsMessage && {settingsMessage}} + {/* Session STT Provider */} + + {tr('settings.session_stt_provider')} + + {availableSessionSttProviders.map(provider => ( + onSetSessionSttProvider(provider)} + disabled={settingsLoading} + style={[styles.chip, sessionSttProvider === provider && styles.chipActive, settingsLoading && styles.chipDisabled]} + > + + {tr(`settings.session_stt_provider_${provider}`)} + + + ))} + + {tr('settings.session_stt_provider_hint')} + + {/* Coach Voice */} {tr('settings.voice_profile_current')} @@ -237,6 +277,7 @@ export function SettingsScreen({ !unavailable && onSelectVoiceProfile(profile)} + disabled={settingsLoading || unavailable} style={[styles.voiceCatalogChip, active && styles.chipActive, unavailable && styles.chipDisabled]} > {voiceProfileName(profile)} @@ -260,25 +301,32 @@ export function SettingsScreen({ {tr('settings.tts_speed')} - onAdjustSpeed(-0.1)} style={styles.speedBtn}> + onAdjustSpeed(-0.1)} disabled={settingsLoading} style={[styles.speedBtn, settingsLoading && styles.chipDisabled]}> − - onAdjustSpeed(0.1)} style={styles.speedBtn}> + onAdjustSpeed(0.1)} disabled={settingsLoading} style={[styles.speedBtn, settingsLoading && styles.chipDisabled]}> + {ttsSpeed.toFixed(1)}× - + {tr('settings.save')} {/* API URL */} - {tr('settings.api_url')} + + {tr('settings.api_url')} + {apiBaseUrlSource === 'user' && ( + + {tr('settings.api_url_reset')} + + )} + + + {apiBaseUrlSource === 'user' + ? tr('settings.api_url_source_user') + : `${tr('settings.api_url_source_default')}: ${defaultApiBaseUrl}`} + {/* Diagnostics */} @@ -297,7 +350,10 @@ export function SettingsScreen({ Voice diagnostics - Share + Logs + + + ASR Clear @@ -307,7 +363,7 @@ export function SettingsScreen({ - {voiceMetricsText || 'No voice metrics yet.'} + {voiceMetricsText || asrEvaluationText || 'No voice metrics yet.'} @@ -324,7 +380,12 @@ export function SettingsScreen({ ) : ( {(['sign-in', 'sign-up'] as const).map(m => ( - onSetAuthMode(m)} style={[styles.modeBtn, authMode === m && styles.modeBtnActive]}> + onSetAuthMode(m)} + disabled={authSubmitting || auth.state === 'loading'} + style={[styles.modeBtn, authMode === m && styles.modeBtnActive, (authSubmitting || auth.state === 'loading') && styles.disabled]} + > {m === 'sign-in' ? tr('login.signin') : tr('login.signup')} @@ -348,15 +409,21 @@ export function SettingsScreen({ - + - {auth.state === 'loading' ? tr('login.loading') : authMode === 'sign-in' ? tr('login.signin') : tr('login.signup')} + {authSubmitting || auth.state === 'loading' ? tr('login.loading') : authMode === 'sign-in' ? tr('login.signin') : tr('login.signup')} diff --git a/apps/mobile/src/voicePcmCapture.ts b/apps/mobile/src/voicePcmCapture.ts new file mode 100644 index 0000000..3c36b2f --- /dev/null +++ b/apps/mobile/src/voicePcmCapture.ts @@ -0,0 +1,99 @@ +import { requireOptionalNativeModule, type EventSubscription } from 'expo-modules-core' +import { Platform } from 'react-native' + +export type PcmCaptureOptions = { + sampleRate?: number + frameDurationMs?: number +} + +export type PcmCaptureStatus = { + isCapturing: boolean + sampleRate: number + channels: number + bitDepth: number + frameDurationMs: number + frameSizeBytes: number + frameCount: number + totalBytes: number + elapsedMs: number + reason?: string +} + +export type PcmCaptureFrameEvent = { + sequence: number + timestampMs: number + elapsedMs: number + audioBase64: string + byteCount: number + sampleRate: number + channels: number + bitDepth: number + durationMs: number +} + +export type PcmCaptureStateEvent = PcmCaptureStatus & { + state: 'started' | 'stopped' | 'error' | 'restarting' | 'restarted' | 'interrupted' | 'route_change_ignored' + message?: string +} + +type VoicePcmCaptureNativeModule = { + start: (options: PcmCaptureOptions) => Promise + stop: (reason?: string) => Promise + getStatus: () => Promise + addListener: ( + eventName: 'onPcmCaptureFrame' | 'onPcmCaptureState', + listener: (event: PcmCaptureFrameEvent | PcmCaptureStateEvent) => void, + ) => EventSubscription +} + +function getNativeModule() { + if (Platform.OS !== 'ios') return null + return requireOptionalNativeModule('VoicePcmCapture') +} + +export function isPcmCaptureAvailable() { + return Boolean(getNativeModule()) +} + +export async function startPcmCapture(options: PcmCaptureOptions = {}) { + const nativeModule = getNativeModule() + if (!nativeModule) { + throw new Error('VoicePcmCapture native module is unavailable.') + } + return nativeModule.start({ + sampleRate: options.sampleRate ?? 16000, + frameDurationMs: options.frameDurationMs ?? 40, + }) +} + +export async function stopPcmCapture(reason = 'manual') { + const nativeModule = getNativeModule() + if (!nativeModule) { + throw new Error('VoicePcmCapture native module is unavailable.') + } + return nativeModule.stop(reason) +} + +export async function getPcmCaptureStatus() { + const nativeModule = getNativeModule() + if (!nativeModule) { + throw new Error('VoicePcmCapture native module is unavailable.') + } + return nativeModule.getStatus() +} + +export function addPcmFrameListener(listener: (event: PcmCaptureFrameEvent) => void) { + const nativeModule = getNativeModule() + if (!nativeModule) return { remove() {} } as EventSubscription + return nativeModule.addListener('onPcmCaptureFrame', event => { + listener(event as PcmCaptureFrameEvent) + }) +} + +export function addPcmStateListener(listener: (event: PcmCaptureStateEvent) => void) { + const nativeModule = getNativeModule() + if (!nativeModule) return { remove() {} } as EventSubscription + return nativeModule.addListener('onPcmCaptureState', event => { + listener(event as PcmCaptureStateEvent) + }) +} diff --git a/apps/web/.env.local.example b/apps/web/.env.local.example index 238c8fc..9f2ce0c 100644 --- a/apps/web/.env.local.example +++ b/apps/web/.env.local.example @@ -7,14 +7,15 @@ SUPABASE_SERVICE_ROLE_KEY= DEEPSEEK_API_KEY= DEEPSEEK_BASE_URL=https://api.deepseek.com -# Optional: Production STT/TTS providers -# STT_PROVIDER=mock|browser|whisper +# Optional: Production ASR/TTS providers +ASR_PROVIDER=native # TTS_PROVIDER=mock|xunfei|volcengine|tencent|azure -# Xunfei TTS (recommended first provider in China) +# Xunfei ASR/TTS (recommended first provider in China) XUNFEI_APP_ID= XUNFEI_API_KEY= XUNFEI_API_SECRET= +XUNFEI_ASR_PRODUCT=zh_iat # Required when TTS_PROVIDER=xunfei. Use a V3-compatible vcn from the Xunfei console. XUNFEI_TTS_VOICE= # Specific Xunfei coach voices are selected in Settings from the app voice catalog. diff --git a/apps/web/app/api/asr/providers/route.ts b/apps/web/app/api/asr/providers/route.ts new file mode 100644 index 0000000..e4316c2 --- /dev/null +++ b/apps/web/app/api/asr/providers/route.ts @@ -0,0 +1,15 @@ +import { jsonApiResult, jsonServerError } from '@/lib/server/http' +import { getASRProviders, getDefaultASRProvider } from '@/lib/server/asr' + +export const runtime = 'nodejs' + +export async function GET() { + try { + return jsonApiResult({ + providers: getASRProviders(), + default_provider: getDefaultASRProvider(), + }) + } catch (error) { + return jsonServerError(error, 'Failed to load ASR providers') + } +} diff --git a/apps/web/app/api/asr/session/route.ts b/apps/web/app/api/asr/session/route.ts new file mode 100644 index 0000000..099d0de --- /dev/null +++ b/apps/web/app/api/asr/session/route.ts @@ -0,0 +1,18 @@ +import { guardApiRequest, jsonApiResult, jsonServerError, requireApiUser } from '@/lib/server/http' +import { createASRSessionFromRequest } from '@/lib/server/asr' +import type { ASRSessionBootstrapRequest } from '@meteorvoice/shared' + +export const runtime = 'nodejs' + +export async function POST(request: Request) { + try { + const guard = guardApiRequest(request, { name: 'asr_session', windowMs: 60_000, maxRequests: 30, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) + const body = await request.json() as ASRSessionBootstrapRequest + return jsonApiResult(await createASRSessionFromRequest(body)) + } catch (error) { + return jsonServerError(error, 'Failed to create ASR session') + } +} diff --git a/apps/web/app/api/chat/route.ts b/apps/web/app/api/chat/route.ts index 644f4d7..bcf1bca 100644 --- a/apps/web/app/api/chat/route.ts +++ b/apps/web/app/api/chat/route.ts @@ -1,9 +1,13 @@ import type { ConversationMessage, ConversationContext } from '@/lib/providers/types' import { generateCoachReply } from '@/lib/server/chat' -import { jsonApiResult, jsonServerError } from '@/lib/server/http' +import { guardApiRequest, jsonApiResult, jsonServerError, requireApiUser } from '@/lib/server/http' export async function POST(request: Request) { try { + const guard = guardApiRequest(request, { name: 'chat', windowMs: 60_000, maxRequests: 30, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) const body = await request.json() as { messages: ConversationMessage[] context: ConversationContext diff --git a/apps/web/app/api/preferences/route.ts b/apps/web/app/api/preferences/route.ts index b395398..a824ae7 100644 --- a/apps/web/app/api/preferences/route.ts +++ b/apps/web/app/api/preferences/route.ts @@ -1,8 +1,12 @@ -import { jsonApiResult, jsonServerError } from '@/lib/server/http' +import { guardApiRequest, jsonApiResult, jsonServerError, requireApiUser } from '@/lib/server/http' import { getPreferences, setPreferences } from '@/lib/server/preferences' -export async function GET() { +export async function GET(request: Request) { try { + const guard = guardApiRequest(request, { name: 'preferences_get', windowMs: 60_000, maxRequests: 30, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) return jsonApiResult(await getPreferences()) } catch (error) { return jsonServerError(error, 'Failed to load preferences') @@ -11,6 +15,10 @@ export async function GET() { export async function PATCH(request: Request) { try { + const guard = guardApiRequest(request, { name: 'preferences_patch', windowMs: 60_000, maxRequests: 30, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) const body = await request.json() as { tts_provider?: string locale?: string diff --git a/apps/web/app/api/semantic-endpoint/route.ts b/apps/web/app/api/semantic-endpoint/route.ts index 1e9f528..169f415 100644 --- a/apps/web/app/api/semantic-endpoint/route.ts +++ b/apps/web/app/api/semantic-endpoint/route.ts @@ -1,4 +1,4 @@ -import { jsonApiResult, jsonServerError } from '@/lib/server/http' +import { guardApiRequest, jsonApiResult, jsonServerError, requireApiUser } from '@/lib/server/http' import { createSemanticEndpointCheck } from '@/lib/server/semantic-endpoint' const MAX_TRANSCRIPT_LENGTH = 2000 @@ -28,6 +28,10 @@ async function getCheck() { export async function POST(request: Request) { try { + const guard = guardApiRequest(request, { name: 'semantic_endpoint', windowMs: 60_000, maxRequests: 120, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) const body = await request.json() as unknown if (!body || typeof body !== 'object') { return jsonApiResult({ error: 'Invalid request body', status: 400 }) diff --git a/apps/web/app/api/tts/route.ts b/apps/web/app/api/tts/route.ts index f9ff17f..39d1630 100644 --- a/apps/web/app/api/tts/route.ts +++ b/apps/web/app/api/tts/route.ts @@ -1,10 +1,14 @@ -import { jsonApiResult, jsonServerError } from '@/lib/server/http' +import { guardApiRequest, jsonApiResult, jsonServerError, requireApiUser } from '@/lib/server/http' import { synthesizeSpeechFromRequest } from '@/lib/server/tts' export const runtime = 'nodejs' export async function POST(request: Request) { try { + const guard = guardApiRequest(request, { name: 'tts', windowMs: 60_000, maxRequests: 60, requireClientHeader: true }) + if (guard) return jsonApiResult(guard) + const auth = await requireApiUser() + if (auth) return jsonApiResult(auth) const body = await request.json() as { text?: string accent?: string diff --git a/apps/web/app/history/page.tsx b/apps/web/app/history/page.tsx index 61dc667..3c4dee1 100644 --- a/apps/web/app/history/page.tsx +++ b/apps/web/app/history/page.tsx @@ -5,6 +5,8 @@ import { useLocale, useT } from '@/components/LanguageProvider' import { Card, CardContent } from '@/components/ui/card' import { scenarios, findAccentByKeyOrName, findScenarioByKeyOrName, getAccentLabel, getScenarioLabel } from '@/lib/scenarios' import { flushPendingPreferences } from '@/lib/tts-speed' +import { formatApiRequestError, readApiJsonResponse } from '@meteorvoice/api-client' +import { displayErrorFeedback, hideAppFeedback, runAppOperationGroup } from '@meteorvoice/shared' interface HistorySession { id: string @@ -34,6 +36,8 @@ interface TurnData { } const PAGE_SIZE = 20 +const historyFeedbackSource = 'web_history' +const historyErrorFeedbackSource = 'web_history_error' export default function HistoryPage() { const { locale } = useLocale() @@ -50,6 +54,7 @@ export default function HistoryPage() { const [turnsLoading, setTurnsLoading] = useState(false) const [turnsError, setTurnsError] = useState(null) const [deletingId, setDeletingId] = useState(null) + const historyLoadingMessage = t('history.loading') function statusLabel(status: string) { const key = `history.status.${status}` @@ -78,15 +83,27 @@ export default function HistoryPage() { if (scenario) params.set('scenario', scenario) const res = await fetch(`/api/history?${params.toString()}`) - if (!res.ok) throw new Error('Failed to load') - return res.json() as Promise<{ sessions: HistorySession[]; hasMore: boolean }> + return readApiJsonResponse<{ sessions: HistorySession[]; hasMore: boolean }>(res, 'History request failed') }, []) // 首次加载 useEffect(() => { void flushPendingPreferences() - loadSessions(0, filterScenario) - .then(data => { + async function loadInitialSessions() { + setLoading(true) + const results = await runAppOperationGroup({ + source: historyFeedbackSource, + feedback: { + message: historyLoadingMessage, + variant: 'hud', + blocksInteraction: true, + }, + tasks: { + sessions: () => loadSessions(0, filterScenario), + }, + }) + if (results.sessions.status === 'fulfilled') { + const data = results.sessions.value if (data.sessions && data.sessions.length > 0) { setSessions(data.sessions) setHasMore(data.hasMore) @@ -94,26 +111,46 @@ export default function HistoryPage() { } else { setSource('none') } - }) - .catch(err => { - setError(err instanceof Error ? err.message : t('history.load_error')) - }) - .finally(() => setLoading(false)) - }, [filterScenario, loadSessions, t]) + } else { + const requestError = formatApiRequestError(results.sessions.reason, { + context: 'web_history_list', + presentation: 'inline', + }) + setError(requestError.displayMessage) + } + setLoading(false) + } + void loadInitialSessions() + return () => hideAppFeedback(historyFeedbackSource) + }, [filterScenario, historyLoadingMessage, loadSessions]) // 加载更多 async function handleLoadMore() { if (loadingMore || !hasMore) return setLoadingMore(true) - try { - const data = await loadSessions(sessions.length, filterScenario) + const results = await runAppOperationGroup({ + source: historyFeedbackSource, + feedback: { + message: historyLoadingMessage, + variant: 'hud', + blocksInteraction: true, + }, + tasks: { + sessions: () => loadSessions(sessions.length, filterScenario), + }, + }) + if (results.sessions.status === 'fulfilled') { + const data = results.sessions.value setSessions(prev => [...prev, ...(data.sessions ?? [])]) setHasMore(data.hasMore) - } catch { - // 静默失败 - } finally { - setLoadingMore(false) + } else { + const requestError = formatApiRequestError(results.sessions.reason, { + context: 'web_history_load_more', + presentation: 'banner', + }) + displayErrorFeedback(requestError, historyErrorFeedbackSource) } + setLoadingMore(false) } // 展开/收起 session turns @@ -127,32 +164,61 @@ export default function HistoryPage() { setTurns([]) setTurnsError(null) setTurnsLoading(true) - try { - const res = await fetch(`/api/sessions/${encodeURIComponent(id)}/turns`) - if (!res.ok) throw new Error('Failed to load turns') - const data = await res.json() as { turns: TurnData[] } - setTurns(data.turns ?? []) - } catch { - setTurnsError(t('history.load_error')) - } finally { - setTurnsLoading(false) + const results = await runAppOperationGroup({ + source: historyFeedbackSource, + feedback: { + message: historyLoadingMessage, + variant: 'hud', + blocksInteraction: true, + }, + tasks: { + turns: async () => { + const res = await fetch(`/api/sessions/${encodeURIComponent(id)}/turns`) + return readApiJsonResponse<{ turns: TurnData[] }>(res, 'History turns request failed') + }, + }, + }) + if (results.turns.status === 'fulfilled') { + setTurns(results.turns.value.turns ?? []) + } else { + const requestError = formatApiRequestError(results.turns.reason, { + context: 'web_history_turns', + presentation: 'inline', + }) + setTurnsError(requestError.displayMessage) } + setTurnsLoading(false) } // 软删除 async function handleDelete(id: string) { if (!confirm(t('history.delete_confirm'))) return setDeletingId(id) - try { - const res = await fetch(`/api/session?id=${encodeURIComponent(id)}`, { method: 'DELETE' }) - if (res.ok) { - setSessions(prev => prev.map(s => s.id === id ? { ...s, status: 'deleted' } : s)) - } - } catch { - // 静默失败 - } finally { - setDeletingId(null) + const results = await runAppOperationGroup({ + source: historyFeedbackSource, + feedback: { + message: historyLoadingMessage, + variant: 'hud', + blocksInteraction: true, + }, + tasks: { + deleted: async () => { + const res = await fetch(`/api/session?id=${encodeURIComponent(id)}`, { method: 'DELETE' }) + if (!res.ok) throw new Error(`Delete failed: ${res.status}`) + return true + }, + }, + }) + if (results.deleted.status === 'fulfilled') { + setSessions(prev => prev.map(s => s.id === id ? { ...s, status: 'deleted' } : s)) + } else { + const requestError = formatApiRequestError(results.deleted.reason, { + context: 'web_history_delete', + presentation: 'banner', + }) + displayErrorFeedback(requestError, historyErrorFeedbackSource) } + setDeletingId(null) } return ( diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index c7aa594..641718d 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -5,6 +5,7 @@ import ThemeProvider from '@/components/ThemeProvider' import LanguageProvider from '@/components/LanguageProvider' import VoiceSessionProvider from '@/components/VoiceSessionProvider' import ActiveSessionBar from '@/components/ActiveSessionBar' +import AppFeedbackPresenter from '@/components/AppFeedbackPresenter' export const metadata: Metadata = { title: 'MeteorVoice', @@ -27,6 +28,7 @@ export default function RootLayout({ children }: { children: React.ReactNode }) {children} + diff --git a/apps/web/app/settings/page.tsx b/apps/web/app/settings/page.tsx index 882a675..4c9ca43 100644 --- a/apps/web/app/settings/page.tsx +++ b/apps/web/app/settings/page.tsx @@ -2,11 +2,12 @@ import { useTheme, themes } from '@/components/ThemeProvider' import { useLocale, useT } from '@/components/LanguageProvider' -import { persistPreference, persistTTSSpeedPreference, readTTSSpeedPreference, ttsSpeedOptions, writeTTSSpeedPreference, type TTSSpeed } from '@/lib/tts-speed' +import { readTTSSpeedPreference, ttsSpeedOptions, writeTTSSpeedPreference, type TTSSpeed } from '@/lib/tts-speed' import { writeTTSVoiceIdPreference } from '@/lib/tts-voice' -import type { Locale, VoiceProfile } from '@meteorvoice/shared' +import { displayErrorFeedback, hideAppFeedback, runAppOperationGroup, type Locale, type VoiceProfile } from '@meteorvoice/shared' +import { formatApiRequestError, readApiJsonResponse } from '@meteorvoice/api-client' import { Card, CardHeader, CardTitle, CardDescription, CardContent } from '@/components/ui/card' -import { useEffect, useState } from 'react' +import { useCallback, useEffect, useState } from 'react' const allTtsProviders = [ { key: 'mock', labelKey: 'settings.tts_provider_mock' }, @@ -39,6 +40,19 @@ function getVoiceProfileName(profile: VoiceProfile, locale: Locale) { return locale === 'zh' ? profile.displayNameZh ?? profile.displayName : profile.displayName } +type PreferencesPayload = { + tts_provider?: string + available_providers?: string[] + locale?: Locale + tts_speed?: number + tts_voice_id?: string | null + voice_profiles?: VoiceProfile[] + selected_voice_profile_id?: string | null +} + +const settingsFeedbackSource = 'web_settings' +const settingsErrorFeedbackSource = 'web_settings_error' + export default function SettingsPage() { const { theme, setTheme } = useTheme() const { locale, setLocale } = useLocale() @@ -49,66 +63,150 @@ export default function SettingsPage() { const [voiceProfiles, setVoiceProfiles] = useState([]) const [selectedVoiceProfileId, setSelectedVoiceProfileId] = useState(null) const [availableProviders, setAvailableProviders] = useState(['mock']) + const [settingsError, setSettingsError] = useState(null) + const [settingsLoading, setSettingsLoading] = useState(false) + const settingsSyncingMessage = t('settings.syncing') + + const applyPreferences = useCallback((data: PreferencesPayload) => { + setSettingsError(null) + if (data.locale) setLocale(data.locale) + if (data.tts_provider) setTtsProvider(data.tts_provider) + if (data.available_providers) setAvailableProviders(data.available_providers) + if (data.voice_profiles) setVoiceProfiles(data.voice_profiles) + if ('selected_voice_profile_id' in data) setSelectedVoiceProfileId(data.selected_voice_profile_id ?? null) + if ('tts_voice_id' in data) { + setTtsVoiceId(data.tts_voice_id ?? null) + writeTTSVoiceIdPreference(data.tts_voice_id ?? null) + } + if (typeof data.tts_speed === 'number') { + const serverSpeed = data.tts_speed + const nextSpeed = ttsSpeedOptions.reduce((best, option) => + Math.abs(option - serverSpeed) < Math.abs(best - serverSpeed) ? option : best, + ttsSpeedOptions[2]) + setTtsSpeed(nextSpeed) + writeTTSSpeedPreference(nextSpeed) + } + }, [setLocale]) + + const applyPreferenceUpdate = useCallback((data: PreferencesPayload, body: Record) => { + setSettingsError(null) + if ('locale' in body && data.locale) setLocale(data.locale) + if ('tts_provider' in body && data.tts_provider) setTtsProvider(data.tts_provider) + if ('tts_speed' in body && typeof data.tts_speed === 'number') { + const serverSpeed = data.tts_speed + const nextSpeed = ttsSpeedOptions.reduce((best, option) => + Math.abs(option - serverSpeed) < Math.abs(best - serverSpeed) ? option : best, + ttsSpeedOptions[2]) + setTtsSpeed(nextSpeed) + writeTTSSpeedPreference(nextSpeed) + } + if ('selected_voice_profile_id' in body || 'tts_provider' in body) { + if ('selected_voice_profile_id' in data) setSelectedVoiceProfileId(data.selected_voice_profile_id ?? null) + if ('tts_voice_id' in data) { + setTtsVoiceId(data.tts_voice_id ?? null) + writeTTSVoiceIdPreference(data.tts_voice_id ?? null) + } + const profile = voiceProfiles.find(item => item.id === data.selected_voice_profile_id) + if (profile?.provider) setTtsProvider(profile.provider) + } + }, [setLocale, voiceProfiles]) + + const fetchPreferences = useCallback(async () => { + const res = await fetch('/api/preferences', { + headers: { 'X-MeteorVoice-Client': 'meteorvoice-web' }, + }) + return readApiJsonResponse(res, 'Preferences request failed') + }, []) useEffect(() => { - async function loadTtsProvider() { - try { - const res = await fetch('/api/preferences') - const data = await res.json() as { - tts_provider?: string - available_providers?: string[] - tts_speed?: number - tts_voice_id?: string | null - voice_profiles?: VoiceProfile[] - selected_voice_profile_id?: string | null - } - if (data.tts_provider) setTtsProvider(data.tts_provider) - if (data.available_providers) setAvailableProviders(data.available_providers) - if (data.voice_profiles) setVoiceProfiles(data.voice_profiles) - if ('selected_voice_profile_id' in data) setSelectedVoiceProfileId(data.selected_voice_profile_id ?? null) - if ('tts_voice_id' in data) { - setTtsVoiceId(data.tts_voice_id ?? null) - writeTTSVoiceIdPreference(data.tts_voice_id ?? null) - } - if (typeof data.tts_speed === 'number') { - const serverSpeed = data.tts_speed - const nextSpeed = ttsSpeedOptions.reduce((best, option) => - Math.abs(option - serverSpeed) < Math.abs(best - serverSpeed) ? option : best, - ttsSpeedOptions[2]) - setTtsSpeed(nextSpeed) - writeTTSSpeedPreference(nextSpeed) - } - } catch {} + async function loadInitialPreferences() { + const results = await runAppOperationGroup({ + source: settingsFeedbackSource, + tasks: { + preferences: fetchPreferences, + }, + }) + if (results.preferences.status === 'fulfilled') { + applyPreferences(results.preferences.value) + return + } + const requestError = formatApiRequestError(results.preferences.reason, { + context: 'web_settings_preferences_load', + presentation: 'inline', + }) + setSettingsError(requestError.displayMessage) } - void loadTtsProvider() + void loadInitialPreferences() + }, [applyPreferences, fetchPreferences]) + + useEffect(() => () => { + hideAppFeedback(settingsFeedbackSource) }, []) + async function savePreferences(body: Record, context: string) { + if (settingsLoading) return + setSettingsLoading(true) + setSettingsError(null) + const results = await runAppOperationGroup({ + source: settingsFeedbackSource, + feedback: { + message: settingsSyncingMessage, + variant: 'hud', + blocksInteraction: true, + }, + tasks: { + preferences: async () => { + const res = await fetch('/api/preferences', { + method: 'PATCH', + headers: { 'Content-Type': 'application/json', 'X-MeteorVoice-Client': 'meteorvoice-web' }, + body: JSON.stringify(body), + }) + return readApiJsonResponse(res, 'Preferences update failed') + }, + }, + }) + if (results.preferences.status === 'fulfilled') { + applyPreferenceUpdate(results.preferences.value, body) + setSettingsLoading(false) + return + } + try { + throw results.preferences.reason + } catch (error) { + const requestError = formatApiRequestError(error, { + context, + presentation: 'banner', + }) + setSettingsError(requestError.displayMessage) + displayErrorFeedback(requestError, settingsErrorFeedbackSource) + } finally { + setSettingsLoading(false) + } + } + function handleTtsProviderChange(key: string) { - setTtsProvider(key) const nextProfile = voiceProfiles.find(profile => profile.provider === key && profile.status === 'active') - if (nextProfile) { - setSelectedVoiceProfileId(nextProfile.id) - setTtsVoiceId(nextProfile.providerVoiceId) - writeTTSVoiceIdPreference(nextProfile.providerVoiceId) - } - void persistPreference('tts_provider', key) - if (nextProfile) void persistPreference('selected_voice_profile_id', nextProfile.id) + void savePreferences({ + tts_provider: key, + ...(nextProfile ? { selected_voice_profile_id: nextProfile.id } : {}), + }, 'web_settings_tts_provider_save') } function handleTtsSpeedChange(index: number) { const next = ttsSpeedOptions[index] ?? 1 - setTtsSpeed(next) - void persistTTSSpeedPreference(next) + void savePreferences({ tts_speed: next }, 'web_settings_tts_speed_save') } function handleVoiceProfileChange(profile: VoiceProfile) { if (profile.status !== 'active') return - setSelectedVoiceProfileId(profile.id) - setTtsProvider(profile.provider) - setTtsVoiceId(profile.providerVoiceId) - writeTTSVoiceIdPreference(profile.providerVoiceId) - void persistPreference('selected_voice_profile_id', profile.id) + void savePreferences({ selected_voice_profile_id: profile.id }, 'web_settings_voice_profile_save') + } + + function handleLocaleChange(nextLocale: Locale) { + if (nextLocale === locale) return + setLocale(nextLocale) + void savePreferences({ locale: nextLocale }, 'web_settings_locale_save') } const providerVoiceProfiles = voiceProfiles.filter(profile => profile.provider === ttsProvider) @@ -124,6 +222,11 @@ export default function SettingsPage() { {t('settings.subtitle')}

+ {settingsError && ( +
+ {settingsError} +
+ )} @@ -137,6 +240,7 @@ export default function SettingsPage() { key={th.key} type="button" onClick={() => setTheme(th.key)} + disabled={settingsLoading} className={`chip-action ${th.key === theme ? 'is-active' : ''}`} > {t(th.labelKey)} @@ -160,7 +264,8 @@ export default function SettingsPage() {