From 241de61866b9ef7ef5d4b4e32f77cb34d49d770b Mon Sep 17 00:00:00 2001 From: clh02467605 Date: Wed, 12 Aug 2026 17:12:23 +0800 Subject: [PATCH 1/5] fix: support sync-flash and qwen3-filetrans ASR models in speech recognize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add asr-routes.ts with resolveAsrApi() to route models to the correct DashScope endpoint instead of always hitting asr/transcription - Async filetrans: fun-asr / paraformer / *-filetrans → file_urls (plural) - Async filetrans (qwen3): qwen3-asr-flash-filetrans* → file_url (singular) - Sync flash (input-audio): fun-asr-flash* / qwen-audio-*-asr-flash → multimodal-generation - Sync flash (qwen3): qwen3-asr-flash* → multimodal-generation + asr_options - Realtime/streaming models now give a clear USAGE error instead of a confusing server-side "url error" - Propagate same routing logic to pipeline speechRecognize step - Add table-driven unit tests and dry-run e2e assertions Fixes #146 --- .../commands/src/commands/speech/recognize.ts | 162 +++++++++-- .../tests/e2e/speech-recognize.e2e.test.ts | 70 +++++ packages/core/src/client/asr-routes.ts | 254 ++++++++++++++++++ packages/core/src/client/index.ts | 10 + packages/core/src/types/api.ts | 3 +- packages/core/tests/asr-routes.test.ts | 154 +++++++++++ packages/runtime/src/pipeline/steps/bl-api.ts | 85 +++++- skills/bailian-gen/reference/index.md | 2 +- skills/bailian-gen/reference/speech.md | 50 ++-- 9 files changed, 740 insertions(+), 50 deletions(-) create mode 100644 packages/core/src/client/asr-routes.ts create mode 100644 packages/core/tests/asr-routes.test.ts diff --git a/packages/commands/src/commands/speech/recognize.ts b/packages/commands/src/commands/speech/recognize.ts index 5af606bb..31a29637 100644 --- a/packages/commands/src/commands/speech/recognize.ts +++ b/packages/commands/src/commands/speech/recognize.ts @@ -12,6 +12,11 @@ import { stripUndefined, taskPath, speechRecognizePath, + resolveAsrApi, + buildAsrFlashRequest, + extractAsrFlashText, + type AsrApiRoute, + type AsrFlashFamily, type OutputFormat, type FlagsDef, type ParsedFlags, @@ -27,8 +32,18 @@ const RECOGNIZE_FLAGS = { description: "Audio file URL or local file path (repeatable, max 100)", required: true, }, - model: { type: "string", valueHint: "", description: "Model ID (default: fun-asr)" }, - language: { type: "string", valueHint: "", description: "Language hint (e.g. zh, en, ja)" }, + model: { + type: "string", + valueHint: "", + description: + "Model ID (default: fun-asr). Async: fun-asr / *-filetrans / paraformer-*; sync: qwen3-asr-flash* / fun-asr-flash* / qwen-audio-*-asr-flash", + }, + language: { + type: "string", + valueHint: "", + description: + "Language hint (e.g. zh, en, ja). Async & input-audio sync: language_hints; qwen3 sync: asr_options.language", + }, diarization: { type: "switch", description: "Enable automatic speaker diarization" }, speakerCount: { type: "number", @@ -55,8 +70,26 @@ const RECOGNIZE_FLAGS = { } satisfies FlagsDef; type RecognizeFlags = ParsedFlags; +function assertSyncFlashFlagsAllowed(flags: RecognizeFlags, model: string): void { + const unsupported: string[] = []; + if (flags.diarization === true) unsupported.push("--diarization"); + if (flags.speakerCount !== undefined) unsupported.push("--speaker-count"); + if (flags.vocabularyId !== undefined) unsupported.push("--vocabulary-id"); + if (flags.channelId !== undefined) unsupported.push("--channel-id"); + if (flags.async === true) unsupported.push("--async"); + if (flags.pollInterval !== undefined) unsupported.push("--poll-interval"); + + if (unsupported.length > 0) { + throw new BailianError( + `Model "${model}" uses sync Flash ASR and does not support: ${unsupported.join(", ")}.\n` + + `Hint: Use an async filetrans model (e.g. fun-asr, qwen3-asr-flash-filetrans) for those flags.`, + ExitCode.USAGE, + ); + } +} + export default defineCommand({ - description: "Recognize speech from audio files (FunAudio-ASR)", + description: "Recognize speech from audio files (FunAudio-ASR / Qwen-ASR Flash)", auth: "apiKey", usageArgs: "--url [flags]", flags: RECOGNIZE_FLAGS, @@ -68,6 +101,7 @@ export default defineCommand({ "--url https://example.com/audio.mp3 --vocabulary-id vocab-abc123", "--url https://example.com/audio.mp3 --out result.json", "--url https://example.com/audio.mp3 --async --quiet", + "--url https://example.com/audio.mp3 --model qwen-audio-3.0-asr-flash --language en", ], async run(ctx) { const { settings, flags } = ctx; @@ -90,19 +124,64 @@ export default defineCommand({ } const model = flags.model || "fun-asr"; + const route = resolveAsrApi(model); + if (route.kind === "unsupported") { + throw new BailianError( + route.unsupportedReason ?? `Unsupported ASR model: ${model}`, + ExitCode.USAGE, + ); + } + + if (route.kind === "sync-flash") { + assertSyncFlashFlagsAllowed(flags, model); + if (rawUrls.length !== 1) { + throw new BailianError( + `Model "${model}" is a sync Flash ASR model and accepts exactly one --url (got ${rawUrls.length}).\n` + + `Hint: Pass a single audio URL, or use an async filetrans model for batch files.`, + ExitCode.USAGE, + ); + } + } + if ( + route.kind === "async-filetrans" && + route.asyncInputStyle === "file_url" && + rawUrls.length !== 1 + ) { + throw new BailianError( + `Model "${model}" accepts exactly one --url (got ${rawUrls.length}).\n` + + "Hint: qwen3-asr-flash-filetrans* requires a single file_url.", + ExitCode.USAGE, + ); + } + const format = detectOutputFormat(settings.output); // Auto-upload local files in parallel - const resolvedUrls = await Promise.all(rawUrls.map((u) => ctx.client.uploadFile(u, model))); + const resolvedUrls = await Promise.all(rawUrls.map((url) => ctx.client.uploadFile(url, model))); + + if (route.kind === "sync-flash") { + await handleSyncFlashMode( + ctx.client, + settings, + flags, + format, + model, + route, + resolvedUrls[0]!, + ); + return; + } + const channelId = flags.channelId; const language = flags.language; const vocabularyId = flags.vocabularyId; const body: DashScopeASRRequest = { model, - input: { - file_urls: resolvedUrls, - }, + input: + route.asyncInputStyle === "file_url" + ? { file_url: resolvedUrls[0]! } + : { file_urls: resolvedUrls }, parameters: { channel_id: channelId !== undefined ? [channelId] : [0], language_hints: language ? [language] : undefined, @@ -116,7 +195,7 @@ export default defineCommand({ stripUndefined(body.parameters as Record); if (settings.dryRun) { - emitResult({ request: body, mode: "async" }, format); + emitResult({ request: body, mode: "async", path: speechRecognizePath() }, format); return; } @@ -128,6 +207,53 @@ export default defineCommand({ }, }); +async function handleSyncFlashMode( + client: Client, + settings: Settings, + flags: RecognizeFlags, + format: OutputFormat, + model: string, + route: AsrApiRoute, + audioUrl: string, +): Promise { + const flashFamily = route.flashFamily as AsrFlashFamily; + const body = buildAsrFlashRequest({ + model, + audioUrl, + language: flags.language, + flashFamily, + }); + + if (settings.dryRun) { + emitResult({ request: body, mode: "sync", path: route.path }, format); + return; + } + + if (!settings.quiet) { + process.stderr.write(`[Model: ${model}] [Mode: sync] [Files: 1]\n`); + } + + const response = await client.requestJson>({ + path: route.path, + method: "POST", + body, + }); + + const text = extractAsrFlashText(response, flashFamily); + if (text) { + process.stdout.write(text.endsWith("\n") ? text : `${text}\n`); + } else { + emitBare(JSON.stringify(response)); + } + + if (flags.out) { + writeFileSync(flags.out, JSON.stringify(response, null, 2) + "\n"); + if (!settings.quiet) { + process.stderr.write(`Full result saved to: ${flags.out}\n`); + } + } +} + async function handleAsyncMode( client: Client, settings: Settings, @@ -160,12 +286,12 @@ async function handleAsyncMode( url: pollUrl, intervalSec: pollInterval, timeoutSec: settings.timeout, - isComplete: (d) => (d as DashScopeASRTaskResult).output.task_status === "SUCCEEDED", - isFailed: (d) => (d as DashScopeASRTaskResult).output.task_status === "FAILED", - getStatus: (d) => (d as DashScopeASRTaskResult).output.task_status, - getErrorMessage: (d) => { - const o = (d as DashScopeASRTaskResult).output; - return (o as unknown as Record).message as string | undefined; + isComplete: (data) => (data as DashScopeASRTaskResult).output.task_status === "SUCCEEDED", + isFailed: (data) => (data as DashScopeASRTaskResult).output.task_status === "FAILED", + getStatus: (data) => (data as DashScopeASRTaskResult).output.task_status, + getErrorMessage: (data) => { + const output = (data as DashScopeASRTaskResult).output; + return (output as unknown as Record).message as string | undefined; }, }); @@ -179,12 +305,14 @@ async function handleAsyncMode( // Collect all transcription data for --out const allTransData: Record[] = []; - for (let i = 0; i < results.length; i++) { - const subResult = results[i]!; + for (let index = 0; index < results.length; index++) { + const subResult = results[index]!; const isMulti = fileCount > 1; if (isMulti) { - process.stdout.write(`=== [${i + 1}/${results.length}] ${subResult.file_url ?? ""} ===\n`); + process.stdout.write( + `=== [${index + 1}/${results.length}] ${subResult.file_url ?? ""} ===\n`, + ); } if (subResult.subtask_status === "FAILED") { diff --git a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts index 926af0ef..db70875a 100644 --- a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts +++ b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts @@ -16,6 +16,32 @@ import { SPEECH_ROUTES } from "./topic-routes.ts"; */ describe("e2e: speech recognize", () => { + async function runRecognizeDryRun(args: string[]) { + const { stdout, stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ + "speech", + "recognize", + ...args, + "--dry-run", + "--output", + "json", + "--quiet", + ]); + expect(exitCode, stderr).toBe(0); + return parseStdoutJson<{ + mode?: string; + path?: string; + request?: { + model?: string; + parameters?: { format?: string; language_hints?: string[] }; + input?: { + file_url?: string; + file_urls?: string[]; + messages?: Array<{ content?: Array<{ type?: string }> }>; + }; + }; + }>(stdout); + } + test("speech recognize --help 正常退出", async () => { const { stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ "speech", @@ -25,6 +51,50 @@ describe("e2e: speech recognize", () => { expect(exitCode, stderr).toBe(0); expect(stderr).toMatch(/recognize|--url|model|audio/i); }); + + test("speech recognize sync-flash dry-run 走 multimodal-generation", async () => { + const body = await runRecognizeDryRun([ + "--model", + "qwen-audio-3.0-asr-flash", + "--url", + "https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav", + "--language", + "en", + ]); + expect(body.mode).toBe("sync"); + expect(body.path).toBe("/api/v1/services/aigc/multimodal-generation/generation"); + expect(body.request?.model).toBe("qwen-audio-3.0-asr-flash"); + expect(body.request?.parameters?.format).toBe("wav"); + expect(body.request?.parameters?.language_hints).toEqual(["en"]); + expect(body.request?.input?.messages?.[0]?.content?.[0]?.type).toBe("input_audio"); + }); + + test("speech recognize qwen3 filetrans dry-run 使用 file_url 单数字段", async () => { + const body = await runRecognizeDryRun([ + "--model", + "qwen3-asr-flash-filetrans", + "--url", + "https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav", + ]); + expect(body.mode).toBe("async"); + expect(body.path).toBe("/api/v1/services/audio/asr/transcription"); + expect(body.request?.input?.file_url?.startsWith("https://")).toBe(true); + expect(body.request?.input?.file_urls).toBeUndefined(); + }); + + test("speech recognize realtime 模型报用法错误", async () => { + const { stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ + "speech", + "recognize", + "--model", + "qwen3-asr-flash-realtime", + "--url", + "https://example.com/a.wav", + "--quiet", + ]); + expect(exitCode).toBe(2); + expect(stderr).toMatch(/realtime|WebSocket|unsupported/i); + }); }); describe.skipIf(!isBailianE2EMediaEnabled() || !isDashScopeE2EReady())( diff --git a/packages/core/src/client/asr-routes.ts b/packages/core/src/client/asr-routes.ts new file mode 100644 index 00000000..3297afe3 --- /dev/null +++ b/packages/core/src/client/asr-routes.ts @@ -0,0 +1,254 @@ +import { imageSyncPath, speechRecognizePath } from "./endpoints.ts"; + +/** + * DashScope ASR APIs differ by model family: + * + * - async file transcription (`.../audio/asr/transcription`): + * fun-asr*, paraformer* (non-realtime), *-filetrans, sensevoice* + * language via `parameters.language_hints` + * - sync multimodal (`.../aigc/multimodal-generation/generation`): + * - qwen3: `{ content: [{ audio }] }` + optional `asr_options.language` + * (qwen3-asr-flash*) + * - input-audio: `{ type: input_audio, input_audio.data }` + + * `format`/`sample_rate` + optional `language_hints` + * (fun-asr-flash*, qwen-audio-*-asr-flash*) + * - realtime / streaming: WebSocket — not supported by `speech recognize` + */ + +export type AsrApiKind = "async-filetrans" | "sync-flash" | "unsupported"; + +/** Sync-flash request body shape differs by Flash protocol family. */ +export type AsrFlashFamily = "qwen3" | "input-audio"; + +export interface AsrApiRoute { + kind: AsrApiKind; + path: string; + /** True when the call is synchronous (no X-DashScope-Async / task poll). */ + useSync: boolean; + /** + * Async transcription request input style. + * - `file_urls`: classic async models (fun-asr / paraformer / qwen-audio filetrans...) + * - `file_url`: qwen3-asr-flash-filetrans family + */ + asyncInputStyle?: "file_urls" | "file_url"; + flashFamily?: AsrFlashFamily; + /** Human-readable reason when kind is unsupported. */ + unsupportedReason?: string; +} + +function isRealtimeOrStreaming(model: string): boolean { + return /realtime|streaming/i.test(model); +} + +function isFiletransModel(model: string): boolean { + return /filetrans/i.test(model); +} + +function isQwen3FiletransModel(model: string): boolean { + return /^qwen3-asr-flash-filetrans(?:-|$)/i.test(model); +} + +const INPUT_AUDIO_FLASH_PREFIXES = ["fun-asr-flash", "qwen-audio"] as const; + +/** + * Fun-ASR-Flash / Qwen-Audio-*-ASR-Flash share the input_audio + format protocol. + * Examples: fun-asr-flash-2026-06-15, qwen-audio-3.0-asr-flash + */ +function isInputAudioFlashModel(model: string): boolean { + if (isRealtimeOrStreaming(model) || isFiletransModel(model)) return false; + if (model.startsWith(INPUT_AUDIO_FLASH_PREFIXES[0])) return true; + if (model.startsWith(INPUT_AUDIO_FLASH_PREFIXES[1]) && /asr-flash/i.test(model)) return true; + return false; +} + +/** + * Qwen3-ASR-Flash sync models use content.audio + asr_options. + * Examples: qwen3-asr-flash, qwen3-asr-flash-2025-09-08, qwen3-asr-flash-us + */ +function isQwen3AsrFlashModel(model: string): boolean { + if (!/^qwen3-asr-flash(?:-|$)/i.test(model)) return false; + if (isFiletransModel(model) || isRealtimeOrStreaming(model)) return false; + if (isInputAudioFlashModel(model)) return false; + return true; +} + +/** + * Resolve which DashScope ASR API a model should use for file recognition. + * Unknown models default to async-filetrans (preserves existing CLI behavior). + */ +export function resolveAsrApi(model: string): AsrApiRoute { + if (isRealtimeOrStreaming(model)) { + return { + kind: "unsupported", + path: "", + useSync: false, + unsupportedReason: + `Model "${model}" is a realtime/streaming ASR model and requires a WebSocket API. ` + + `Use an async filetrans model (e.g. fun-asr, qwen3-asr-flash-filetrans) or a sync flash model ` + + `(e.g. qwen3-asr-flash, qwen-audio-3.0-asr-flash) with this command.`, + }; + } + + if (isFiletransModel(model)) { + return { + kind: "async-filetrans", + path: speechRecognizePath(), + useSync: false, + asyncInputStyle: isQwen3FiletransModel(model) ? "file_url" : "file_urls", + }; + } + + if (isInputAudioFlashModel(model)) { + return { + kind: "sync-flash", + path: imageSyncPath(), + useSync: true, + flashFamily: "input-audio", + }; + } + + if (isQwen3AsrFlashModel(model)) { + return { + kind: "sync-flash", + path: imageSyncPath(), + useSync: true, + flashFamily: "qwen3", + }; + } + + // fun-asr / paraformer / sensevoice / unknown → keep legacy async path + return { + kind: "async-filetrans", + path: speechRecognizePath(), + useSync: false, + asyncInputStyle: "file_urls", + }; +} + +/** Infer audio container hint for input-audio Flash `parameters.format`. */ +export function inferAudioFormatHint(audioUrl: string): string { + const pathPart = audioUrl.split("?")[0] ?? audioUrl; + const match = pathPart.match(/\.([a-zA-Z0-9]+)$/); + const extension = match?.[1]?.toLowerCase(); + if (!extension) return "wav"; + if (extension === "mpeg") return "mp3"; + return extension; +} + +export interface BuildAsrFlashRequestOpts { + model: string; + audioUrl: string; + language?: string; + flashFamily: AsrFlashFamily; +} + +/** Build a sync multimodal ASR request body for Flash models. */ +export function buildAsrFlashRequest(opts: BuildAsrFlashRequestOpts): Record { + const { model, audioUrl, language, flashFamily } = opts; + + if (flashFamily === "input-audio") { + // 与官方 Qwen-Audio / Fun-ASR-Flash 文档一致：语种走 language_hints + const parameters: Record = { + format: inferAudioFormatHint(audioUrl), + sample_rate: "16000", + }; + if (language) { + parameters.language_hints = [language]; + } + return { + model, + input: { + messages: [ + { + role: "user", + content: [ + { + type: "input_audio", + input_audio: { data: audioUrl }, + }, + ], + }, + ], + }, + parameters, + }; + } + + const asrOptions: Record = {}; + if (language) { + asrOptions.language = language; + } + + const parameters: Record = {}; + if (Object.keys(asrOptions).length > 0) { + parameters.asr_options = asrOptions; + } + + const body: Record = { + model, + input: { + messages: [ + { + role: "user", + content: [{ audio: audioUrl }], + }, + ], + }, + }; + if (Object.keys(parameters).length > 0) { + body.parameters = parameters; + } + return body; +} + +/** + * Extract recognition text from a sync Flash ASR response. + * Qwen3 uses choices[].message.content; input-audio Flash uses output.text. + */ +export function extractAsrFlashText( + response: Record, + flashFamily: AsrFlashFamily, +): string { + const output = response.output as Record | undefined; + if (!output) return ""; + + if (flashFamily === "input-audio") { + if (typeof output.text === "string" && output.text.length > 0) { + return output.text; + } + const nested = output.output as Record | undefined; + const sentence = nested?.sentence as Record | undefined; + if (typeof sentence?.text === "string") { + return sentence.text; + } + return ""; + } + + const choices = output.choices as Array> | undefined; + if (!choices?.length) return ""; + + const texts: string[] = []; + for (const choice of choices) { + const message = choice.message as Record | undefined; + if (!message) continue; + const content = message.content; + if (typeof content === "string") { + texts.push(content); + continue; + } + if (!Array.isArray(content)) continue; + for (const item of content) { + if (typeof item === "string") { + texts.push(item); + continue; + } + if (item && typeof item === "object") { + const record = item as Record; + if (typeof record.text === "string") { + texts.push(record.text); + } + } + } + } + return texts.join(""); +} diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts index 31bd04a7..06384131 100644 --- a/packages/core/src/client/index.ts +++ b/packages/core/src/client/index.ts @@ -34,6 +34,16 @@ export { type ImageInputStyle, type ImageSizeProfile, } from "./image-routes.ts"; +export { + buildAsrFlashRequest, + extractAsrFlashText, + inferAudioFormatHint, + resolveAsrApi, + type AsrApiKind, + type AsrApiRoute, + type AsrFlashFamily, + type BuildAsrFlashRequestOpts, +} from "./asr-routes.ts"; export { CHANNEL, sourceConfig, trackingHeaders, type TrackingIdentity } from "./headers.ts"; export type { HttpDeps, RequestOpts } from "./http.ts"; export { request, requestJson } from "./http.ts"; diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts index c6c00ec0..c4b2e61e 100644 --- a/packages/core/src/types/api.ts +++ b/packages/core/src/types/api.ts @@ -533,7 +533,8 @@ export interface DashScopeTTSStreamChunk { export interface DashScopeASRRequest { model: string; input: { - file_urls: string[]; + file_urls?: string[]; + file_url?: string; }; parameters?: { channel_id?: number[]; diff --git a/packages/core/tests/asr-routes.test.ts b/packages/core/tests/asr-routes.test.ts new file mode 100644 index 00000000..2b762c4d --- /dev/null +++ b/packages/core/tests/asr-routes.test.ts @@ -0,0 +1,154 @@ +import { expect, test } from "vite-plus/test"; +import { + buildAsrFlashRequest, + extractAsrFlashText, + inferAudioFormatHint, + resolveAsrApi, +} from "../src/client/asr-routes.ts"; + +test("resolveAsrApi routes model families correctly", () => { + const cases = [ + { + model: "fun-asr", + expected: { + kind: "async-filetrans", + useSync: false, + path: "/api/v1/services/audio/asr/transcription", + asyncInputStyle: "file_urls", + }, + }, + { + model: "qwen3-asr-flash-filetrans-2025-11-17", + expected: { + kind: "async-filetrans", + useSync: false, + path: "/api/v1/services/audio/asr/transcription", + asyncInputStyle: "file_url", + }, + }, + { + model: "qwen-audio-3.0-asr-flash-filetrans", + expected: { + kind: "async-filetrans", + useSync: false, + asyncInputStyle: "file_urls", + }, + }, + { + model: "qwen3-asr-flash-us", + expected: { + kind: "sync-flash", + useSync: true, + flashFamily: "qwen3", + path: "/api/v1/services/aigc/multimodal-generation/generation", + }, + }, + { + model: "qwen-audio-3.0-asr-flash", + expected: { + kind: "sync-flash", + useSync: true, + flashFamily: "input-audio", + }, + }, + { + model: "qwen3-asr-flash-realtime", + expected: { + kind: "unsupported", + }, + }, + { + model: "foo-asr-flash", + expected: { + kind: "async-filetrans", + useSync: false, + path: "/api/v1/services/audio/asr/transcription", + asyncInputStyle: "file_urls", + }, + }, + ] as const; + + for (const { model, expected } of cases) { + const route = resolveAsrApi(model); + expect(route, model).toMatchObject(expected); + if (expected.kind === "unsupported") { + expect(route.unsupportedReason, model).toMatch(/realtime|streaming|WebSocket/i); + } + } +}); + +test("unknown models default to async-filetrans for backward compatibility", () => { + expect(resolveAsrApi("custom-asr-model")).toMatchObject({ + kind: "async-filetrans", + useSync: false, + }); +}); + +test("inferAudioFormatHint reads extension from url", () => { + expect(inferAudioFormatHint("https://example.com/a.mp3")).toBe("mp3"); + expect(inferAudioFormatHint("oss://bucket/path/file.WAV")).toBe("wav"); + expect(inferAudioFormatHint("https://example.com/a.mpeg?x=1")).toBe("mp3"); + expect(inferAudioFormatHint("https://example.com/noext")).toBe("wav"); +}); + +test("buildAsrFlashRequest shapes qwen3 and input-audio bodies", () => { + expect( + buildAsrFlashRequest({ + model: "qwen3-asr-flash", + audioUrl: "https://example.com/a.mp3", + language: "en", + flashFamily: "qwen3", + }), + ).toEqual({ + model: "qwen3-asr-flash", + input: { + messages: [{ role: "user", content: [{ audio: "https://example.com/a.mp3" }] }], + }, + parameters: { asr_options: { language: "en" } }, + }); + + expect( + buildAsrFlashRequest({ + model: "qwen-audio-3.0-asr-flash", + audioUrl: "https://example.com/a.wav", + language: "en", + flashFamily: "input-audio", + }), + ).toEqual({ + model: "qwen-audio-3.0-asr-flash", + input: { + messages: [ + { + role: "user", + content: [{ type: "input_audio", input_audio: { data: "https://example.com/a.wav" } }], + }, + ], + }, + parameters: { format: "wav", sample_rate: "16000", language_hints: ["en"] }, + }); +}); + +test("extractAsrFlashText reads qwen3 choices and input-audio text fields", () => { + expect( + extractAsrFlashText( + { + output: { + choices: [{ message: { content: [{ text: "你好" }] } }], + }, + }, + "qwen3", + ), + ).toBe("你好"); + + expect( + extractAsrFlashText( + { + output: { + text: "Hello World", + output: { sentence: { text: "ignored when text present" } }, + }, + }, + "input-audio", + ), + ).toBe("Hello World"); +}); diff --git a/packages/runtime/src/pipeline/steps/bl-api.ts b/packages/runtime/src/pipeline/steps/bl-api.ts index 886ad5b6..b6def2d3 100644 --- a/packages/runtime/src/pipeline/steps/bl-api.ts +++ b/packages/runtime/src/pipeline/steps/bl-api.ts @@ -10,6 +10,9 @@ import { taskPath, speechSynthesizePath, speechRecognizePath, + resolveAsrApi, + buildAsrFlashRequest, + extractAsrFlashText, stripUndefined, resolveBooleanFlag, resolveWatermark, @@ -573,24 +576,91 @@ export async function speechRecognize( }); } + const model = input.model || "fun-asr"; + const route = resolveAsrApi(model); + if (route.kind === "unsupported") { + throw new PipelineError( + "invalid_input", + route.unsupportedReason ?? `Unsupported ASR model: ${model}`, + { + step: "speech/recognize", + }, + ); + } + + if (route.kind === "sync-flash") { + if (rawUrls.length !== 1) { + throw new PipelineError( + "invalid_input", + `Model "${model}" is a sync Flash ASR model and accepts exactly one url (got ${rawUrls.length})`, + { step: "speech/recognize" }, + ); + } + if ( + input.diarization || + input["speaker-count"] !== undefined || + input["vocabulary-id"] !== undefined || + input["channel-id"] !== undefined + ) { + throw new PipelineError( + "invalid_input", + `Model "${model}" uses sync Flash ASR and does not support diarization / speaker-count / vocabulary-id / channel-id`, + { step: "speech/recognize" }, + ); + } + } + if ( + route.kind === "async-filetrans" && + route.asyncInputStyle === "file_url" && + rawUrls.length !== 1 + ) { + throw new PipelineError( + "invalid_input", + `Model "${model}" accepts exactly one url (got ${rawUrls.length})`, + { step: "speech/recognize" }, + ); + } + // Resolve local files to upload URLs const fileUrls: string[] = []; - for (const u of rawUrls) { - if (isLocalFile(u)) { + for (const audioUrl of rawUrls) { + if (isLocalFile(audioUrl)) { fileUrls.push( - await env.client.uploadFile(u, input.model || "fun-asr", { + await env.client.uploadFile(audioUrl, model, { signal: ctx.signal, }), ); } else { - fileUrls.push(u); + fileUrls.push(audioUrl); } } - const model = input.model || "fun-asr"; + if (route.kind === "sync-flash") { + const flashFamily = route.flashFamily!; + const body = buildAsrFlashRequest({ + model, + audioUrl: fileUrls[0]!, + language: input.language, + flashFamily, + }); + const response = await env.client.requestJson>({ + path: route.path, + method: "POST", + body, + signal: ctx.signal, + }); + return { + text: extractAsrFlashText(response, flashFamily), + model, + mode: "sync", + raw: response, + }; + } + const body: DashScopeASRRequest = { model, - input: { file_urls: fileUrls }, + input: + route.asyncInputStyle === "file_url" ? { file_url: fileUrls[0]! } : { file_urls: fileUrls }, parameters: { channel_id: input["channel-id"] !== undefined ? [input["channel-id"]] : undefined, language_hints: input.language ? [input.language] : undefined, @@ -601,9 +671,8 @@ export async function speechRecognize( }; stripUndefined(body.parameters as Record); - const url = speechRecognizePath(); const asyncResp = await env.client.requestJson({ - path: url, + path: speechRecognizePath(), method: "POST", body, async: true, diff --git a/skills/bailian-gen/reference/index.md b/skills/bailian-gen/reference/index.md index 631f8243..b4a8462c 100644 --- a/skills/bailian-gen/reference/index.md +++ b/skills/bailian-gen/reference/index.md @@ -14,7 +14,7 @@ Use this index for the skill-scoped quick index and global flags. | `bl image edit` | Edit an existing image with text instructions (Qwen-Image / Wan 2.7) | [image.md](image.md) | | `bl image generate` | Generate images (Qwen-Image / wan2.x) | [image.md](image.md) | | `bl omni` | Multimodal chat with text + audio output (Qwen-Omni) | [omni.md](omni.md) | -| `bl speech recognize` | Recognize speech from audio files (FunAudio-ASR) | [speech.md](speech.md) | +| `bl speech recognize` | Recognize speech from audio files (FunAudio-ASR / Qwen-ASR Flash) | [speech.md](speech.md) | | `bl speech synthesize` | Synthesize speech from text (CosyVoice TTS) | [speech.md](speech.md) | | `bl video download` | Download a completed video by task ID | [video.md](video.md) | | `bl video edit` | Edit a video with happyhorse-1.0-video-edit (style transfer, object replacement, etc.) | [video.md](video.md) | diff --git a/skills/bailian-gen/reference/speech.md b/skills/bailian-gen/reference/speech.md index 5a23db62..8da88fd8 100644 --- a/skills/bailian-gen/reference/speech.md +++ b/skills/bailian-gen/reference/speech.md @@ -7,37 +7,37 @@ Index: [index.md](index.md) ## Commands in this group -| Command | Description | -| ---------------------- | ------------------------------------------------ | -| `bl speech recognize` | Recognize speech from audio files (FunAudio-ASR) | -| `bl speech synthesize` | Synthesize speech from text (CosyVoice TTS) | +| Command | Description | +| ---------------------- | ----------------------------------------------------------------- | +| `bl speech recognize` | Recognize speech from audio files (FunAudio-ASR / Qwen-ASR Flash) | +| `bl speech synthesize` | Synthesize speech from text (CosyVoice TTS) | ## Command details ### `bl speech recognize` -| Field | Value | -| --------------- | ------------------------------------------------ | -| **Name** | `speech recognize` | -| **Description** | Recognize speech from audio files (FunAudio-ASR) | -| **Usage** | `bl speech recognize --url [flags]` | +| Field | Value | +| --------------- | ----------------------------------------------------------------- | +| **Name** | `speech recognize` | +| **Description** | Recognize speech from audio files (FunAudio-ASR / Qwen-ASR Flash) | +| **Usage** | `bl speech recognize --url [flags]` | #### Flags -| Flag | Type | Required | Description | -| --------------------------- | ------ | -------- | ------------------------------------------------------- | -| `--url ` | array | yes | Audio file URL or local file path (repeatable, max 100) | -| `--model ` | string | no | Model ID (default: fun-asr) | -| `--language ` | string | no | Language hint (e.g. zh, en, ja) | -| `--diarization` | switch | no | Enable automatic speaker diarization | -| `--speaker-count ` | number | no | Expected number of speakers (requires --diarization) | -| `--vocabulary-id ` | string | no | Hot-word vocabulary ID for improved accuracy | -| `--channel-id ` | number | no | Audio channel ID (default: 0) | -| `--out ` | string | no | Save full transcription result to JSON file | -| `--async` | switch | no | Return async task id without waiting | -| `--poll-interval ` | number | no | Polling interval in seconds (default: 2) | -| `--api-key ` | string | no | API key | -| `--base-url ` | string | no | API base URL | +| Flag | Type | Required | Description | +| --------------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------- | +| `--url ` | array | yes | Audio file URL or local file path (repeatable, max 100) | +| `--model ` | string | no | Model ID (default: fun-asr). Async: fun-asr / _-filetrans / paraformer-_; sync: qwen3-asr-flash* / fun-asr-flash* / qwen-audio-\*-asr-flash | +| `--language ` | string | no | Language hint (e.g. zh, en, ja). Async & input-audio sync: language_hints; qwen3 sync: asr_options.language | +| `--diarization` | switch | no | Enable automatic speaker diarization | +| `--speaker-count ` | number | no | Expected number of speakers (requires --diarization) | +| `--vocabulary-id ` | string | no | Hot-word vocabulary ID for improved accuracy | +| `--channel-id ` | number | no | Audio channel ID (default: 0) | +| `--out ` | string | no | Save full transcription result to JSON file | +| `--async` | switch | no | Return async task id without waiting | +| `--poll-interval ` | number | no | Polling interval in seconds (default: 2) | +| `--api-key ` | string | no | API key | +| `--base-url ` | string | no | API base URL | #### Examples @@ -69,6 +69,10 @@ bl speech recognize --url https://example.com/audio.mp3 --out result.json bl speech recognize --url https://example.com/audio.mp3 --async --quiet ``` +```bash +bl speech recognize --url https://example.com/audio.mp3 --model qwen-audio-3.0-asr-flash --language en +``` + ### `bl speech synthesize` | Field | Value | From ddcd564e61e019e9383f842b52d4a4947e23b62c Mon Sep 17 00:00:00 2001 From: clh02467605 Date: Wed, 12 Aug 2026 17:22:25 +0800 Subject: [PATCH 2/5] test: dry-run realtime ASR usage-error e2e to skip auth in CI --- packages/commands/tests/e2e/speech-recognize.e2e.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts index db70875a..8612cc95 100644 --- a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts +++ b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts @@ -83,6 +83,7 @@ describe("e2e: speech recognize", () => { }); test("speech recognize realtime 模型报用法错误", async () => { + // 使用 --dry-run：跳过 auth，避免 CI 无密钥时先以 AUTH(3) 退出 const { stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ "speech", "recognize", @@ -90,6 +91,7 @@ describe("e2e: speech recognize", () => { "qwen3-asr-flash-realtime", "--url", "https://example.com/a.wav", + "--dry-run", "--quiet", ]); expect(exitCode).toBe(2); From 9379da7a4c4687ddf307fa3dfd8e345d13fc1a4e Mon Sep 17 00:00:00 2001 From: clh02467605 Date: Thu, 13 Aug 2026 09:47:09 +0800 Subject: [PATCH 3/5] fix(speech): align flash vocabulary_id and qwen3-filetrans language params --- .../commands/src/commands/speech/recognize.ts | 24 +++++++++---- .../tests/e2e/speech-recognize.e2e.test.ts | 16 +++++++-- packages/core/src/client/asr-routes.ts | 35 +++++++++++++++++-- packages/core/src/client/index.ts | 1 + packages/core/src/types/api.ts | 3 ++ packages/core/tests/asr-routes.test.ts | 19 +++++++++- packages/runtime/src/pipeline/steps/bl-api.ts | 25 ++++++++----- skills/bailian-gen/reference/speech.md | 2 +- 8 files changed, 104 insertions(+), 21 deletions(-) diff --git a/packages/commands/src/commands/speech/recognize.ts b/packages/commands/src/commands/speech/recognize.ts index 31a29637..f6910fca 100644 --- a/packages/commands/src/commands/speech/recognize.ts +++ b/packages/commands/src/commands/speech/recognize.ts @@ -14,6 +14,7 @@ import { speechRecognizePath, resolveAsrApi, buildAsrFlashRequest, + buildAsyncAsrLanguageFields, extractAsrFlashText, type AsrApiRoute, type AsrFlashFamily, @@ -42,7 +43,7 @@ const RECOGNIZE_FLAGS = { type: "string", valueHint: "", description: - "Language hint (e.g. zh, en, ja). Async & input-audio sync: language_hints; qwen3 sync: asr_options.language", + "Language hint (e.g. zh, en, ja). Classic async/input-audio: language_hints; qwen3-filetrans: language; qwen3 sync: asr_options.language", }, diarization: { type: "switch", description: "Enable automatic speaker diarization" }, speakerCount: { @@ -70,11 +71,18 @@ const RECOGNIZE_FLAGS = { } satisfies FlagsDef; type RecognizeFlags = ParsedFlags; -function assertSyncFlashFlagsAllowed(flags: RecognizeFlags, model: string): void { +function assertSyncFlashFlagsAllowed( + flags: RecognizeFlags, + model: string, + flashFamily: AsrFlashFamily, +): void { const unsupported: string[] = []; if (flags.diarization === true) unsupported.push("--diarization"); if (flags.speakerCount !== undefined) unsupported.push("--speaker-count"); - if (flags.vocabularyId !== undefined) unsupported.push("--vocabulary-id"); + // qwen3 sync Flash 不走 vocabulary_id；input-audio Flash（fun-asr-flash* / qwen-audio-*-asr-flash）官方支持 + if (flashFamily === "qwen3" && flags.vocabularyId !== undefined) { + unsupported.push("--vocabulary-id"); + } if (flags.channelId !== undefined) unsupported.push("--channel-id"); if (flags.async === true) unsupported.push("--async"); if (flags.pollInterval !== undefined) unsupported.push("--poll-interval"); @@ -133,7 +141,7 @@ export default defineCommand({ } if (route.kind === "sync-flash") { - assertSyncFlashFlagsAllowed(flags, model); + assertSyncFlashFlagsAllowed(flags, model, route.flashFamily!); if (rawUrls.length !== 1) { throw new BailianError( `Model "${model}" is a sync Flash ASR model and accepts exactly one --url (got ${rawUrls.length}).\n` + @@ -173,8 +181,11 @@ export default defineCommand({ } const channelId = flags.channelId; - const language = flags.language; const vocabularyId = flags.vocabularyId; + const languageFields = buildAsyncAsrLanguageFields( + route.asyncLanguageStyle ?? "language_hints", + flags.language, + ); const body: DashScopeASRRequest = { model, @@ -184,7 +195,7 @@ export default defineCommand({ : { file_urls: resolvedUrls }, parameters: { channel_id: channelId !== undefined ? [channelId] : [0], - language_hints: language ? [language] : undefined, + ...languageFields, diarization_enabled: diarization ? true : undefined, speaker_count: speakerCount, vocabulary_id: vocabularyId, @@ -221,6 +232,7 @@ async function handleSyncFlashMode( model, audioUrl, language: flags.language, + vocabularyId: flags.vocabularyId, flashFamily, }); diff --git a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts index 8612cc95..7ecf8289 100644 --- a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts +++ b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts @@ -32,7 +32,12 @@ describe("e2e: speech recognize", () => { path?: string; request?: { model?: string; - parameters?: { format?: string; language_hints?: string[] }; + parameters?: { + format?: string; + language_hints?: string[]; + language?: string; + vocabulary_id?: string; + }; input?: { file_url?: string; file_urls?: string[]; @@ -60,26 +65,33 @@ describe("e2e: speech recognize", () => { "https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav", "--language", "en", + "--vocabulary-id", + "vocab-e2e", ]); expect(body.mode).toBe("sync"); expect(body.path).toBe("/api/v1/services/aigc/multimodal-generation/generation"); expect(body.request?.model).toBe("qwen-audio-3.0-asr-flash"); expect(body.request?.parameters?.format).toBe("wav"); expect(body.request?.parameters?.language_hints).toEqual(["en"]); + expect(body.request?.parameters?.vocabulary_id).toBe("vocab-e2e"); expect(body.request?.input?.messages?.[0]?.content?.[0]?.type).toBe("input_audio"); }); - test("speech recognize qwen3 filetrans dry-run 使用 file_url 单数字段", async () => { + test("speech recognize qwen3 filetrans dry-run 使用 file_url 与 language", async () => { const body = await runRecognizeDryRun([ "--model", "qwen3-asr-flash-filetrans", "--url", "https://dashscope.oss-cn-beijing.aliyuncs.com/samples/audio/paraformer/hello_world_female2.wav", + "--language", + "zh", ]); expect(body.mode).toBe("async"); expect(body.path).toBe("/api/v1/services/audio/asr/transcription"); expect(body.request?.input?.file_url?.startsWith("https://")).toBe(true); expect(body.request?.input?.file_urls).toBeUndefined(); + expect(body.request?.parameters?.language).toBe("zh"); + expect(body.request?.parameters?.language_hints).toBeUndefined(); }); test("speech recognize realtime 模型报用法错误", async () => { diff --git a/packages/core/src/client/asr-routes.ts b/packages/core/src/client/asr-routes.ts index 3297afe3..4abd7416 100644 --- a/packages/core/src/client/asr-routes.ts +++ b/packages/core/src/client/asr-routes.ts @@ -31,6 +31,12 @@ export interface AsrApiRoute { * - `file_url`: qwen3-asr-flash-filetrans family */ asyncInputStyle?: "file_urls" | "file_url"; + /** + * Async transcription language field style. + * - `language_hints`: fun-asr / paraformer / qwen-audio filetrans... + * - `language`: qwen3-asr-flash-filetrans* + */ + asyncLanguageStyle?: "language_hints" | "language"; flashFamily?: AsrFlashFamily; /** Human-readable reason when kind is unsupported. */ unsupportedReason?: string; @@ -90,11 +96,13 @@ export function resolveAsrApi(model: string): AsrApiRoute { } if (isFiletransModel(model)) { + const isQwen3Filetrans = isQwen3FiletransModel(model); return { kind: "async-filetrans", path: speechRecognizePath(), useSync: false, - asyncInputStyle: isQwen3FiletransModel(model) ? "file_url" : "file_urls", + asyncInputStyle: isQwen3Filetrans ? "file_url" : "file_urls", + asyncLanguageStyle: isQwen3Filetrans ? "language" : "language_hints", }; } @@ -122,6 +130,7 @@ export function resolveAsrApi(model: string): AsrApiRoute { path: speechRecognizePath(), useSync: false, asyncInputStyle: "file_urls", + asyncLanguageStyle: "language_hints", }; } @@ -139,15 +148,32 @@ export interface BuildAsrFlashRequestOpts { model: string; audioUrl: string; language?: string; + /** 预编译热词 ID；仅 input-audio Flash（fun-asr-flash* / qwen-audio-*-asr-flash）官方支持 */ + vocabularyId?: string; flashFamily: AsrFlashFamily; } +/** + * 按异步路由的 language 字段风格构造语种参数。 + * qwen3-asr-flash-filetrans* → `language`；其余异步模型 → `language_hints`。 + */ +export function buildAsyncAsrLanguageFields( + languageStyle: "language_hints" | "language", + language?: string, +): { language_hints?: string[]; language?: string } { + if (!language) return {}; + if (languageStyle === "language") { + return { language }; + } + return { language_hints: [language] }; +} + /** Build a sync multimodal ASR request body for Flash models. */ export function buildAsrFlashRequest(opts: BuildAsrFlashRequestOpts): Record { - const { model, audioUrl, language, flashFamily } = opts; + const { model, audioUrl, language, vocabularyId, flashFamily } = opts; if (flashFamily === "input-audio") { - // 与官方 Qwen-Audio / Fun-ASR-Flash 文档一致：语种走 language_hints + // 与官方 Qwen-Audio / Fun-ASR-Flash 文档一致：语种走 language_hints，热词走 vocabulary_id const parameters: Record = { format: inferAudioFormatHint(audioUrl), sample_rate: "16000", @@ -155,6 +181,9 @@ export function buildAsrFlashRequest(opts: BuildAsrFlashRequestOpts): Record { useSync: false, path: "/api/v1/services/audio/asr/transcription", asyncInputStyle: "file_url", + asyncLanguageStyle: "language", }, }, { @@ -32,6 +34,7 @@ test("resolveAsrApi routes model families correctly", () => { kind: "async-filetrans", useSync: false, asyncInputStyle: "file_urls", + asyncLanguageStyle: "language_hints", }, }, { @@ -112,6 +115,7 @@ test("buildAsrFlashRequest shapes qwen3 and input-audio bodies", () => { model: "qwen-audio-3.0-asr-flash", audioUrl: "https://example.com/a.wav", language: "en", + vocabularyId: "vocab-abc", flashFamily: "input-audio", }), ).toEqual({ @@ -124,8 +128,21 @@ test("buildAsrFlashRequest shapes qwen3 and input-audio bodies", () => { }, ], }, - parameters: { format: "wav", sample_rate: "16000", language_hints: ["en"] }, + parameters: { + format: "wav", + sample_rate: "16000", + language_hints: ["en"], + vocabulary_id: "vocab-abc", + }, + }); +}); + +test("buildAsyncAsrLanguageFields maps language by async style", () => { + expect(buildAsyncAsrLanguageFields("language_hints", "zh")).toEqual({ + language_hints: ["zh"], }); + expect(buildAsyncAsrLanguageFields("language", "zh")).toEqual({ language: "zh" }); + expect(buildAsyncAsrLanguageFields("language", undefined)).toEqual({}); }); test("extractAsrFlashText reads qwen3 choices and input-audio text fields", () => { diff --git a/packages/runtime/src/pipeline/steps/bl-api.ts b/packages/runtime/src/pipeline/steps/bl-api.ts index b6def2d3..a3ea8ca8 100644 --- a/packages/runtime/src/pipeline/steps/bl-api.ts +++ b/packages/runtime/src/pipeline/steps/bl-api.ts @@ -12,6 +12,7 @@ import { speechRecognizePath, resolveAsrApi, buildAsrFlashRequest, + buildAsyncAsrLanguageFields, extractAsrFlashText, stripUndefined, resolveBooleanFlag, @@ -596,15 +597,18 @@ export async function speechRecognize( { step: "speech/recognize" }, ); } - if ( - input.diarization || - input["speaker-count"] !== undefined || - input["vocabulary-id"] !== undefined || - input["channel-id"] !== undefined - ) { + const unsupportedFlags: string[] = []; + if (input.diarization) unsupportedFlags.push("diarization"); + if (input["speaker-count"] !== undefined) unsupportedFlags.push("speaker-count"); + // input-audio Flash 官方支持 vocabulary_id；qwen3 sync Flash 不支持 + if (route.flashFamily === "qwen3" && input["vocabulary-id"] !== undefined) { + unsupportedFlags.push("vocabulary-id"); + } + if (input["channel-id"] !== undefined) unsupportedFlags.push("channel-id"); + if (unsupportedFlags.length > 0) { throw new PipelineError( "invalid_input", - `Model "${model}" uses sync Flash ASR and does not support diarization / speaker-count / vocabulary-id / channel-id`, + `Model "${model}" uses sync Flash ASR and does not support: ${unsupportedFlags.join(", ")}`, { step: "speech/recognize" }, ); } @@ -641,6 +645,7 @@ export async function speechRecognize( model, audioUrl: fileUrls[0]!, language: input.language, + vocabularyId: input["vocabulary-id"], flashFamily, }); const response = await env.client.requestJson>({ @@ -657,13 +662,17 @@ export async function speechRecognize( }; } + const languageFields = buildAsyncAsrLanguageFields( + route.asyncLanguageStyle ?? "language_hints", + input.language, + ); const body: DashScopeASRRequest = { model, input: route.asyncInputStyle === "file_url" ? { file_url: fileUrls[0]! } : { file_urls: fileUrls }, parameters: { channel_id: input["channel-id"] !== undefined ? [input["channel-id"]] : undefined, - language_hints: input.language ? [input.language] : undefined, + ...languageFields, diarization_enabled: input.diarization, speaker_count: input["speaker-count"], vocabulary_id: input["vocabulary-id"], diff --git a/skills/bailian-gen/reference/speech.md b/skills/bailian-gen/reference/speech.md index 8da88fd8..5a65f356 100644 --- a/skills/bailian-gen/reference/speech.md +++ b/skills/bailian-gen/reference/speech.md @@ -28,7 +28,7 @@ Index: [index.md](index.md) | --------------------------- | ------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------- | | `--url ` | array | yes | Audio file URL or local file path (repeatable, max 100) | | `--model ` | string | no | Model ID (default: fun-asr). Async: fun-asr / _-filetrans / paraformer-_; sync: qwen3-asr-flash* / fun-asr-flash* / qwen-audio-\*-asr-flash | -| `--language ` | string | no | Language hint (e.g. zh, en, ja). Async & input-audio sync: language_hints; qwen3 sync: asr_options.language | +| `--language ` | string | no | Language hint (e.g. zh, en, ja). Classic async/input-audio: language_hints; qwen3-filetrans: language; qwen3 sync: asr_options.language | | `--diarization` | switch | no | Enable automatic speaker diarization | | `--speaker-count ` | number | no | Expected number of speakers (requires --diarization) | | `--vocabulary-id ` | string | no | Hot-word vocabulary ID for improved accuracy | From e244771ee9debaf8f743020f1fa1ecdddc67ee9b Mon Sep 17 00:00:00 2001 From: clh02467605 Date: Thu, 13 Aug 2026 14:25:47 +0800 Subject: [PATCH 4/5] test(speech): harden flash ASR contract coverage and docs Add SSE disable header, data-URI format inference, broader response text parsing, HTTP contract e2e, pipeline routing tests, and ASR model selection guidance in bailian-gen. --- .../commands/src/commands/speech/recognize.ts | 1 + .../tests/e2e/speech-recognize.e2e.test.ts | 79 +++++++++++ packages/core/src/client/asr-routes.ts | 23 +++- packages/core/tests/asr-routes.test.ts | 25 ++++ packages/runtime/src/pipeline/steps/bl-api.ts | 1 + .../tests/speech-recognize-pipeline.test.ts | 130 ++++++++++++++++++ skills/bailian-gen/SKILL.md | 2 + 7 files changed, 256 insertions(+), 5 deletions(-) create mode 100644 packages/runtime/tests/speech-recognize-pipeline.test.ts diff --git a/packages/commands/src/commands/speech/recognize.ts b/packages/commands/src/commands/speech/recognize.ts index f6910fca..92b98423 100644 --- a/packages/commands/src/commands/speech/recognize.ts +++ b/packages/commands/src/commands/speech/recognize.ts @@ -248,6 +248,7 @@ async function handleSyncFlashMode( const response = await client.requestJson>({ path: route.path, method: "POST", + headers: { "X-DashScope-SSE": "disable" }, body, }); diff --git a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts index 7ecf8289..fcdac5c0 100644 --- a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts +++ b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts @@ -1,4 +1,6 @@ import { readFileSync } from "node:fs"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; import { join } from "node:path"; import { describe, expect, test } from "vite-plus/test"; import { @@ -109,6 +111,83 @@ describe("e2e: speech recognize", () => { expect(exitCode).toBe(2); expect(stderr).toMatch(/realtime|WebSocket|unsupported/i); }); + + test("speech recognize flash 真实请求走 sync endpoint 并落盘 --out", async () => { + let requestPath = ""; + let requestBody: Record = {}; + let sseHeader: string | undefined; + const server = http.createServer((request, response) => { + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + requestPath = request.url ?? ""; + requestBody = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; + sseHeader = request.headers["x-dashscope-sse"] as string | undefined; + response.writeHead(200, { "Content-Type": "application/json" }); + response.end( + JSON.stringify({ + output: { text: "flash recognition works" }, + request_id: "request-146", + }), + ); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as AddressInfo; + const outDir = makeE2eOutputDir("speech-recognize-flash-sync"); + const outPath = join(outDir, "result.json"); + + try { + const { stdout, stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ + "speech", + "recognize", + "--model", + "fun-asr-flash-2026-06-15", + "--url", + "https://example.com/sample.wav", + "--api-key", + "sk-e2e-placeholder", + "--base-url", + `http://127.0.0.1:${address.port}`, + "--out", + outPath, + "--quiet", + ]); + + expect(exitCode, stderr).toBe(0); + expect(stdout).toContain("flash recognition works"); + expect(requestPath).toBe("/api/v1/services/aigc/multimodal-generation/generation"); + expect(sseHeader).toBe("disable"); + expect(requestBody).toMatchObject({ + model: "fun-asr-flash-2026-06-15", + parameters: { format: "wav" }, + }); + expect(JSON.parse(readFileSync(outPath, "utf8"))).toMatchObject({ + output: { text: "flash recognition works" }, + request_id: "request-146", + }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + test("speech recognize flash 多 --url 在发请求前报用法错误", async () => { + const { stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ + "speech", + "recognize", + "--model", + "qwen-audio-3.0-asr-flash", + "--url", + "https://example.com/a.wav", + "--url", + "https://example.com/b.wav", + "--dry-run", + "--quiet", + ]); + + expect(exitCode).toBe(2); + expect(stderr).toMatch(/exactly one --url|sync Flash/i); + }); }); describe.skipIf(!isBailianE2EMediaEnabled() || !isDashScopeE2EReady())( diff --git a/packages/core/src/client/asr-routes.ts b/packages/core/src/client/asr-routes.ts index 4abd7416..ddf4ead3 100644 --- a/packages/core/src/client/asr-routes.ts +++ b/packages/core/src/client/asr-routes.ts @@ -136,7 +136,15 @@ export function resolveAsrApi(model: string): AsrApiRoute { /** Infer audio container hint for input-audio Flash `parameters.format`. */ export function inferAudioFormatHint(audioUrl: string): string { - const pathPart = audioUrl.split("?")[0] ?? audioUrl; + // data URI：data:audio/mpeg;base64,... → mp3；data:audio/x-wav;... → wav + const dataType = /^data:audio\/([^;,]+)/i.exec(audioUrl)?.[1]?.toLowerCase(); + if (dataType) { + if (dataType === "mpeg") return "mp3"; + if (dataType === "x-wav" || dataType === "wave") return "wav"; + return dataType; + } + + const pathPart = audioUrl.split(/[?#]/, 1)[0] ?? audioUrl; const match = pathPart.match(/\.([a-zA-Z0-9]+)$/); const extension = match?.[1]?.toLowerCase(); if (!extension) return "wav"; @@ -232,7 +240,8 @@ export function buildAsrFlashRequest(opts: BuildAsrFlashRequestOpts): Record, @@ -245,10 +254,14 @@ export function extractAsrFlashText( if (typeof output.text === "string" && output.text.length > 0) { return output.text; } + const topSentence = output.sentence as Record | undefined; + if (typeof topSentence?.text === "string" && topSentence.text.length > 0) { + return topSentence.text; + } const nested = output.output as Record | undefined; - const sentence = nested?.sentence as Record | undefined; - if (typeof sentence?.text === "string") { - return sentence.text; + const nestedSentence = nested?.sentence as Record | undefined; + if (typeof nestedSentence?.text === "string") { + return nestedSentence.text; } return ""; } diff --git a/packages/core/tests/asr-routes.test.ts b/packages/core/tests/asr-routes.test.ts index 03c4ee07..dc2e021c 100644 --- a/packages/core/tests/asr-routes.test.ts +++ b/packages/core/tests/asr-routes.test.ts @@ -92,6 +92,9 @@ test("inferAudioFormatHint reads extension from url", () => { expect(inferAudioFormatHint("oss://bucket/path/file.WAV")).toBe("wav"); expect(inferAudioFormatHint("https://example.com/a.mpeg?x=1")).toBe("mp3"); expect(inferAudioFormatHint("https://example.com/noext")).toBe("wav"); + expect(inferAudioFormatHint("data:audio/mpeg;base64,AAA")).toBe("mp3"); + expect(inferAudioFormatHint("data:audio/x-wav;base64,AAA")).toBe("wav"); + expect(inferAudioFormatHint("data:audio/ogg;codecs=opus;base64,AAA")).toBe("ogg"); }); test("buildAsrFlashRequest shapes qwen3 and input-audio bodies", () => { @@ -168,4 +171,26 @@ test("extractAsrFlashText reads qwen3 choices and input-audio text fields", () = "input-audio", ), ).toBe("Hello World"); + + expect( + extractAsrFlashText( + { + output: { + sentence: { text: "top-level sentence" }, + }, + }, + "input-audio", + ), + ).toBe("top-level sentence"); + + expect( + extractAsrFlashText( + { + output: { + output: { sentence: { text: "nested sentence" } }, + }, + }, + "input-audio", + ), + ).toBe("nested sentence"); }); diff --git a/packages/runtime/src/pipeline/steps/bl-api.ts b/packages/runtime/src/pipeline/steps/bl-api.ts index a3ea8ca8..51c2a573 100644 --- a/packages/runtime/src/pipeline/steps/bl-api.ts +++ b/packages/runtime/src/pipeline/steps/bl-api.ts @@ -651,6 +651,7 @@ export async function speechRecognize( const response = await env.client.requestJson>({ path: route.path, method: "POST", + headers: { "X-DashScope-SSE": "disable" }, body, signal: ctx.signal, }); diff --git a/packages/runtime/tests/speech-recognize-pipeline.test.ts b/packages/runtime/tests/speech-recognize-pipeline.test.ts new file mode 100644 index 00000000..b779cc4e --- /dev/null +++ b/packages/runtime/tests/speech-recognize-pipeline.test.ts @@ -0,0 +1,130 @@ +import { expect, test } from "vite-plus/test"; +import type { Client } from "bailian-cli-core"; +import { PipelineError } from "../src/pipeline/errors.ts"; +import type { PipelineEnv } from "../src/pipeline/bl-config.ts"; +import { speechRecognize } from "../src/pipeline/steps/bl-api.ts"; +import type { StepContext } from "../src/pipeline/types.ts"; + +type CapturedRequest = { + path?: string; + method?: string; + headers?: Record; + body?: Record; + async?: boolean; +}; + +function makeEnv(requestJsonImpl?: (opts: CapturedRequest) => Promise): { + env: PipelineEnv; + captured: CapturedRequest[]; +} { + const captured: CapturedRequest[] = []; + const client = { + uploadFile: async (source: string) => source, + requestJson: async (opts: CapturedRequest) => { + captured.push(opts); + if (requestJsonImpl) return requestJsonImpl(opts); + return { output: { text: "ok" } }; + }, + } as unknown as Client; + + return { + env: { + client, + settings: { quiet: true, output: "json" } as PipelineEnv["settings"], + }, + captured, + }; +} + +function makeCtx(): StepContext { + return { dryRun: false, signal: new AbortController().signal }; +} + +test("pipeline speechRecognize routes input-audio flash to sync multimodal endpoint", async () => { + const { env, captured } = makeEnv(); + const result = (await speechRecognize( + env, + { + url: "https://example.com/a.wav", + model: "qwen-audio-3.0-asr-flash", + language: "en", + "vocabulary-id": "vocab-1", + }, + makeCtx(), + )) as { mode?: string; text?: string }; + + expect(result.mode).toBe("sync"); + expect(result.text).toBe("ok"); + expect(captured).toHaveLength(1); + expect(captured[0]?.path).toBe("/api/v1/services/aigc/multimodal-generation/generation"); + expect(captured[0]?.headers?.["X-DashScope-SSE"]).toBe("disable"); + expect(captured[0]?.body).toMatchObject({ + model: "qwen-audio-3.0-asr-flash", + parameters: { + format: "wav", + language_hints: ["en"], + vocabulary_id: "vocab-1", + }, + }); +}); + +test("pipeline speechRecognize maps qwen3-filetrans language to parameters.language", async () => { + const { env, captured } = makeEnv(async (opts) => { + if (opts.async || opts.method === "POST") { + return { output: { task_id: "task-1", task_status: "PENDING" } }; + } + return { + output: { task_id: "task-1", task_status: "SUCCEEDED", results: [] }, + request_id: "r1", + }; + }); + + await speechRecognize( + env, + { + url: "https://example.com/a.wav", + model: "qwen3-asr-flash-filetrans", + language: "zh", + "poll-interval": 0, + }, + makeCtx(), + ); + + expect(captured[0]?.path).toBe("/api/v1/services/audio/asr/transcription"); + expect(captured[0]?.async).toBe(true); + expect(captured[0]?.body).toMatchObject({ + model: "qwen3-asr-flash-filetrans", + input: { file_url: "https://example.com/a.wav" }, + parameters: { language: "zh" }, + }); + expect( + (captured[0]?.body?.parameters as Record | undefined)?.language_hints, + ).toBeUndefined(); +}); + +test("pipeline speechRecognize rejects realtime models before requesting", async () => { + const { env, captured } = makeEnv(); + await expect( + speechRecognize( + env, + { url: "https://example.com/a.wav", model: "qwen3-asr-flash-realtime" }, + makeCtx(), + ), + ).rejects.toBeInstanceOf(PipelineError); + expect(captured).toHaveLength(0); +}); + +test("pipeline speechRecognize rejects multiple urls for sync flash", async () => { + const { env, captured } = makeEnv(); + await expect( + speechRecognize( + env, + { + url: ["https://example.com/a.wav", "https://example.com/b.wav"], + model: "fun-asr-flash-2026-06-15", + }, + makeCtx(), + ), + ).rejects.toBeInstanceOf(PipelineError); + expect(captured).toHaveLength(0); +}); diff --git a/skills/bailian-gen/SKILL.md b/skills/bailian-gen/SKILL.md index 233e99b8..238185e8 100644 --- a/skills/bailian-gen/SKILL.md +++ b/skills/bailian-gen/SKILL.md @@ -39,6 +39,8 @@ description: >- | A/V understanding (files the host can't play) | `bl omni --video` / `--audio` | `qwen3.5-omni-plus` | | Image/video describe (user names Bailian) | `bl vision describe` | `qwen-vl-max`; host-first for plain image Q&A | +For ASR model selection, keep `fun-asr` (or other `*-filetrans`) for long recordings, repeated files, speaker diarization, or asynchronous task IDs. For one local or remote audio file up to about five minutes when the user asks for low-latency Flash models, use `--model fun-asr-flash-2026-06-15`, `--model qwen-audio-3.0-asr-flash`, or `--model qwen3-asr-flash`. Flash recognition is synchronous and accepts exactly one file per call. + Flags, usage, and examples: see [`reference/`](reference/index.md) or `bl --help` — do not guess flags. ## Local files (mandatory) From ae0c2c12139060d96b30bff4bc4e231d97ba7878 Mon Sep 17 00:00:00 2001 From: clh02467605 Date: Thu, 13 Aug 2026 15:52:34 +0800 Subject: [PATCH 5/5] fix(speech): handle qwen3-filetrans singular result.transcription_url Normalize async ASR transcription items so waiting mode downloads text and --out works without changing shared media task types. --- .../commands/src/commands/speech/recognize.ts | 5 +- .../tests/e2e/speech-recognize.e2e.test.ts | 83 +++++++++++++++- packages/core/src/client/asr-routes.ts | 41 +++++++- packages/core/src/client/index.ts | 1 + packages/core/src/types/api.ts | 25 +++-- packages/core/src/types/index.ts | 1 + packages/core/tests/asr-routes.test.ts | 17 ++++ packages/runtime/src/pipeline/steps/bl-api.ts | 99 +++++++++++++++++-- .../tests/speech-recognize-pipeline.test.ts | 54 ++++++++++ 9 files changed, 304 insertions(+), 22 deletions(-) diff --git a/packages/commands/src/commands/speech/recognize.ts b/packages/commands/src/commands/speech/recognize.ts index 92b98423..0b047e71 100644 --- a/packages/commands/src/commands/speech/recognize.ts +++ b/packages/commands/src/commands/speech/recognize.ts @@ -15,6 +15,7 @@ import { resolveAsrApi, buildAsrFlashRequest, buildAsyncAsrLanguageFields, + collectAsrTranscriptionItems, extractAsrFlashText, type AsrApiRoute, type AsrFlashFamily, @@ -79,7 +80,7 @@ function assertSyncFlashFlagsAllowed( const unsupported: string[] = []; if (flags.diarization === true) unsupported.push("--diarization"); if (flags.speakerCount !== undefined) unsupported.push("--speaker-count"); - // qwen3 sync Flash 不走 vocabulary_id；input-audio Flash（fun-asr-flash* / qwen-audio-*-asr-flash）官方支持 + // qwen3 sync Flash does not use vocabulary_id; input-audio Flash (fun-asr-flash* / qwen-audio-*-asr-flash) does if (flashFamily === "qwen3" && flags.vocabularyId !== undefined) { unsupported.push("--vocabulary-id"); } @@ -308,7 +309,7 @@ async function handleAsyncMode( }, }); - const results = result.output.results ?? []; + const results = collectAsrTranscriptionItems(result.output); if (results.length === 0) { emitResult({ task_id: taskId, status: result.output.task_status }, format); diff --git a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts index fcdac5c0..2805f165 100644 --- a/packages/commands/tests/e2e/speech-recognize.e2e.test.ts +++ b/packages/commands/tests/e2e/speech-recognize.e2e.test.ts @@ -97,7 +97,7 @@ describe("e2e: speech recognize", () => { }); test("speech recognize realtime 模型报用法错误", async () => { - // 使用 --dry-run：跳过 auth，避免 CI 无密钥时先以 AUTH(3) 退出 + // Use --dry-run to skip auth so CI without API keys still hits USAGE(2) const { stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ "speech", "recognize", @@ -188,6 +188,87 @@ describe("e2e: speech recognize", () => { expect(exitCode).toBe(2); expect(stderr).toMatch(/exactly one --url|sync Flash/i); }); + + test("speech recognize qwen3-filetrans 轮询成功后下载 result.transcription_url", async () => { + const server = http.createServer((request, response) => { + const url = request.url ?? ""; + const chunks: Buffer[] = []; + request.on("data", (chunk: Buffer) => chunks.push(chunk)); + request.on("end", () => { + response.writeHead(200, { "Content-Type": "application/json" }); + if (url.startsWith("/api/v1/services/audio/asr/transcription")) { + response.end( + JSON.stringify({ + output: { task_id: "task-qwen3", task_status: "PENDING" }, + request_id: "req-submit", + }), + ); + return; + } + if (url.startsWith("/api/v1/tasks/")) { + const address = server.address() as AddressInfo; + response.end( + JSON.stringify({ + output: { + task_id: "task-qwen3", + task_status: "SUCCEEDED", + result: { + transcription_url: `http://127.0.0.1:${address.port}/transcription.json`, + }, + }, + request_id: "req-poll", + }), + ); + return; + } + if (url.startsWith("/transcription.json")) { + response.end( + JSON.stringify({ + file_url: "https://example.com/a.wav", + transcripts: [{ text: "你好世界", sentences: [{ text: "你好世界" }] }], + }), + ); + return; + } + response.writeHead(404); + response.end(JSON.stringify({ message: `unexpected path: ${url}` })); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address() as AddressInfo; + const outDir = makeE2eOutputDir("speech-recognize-qwen3-filetrans"); + const outPath = join(outDir, "result.json"); + + try { + const { stdout, stderr, exitCode } = await runCommandE2e(SPEECH_ROUTES, [ + "speech", + "recognize", + "--model", + "qwen3-asr-flash-filetrans", + "--url", + "https://example.com/a.wav", + "--language", + "zh", + "--api-key", + "sk-e2e-placeholder", + "--base-url", + `http://127.0.0.1:${address.port}`, + "--poll-interval", + "1", + "--out", + outPath, + "--quiet", + ]); + + expect(exitCode, stderr).toBe(0); + expect(stdout).toContain("你好世界"); + expect(JSON.parse(readFileSync(outPath, "utf8"))).toMatchObject({ + transcripts: [{ text: "你好世界" }], + }); + } finally { + await new Promise((resolve) => server.close(() => resolve())); + } + }); }); describe.skipIf(!isBailianE2EMediaEnabled() || !isDashScopeE2EReady())( diff --git a/packages/core/src/client/asr-routes.ts b/packages/core/src/client/asr-routes.ts index ddf4ead3..0d982f1a 100644 --- a/packages/core/src/client/asr-routes.ts +++ b/packages/core/src/client/asr-routes.ts @@ -136,7 +136,7 @@ export function resolveAsrApi(model: string): AsrApiRoute { /** Infer audio container hint for input-audio Flash `parameters.format`. */ export function inferAudioFormatHint(audioUrl: string): string { - // data URI：data:audio/mpeg;base64,... → mp3；data:audio/x-wav;... → wav + // data URI: data:audio/mpeg;base64,... → mp3; data:audio/x-wav;... → wav const dataType = /^data:audio\/([^;,]+)/i.exec(audioUrl)?.[1]?.toLowerCase(); if (dataType) { if (dataType === "mpeg") return "mp3"; @@ -156,14 +156,14 @@ export interface BuildAsrFlashRequestOpts { model: string; audioUrl: string; language?: string; - /** 预编译热词 ID；仅 input-audio Flash（fun-asr-flash* / qwen-audio-*-asr-flash）官方支持 */ + /** Precompiled hotword vocabulary ID; supported for input-audio Flash (fun-asr-flash* / qwen-audio-*-asr-flash). */ vocabularyId?: string; flashFamily: AsrFlashFamily; } /** - * 按异步路由的 language 字段风格构造语种参数。 - * qwen3-asr-flash-filetrans* → `language`；其余异步模型 → `language_hints`。 + * Build language fields for async ASR routes. + * qwen3-asr-flash-filetrans* → `language`; other async models → `language_hints`. */ export function buildAsyncAsrLanguageFields( languageStyle: "language_hints" | "language", @@ -181,7 +181,7 @@ export function buildAsrFlashRequest(opts: BuildAsrFlashRequestOpts): Record = { format: inferAudioFormatHint(audioUrl), sample_rate: "16000", @@ -294,3 +294,34 @@ export function extractAsrFlashText( } return texts.join(""); } + +/** + * Normalize async ASR task transcription items: + * - classic models: `output.results[]` + * - qwen3-asr-flash-filetrans*: `output.result.transcription_url` + */ +export function collectAsrTranscriptionItems(output: { + results?: Array<{ + file_url?: string; + transcription_url?: string; + subtask_status?: string; + code?: string; + message?: string; + }>; + result?: { transcription_url?: string }; +}): Array<{ + file_url?: string; + transcription_url?: string; + subtask_status?: string; + code?: string; + message?: string; +}> { + if (output.results && output.results.length > 0) { + return output.results; + } + const transcriptionUrl = output.result?.transcription_url; + if (typeof transcriptionUrl === "string" && transcriptionUrl.length > 0) { + return [{ transcription_url: transcriptionUrl, subtask_status: "SUCCEEDED" }]; + } + return []; +} diff --git a/packages/core/src/client/index.ts b/packages/core/src/client/index.ts index cc042479..cd347a2c 100644 --- a/packages/core/src/client/index.ts +++ b/packages/core/src/client/index.ts @@ -37,6 +37,7 @@ export { export { buildAsrFlashRequest, buildAsyncAsrLanguageFields, + collectAsrTranscriptionItems, extractAsrFlashText, inferAudioFormatHint, resolveAsrApi, diff --git a/packages/core/src/types/api.ts b/packages/core/src/types/api.ts index bd130f60..5c10b4a9 100644 --- a/packages/core/src/types/api.ts +++ b/packages/core/src/types/api.ts @@ -538,9 +538,9 @@ export interface DashScopeASRRequest { }; parameters?: { channel_id?: number[]; - /** fun-asr / paraformer / qwen-audio filetrans 等经典异步模型 */ + /** Classic async models (fun-asr / paraformer / qwen-audio filetrans, etc.) */ language_hints?: string[]; - /** qwen3-asr-flash-filetrans* 使用单数字段 language */ + /** qwen3-asr-flash-filetrans* uses singular `language` */ language?: string; diarization_enabled?: boolean; speaker_count?: number; @@ -548,22 +548,31 @@ export interface DashScopeASRRequest { }; } +export interface DashScopeASRTranscriptionItem { + file_url?: string; + transcription_url?: string; + subtask_status?: string; + code?: string; + message?: string; +} + export interface DashScopeASRTaskResult { output: { task_id: string; task_status: "PENDING" | "RUNNING" | "SUCCEEDED" | "FAILED" | "UNKNOWN"; - results?: Array<{ - file_url?: string; + /** Multi-file async results (fun-asr / paraformer / qwen-audio filetrans, etc.) */ + results?: DashScopeASRTranscriptionItem[]; + /** Singular result returned by qwen3-asr-flash-filetrans* on success */ + result?: { transcription_url?: string; - subtask_status?: string; - code?: string; - message?: string; - }>; + }; task_metrics?: { TOTAL: number; SUCCEEDED: number; FAILED: number; }; + code?: string; + message?: string; }; usage?: Record; request_id: string; diff --git a/packages/core/src/types/index.ts b/packages/core/src/types/index.ts index bafd2425..bf547792 100644 --- a/packages/core/src/types/index.ts +++ b/packages/core/src/types/index.ts @@ -48,6 +48,7 @@ export type { ChatTool, DashScopeASRRequest, DashScopeASRTaskResult, + DashScopeASRTranscriptionItem, DashScopeAsyncResponse, DashScopeImageRequest, DashScopeImageSyncResponse, diff --git a/packages/core/tests/asr-routes.test.ts b/packages/core/tests/asr-routes.test.ts index dc2e021c..7b27a1ec 100644 --- a/packages/core/tests/asr-routes.test.ts +++ b/packages/core/tests/asr-routes.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "vite-plus/test"; import { buildAsrFlashRequest, buildAsyncAsrLanguageFields, + collectAsrTranscriptionItems, extractAsrFlashText, inferAudioFormatHint, resolveAsrApi, @@ -194,3 +195,19 @@ test("extractAsrFlashText reads qwen3 choices and input-audio text fields", () = ), ).toBe("nested sentence"); }); + +test("collectAsrTranscriptionItems prefers results[] then singular result", () => { + expect( + collectAsrTranscriptionItems({ + results: [{ transcription_url: "https://example.com/a.json", file_url: "https://a.wav" }], + }), + ).toEqual([{ transcription_url: "https://example.com/a.json", file_url: "https://a.wav" }]); + + expect( + collectAsrTranscriptionItems({ + result: { transcription_url: "https://example.com/qwen3.json" }, + }), + ).toEqual([{ transcription_url: "https://example.com/qwen3.json", subtask_status: "SUCCEEDED" }]); + + expect(collectAsrTranscriptionItems({})).toEqual([]); +}); diff --git a/packages/runtime/src/pipeline/steps/bl-api.ts b/packages/runtime/src/pipeline/steps/bl-api.ts index 51c2a573..8206068f 100644 --- a/packages/runtime/src/pipeline/steps/bl-api.ts +++ b/packages/runtime/src/pipeline/steps/bl-api.ts @@ -13,6 +13,7 @@ import { resolveAsrApi, buildAsrFlashRequest, buildAsyncAsrLanguageFields, + collectAsrTranscriptionItems, extractAsrFlashText, stripUndefined, resolveBooleanFlag, @@ -27,6 +28,7 @@ import { type DashScopeTTSRequest, type DashScopeTTSResponse, type DashScopeASRRequest, + type DashScopeASRTaskResult, type ChatMessageContent, isLocalFile, } from "bailian-cli-core"; @@ -600,7 +602,7 @@ export async function speechRecognize( const unsupportedFlags: string[] = []; if (input.diarization) unsupportedFlags.push("diarization"); if (input["speaker-count"] !== undefined) unsupportedFlags.push("speaker-count"); - // input-audio Flash 官方支持 vocabulary_id；qwen3 sync Flash 不支持 + // input-audio Flash supports vocabulary_id; qwen3 sync Flash does not if (route.flashFamily === "qwen3" && input["vocabulary-id"] !== undefined) { unsupportedFlags.push("vocabulary-id"); } @@ -693,7 +695,65 @@ export async function speechRecognize( const pollIntervalMs = (input["poll-interval"] ?? 2) * 1000; const timeoutMs = (ctx.timeoutSeconds ?? 300) * 1000; - return await pollTaskWithOptions(env, taskId, pollIntervalMs, timeoutMs, ctx); + // ASR polling reads original output, avoids generic flatten (avoids transcription_url polluting media urls) + const asrTask = await pollAsrTaskWithOptions(env, taskId, pollIntervalMs, timeoutMs, ctx); + const transcriptionItems = collectAsrTranscriptionItems(asrTask.output); + + const base: Record = { + task_id: asrTask.output.task_id, + task_status: asrTask.output.task_status, + request_id: asrTask.request_id, + mode: "async", + model, + }; + if (asrTask.output.results) base.results = asrTask.output.results; + if (asrTask.output.result) { + base.result = asrTask.output.result; + if (typeof asrTask.output.result.transcription_url === "string") { + base.transcription_url = asrTask.output.result.transcription_url; + } + } + if (asrTask.output.task_metrics) base.task_metrics = asrTask.output.task_metrics; + if (asrTask.usage) base.usage = asrTask.usage; + + if (transcriptionItems.length === 0) { + return base; + } + + const texts: string[] = []; + const transcripts: Record[] = []; + for (const item of transcriptionItems) { + if (!item.transcription_url) continue; + const transRes = await fetch(item.transcription_url, { signal: ctx.signal }); + if (!transRes.ok) { + throw new PipelineError( + "async_task_failed", + `Failed to download transcription: HTTP ${transRes.status}`, + { step: "speech/recognize", details: { taskId, url: item.transcription_url } }, + ); + } + const transData = (await transRes.json()) as Record; + transcripts.push(transData); + const transcriptList = transData.transcripts as + | Array<{ text?: string; sentences?: Array<{ text?: string }> }> + | undefined; + if (!transcriptList?.length) continue; + for (const transcript of transcriptList) { + if (transcript.sentences?.length) { + for (const sentence of transcript.sentences) { + if (sentence.text) texts.push(sentence.text); + } + } else if (transcript.text) { + texts.push(transcript.text); + } + } + } + + return { + ...base, + text: texts.join("\n"), + transcripts, + }; } // --- Shared: task polling --- @@ -719,7 +779,7 @@ function flattenTaskResponse(resp: DashScopeTaskResponse): Record 0) flat.urls = urls; } if (output.results) { - const urls = output.results.map((r) => r.url).filter(Boolean); + const urls = output.results.map((item) => item.url).filter(Boolean); if (urls.length > 0 && !flat.urls) flat.urls = urls; } if (output.task_metrics) flat.task_metrics = output.task_metrics; @@ -737,13 +797,13 @@ async function pollTask( return await pollTaskWithOptions(env, taskId, pollIntervalMs, timeoutMs, ctx); } -async function pollTaskWithOptions( +async function pollUntilSucceeded( env: PipelineEnv, taskId: string, pollIntervalMs: number, timeoutMs: number, ctx?: StepContext, -): Promise> { +): Promise { const started = Date.now(); let attempt = 0; @@ -766,7 +826,7 @@ async function pollTaskWithOptions( const status = result.output.task_status; if (status === "SUCCEEDED") { - return flattenTaskResponse(result); + return result; } if (status === "FAILED") { @@ -797,6 +857,33 @@ async function pollTaskWithOptions( } } +async function pollTaskWithOptions( + env: PipelineEnv, + taskId: string, + pollIntervalMs: number, + timeoutMs: number, + ctx?: StepContext, +): Promise> { + return flattenTaskResponse(await pollUntilSucceeded(env, taskId, pollIntervalMs, timeoutMs, ctx)); +} + +/** ASR task polling: preserve original output (includes results[] / result.transcription_url). */ +async function pollAsrTaskWithOptions( + env: PipelineEnv, + taskId: string, + pollIntervalMs: number, + timeoutMs: number, + ctx?: StepContext, +): Promise { + return (await pollUntilSucceeded( + env, + taskId, + pollIntervalMs, + timeoutMs, + ctx, + )) as DashScopeASRTaskResult; +} + function delay(ms: number, signal?: AbortSignal): Promise { if (!signal) return new Promise((resolve) => setTimeout(resolve, ms)); return new Promise((resolve, reject) => { diff --git a/packages/runtime/tests/speech-recognize-pipeline.test.ts b/packages/runtime/tests/speech-recognize-pipeline.test.ts index b779cc4e..3ebef528 100644 --- a/packages/runtime/tests/speech-recognize-pipeline.test.ts +++ b/packages/runtime/tests/speech-recognize-pipeline.test.ts @@ -128,3 +128,57 @@ test("pipeline speechRecognize rejects multiple urls for sync flash", async () = ).rejects.toBeInstanceOf(PipelineError); expect(captured).toHaveLength(0); }); + +test("pipeline speechRecognize downloads qwen3 singular result.transcription_url", async () => { + const originalFetch = globalThis.fetch; + const transcriptionUrl = "https://example.com/transcription.json"; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + expect(url).toBe(transcriptionUrl); + return new Response( + JSON.stringify({ + transcripts: [{ text: "pipeline hello", sentences: [{ text: "pipeline hello" }] }], + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }) as typeof fetch; + + try { + const { env, captured } = makeEnv(async (opts) => { + if (opts.async || opts.method === "POST") { + return { output: { task_id: "task-1", task_status: "PENDING" } }; + } + return { + output: { + task_id: "task-1", + task_status: "SUCCEEDED", + result: { transcription_url: transcriptionUrl }, + }, + request_id: "r1", + }; + }); + + const result = (await speechRecognize( + env, + { + url: "https://example.com/a.wav", + model: "qwen3-asr-flash-filetrans", + "poll-interval": 0, + }, + makeCtx(), + )) as { + mode?: string; + text?: string; + transcription_url?: string; + result?: { transcription_url?: string }; + }; + + expect(captured[0]?.async).toBe(true); + expect(result.mode).toBe("async"); + expect(result.text).toBe("pipeline hello"); + expect(result.transcription_url).toBe(transcriptionUrl); + expect(result.result?.transcription_url).toBe(transcriptionUrl); + } finally { + globalThis.fetch = originalFetch; + } +});