Files
api_health/agent/infra/media/asr_dashscope.js
T
lik 8c6feaf42d feat: 新增语音/图像识别能力(媒体会话、识别接口与路由)
识别只返回文本,不投递、不落库:语音支持 mp3/aac/wav/pcm,WebM 先按文件头识别再转 PCM;图片走 qwen-vl-ocr。新增 /health/ai/asr 与 /health/ai/ocr,登录后按用户限流 20 次/分钟,并对 /health/ai/ 前缀放宽请求体上限。
2026-10-31 17:02:42 +08:00

450 lines
14 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/**
* 语音识别适配器:DashScope 实时 WebSocket(Fun-ASR)
*
* 与 asr.js 的批式适配器保持同一会话接口:
* { push(buffer), finish() -> Promise<text>, abort() }
*
* 协议(DashScope WebSocket):
* - 首帧 run-task:task_group=audio / task=asr / function=recognition,
* 模型默认 fun-asr-realtime
* - 之后发送音频二进制帧(pcm / wav / mp3 / aac 等由参数 format 指定)
* - 末帧 finish-task;收到 task-finished 后取累计文本
* - result-generated 的 payload.output.sentence 里 sentence_end=false 为中间结果,
* true 为当前句定稿;onPartial 用于实时回显(HTTP 整段上传时可不传)
*
* 密钥只从配置或环境变量读取,不回显给调用方。
*/
import WebSocket from 'ws';
import { randomUUID } from 'node:crypto';
/** 通义千问实时 ASR 的固定入口(可在配置里覆盖) */
export const DASHSCOPE_REALTIME_ENDPOINT = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference';
/** 缺省模型:Fun-ASR 实时版 */
export const DASHSCOPE_DEFAULT_MODEL = 'fun-asr-realtime';
/** 内部编码名 → DashScope format 参数 */
const FORMAT_BY_CODEC = {
pcm16: 'pcm',
pcm: 'pcm',
wav: 'wav',
mp3: 'mp3',
aac: 'aac',
opus: 'opus',
amr: 'amr'
};
/** 只有裸 PCM / WAV 需要显式采样率;压缩格式由文件头自带 */
const FORMATS_WITH_SAMPLE_RATE = new Set(['pcm', 'wav']);
/** WebSocket 已连接状态(避免依赖注入实现里的常量) */
const WS_OPEN = 1;
/** 无 logger 时的空实现,避免调用方必须注入 */
const NOOP_LOGGER = { info() {}, warn() {}, debug() {} };
/**
* 创建一轮实时识别会话
*
* 连接懒建立:首块音频到达时才连供应商,避免空轮次占用供应商连接。
*
* @param {Object} [options]
* @param {string} [options.endpoint] - DashScope WebSocket 地址
* @param {string} [options.apiKey] - 显式密钥(优先于环境变量)
* @param {string} [options.apiKeyEnv] - 密钥环境变量名
* @param {string} [options.model] - 模型名(默认 fun-asr-realtime)
* @param {string} [options.codec] - 内部编码(默认 pcm16)
* @param {number} [options.rate] - 采样率(默认 16000,仅 pcm/wav 下发)
* @param {number} [options.channels] - 声道数(实时协议按单声道)
* @param {Object} [options.parameters] - 追加到 parameters 的供应商参数
* @param {string} [options.requestId] - 请求标识(仅日志关联用)
* @param {Object} [options.logger] - 日志实例
* @param {Function} [options.onPartial] - 中间结果回调 (text) => void
* @param {Function} [options.onError] - 采集阶段异步失败回调
* @param {number} [options.connectTimeoutMs] - 建连 + task-started 超时
* @param {number} [options.finishTimeoutMs] - finish-task 到 task-finished 的超时
* @param {number} [options.partialIntervalMs] - 中间结果节流间隔
* @param {Function} [options.WebSocketImpl] - 测试注入的 WebSocket 实现
* @returns {{ push: Function, finish: Function, abort: Function }}
*/
export function createDashscopeRecognizer({
endpoint = DASHSCOPE_REALTIME_ENDPOINT,
apiKey = '',
apiKeyEnv = '',
model = DASHSCOPE_DEFAULT_MODEL,
codec = 'pcm16',
rate = 16000,
channels = 1,
parameters = {},
requestId = '',
logger = null,
onPartial = null,
onError = null,
connectTimeoutMs = 10000,
finishTimeoutMs = 60000,
partialIntervalMs = 120,
WebSocketImpl = WebSocket
} = {}) {
const _logger = logger || NOOP_LOGGER;
const key = apiKey || (apiKeyEnv ? process.env[apiKeyEnv] : '') || '';
const format = FORMAT_BY_CODEC[codec] || codec;
const taskId = randomUUID().replace(/-/g, '');
/** 已结束的句子(sentence_end=true) */
const sentences = [];
/** 当前句中间结果(sentence_end=false) */
let currentText = '';
/** task-finished 兜底文本(部分供应商只在末帧给全量) */
let finalText = '';
/** 建连前到达的音频(供应商连接未就绪时先缓存) */
const pendingAudio = [];
let ws = null;
let started = false;
let ready = false;
let finishing = false;
let completed = false;
let aborted = false;
let failure = null;
let connectTimer = null;
let finishTimer = null;
let partialTimer = null;
let pendingPartial = '';
let lastPartialAt = 0;
let resolveReady = null;
let rejectReady = null;
let resolveFinish = null;
let rejectFinish = null;
const readyPromise = new Promise((resolve, reject) => {
resolveReady = resolve;
rejectReady = reject;
});
// 调用方可能不 await(例如采集阶段就失败,由 onError 接管),先挂处理器避免未捕获拒绝
readyPromise.catch(() => {});
const clearTimer = (timer) => {
if (timer) clearTimeout(timer);
return null;
};
const composeText = () => `${sentences.join('')}${currentText}`.trim();
/** 关闭 socket,不触发失败回调(正常收尾与主动中止都走这里) */
const closeSocket = () => {
if (!ws) return;
const socket = ws;
ws = null;
try {
socket.close();
} catch {
// 关闭失败无需处理
}
};
/** 统一失败出口:先 reject 等待中的 Promise,再按阶段决定是否通知调用方 */
const fail = (err) => {
if (failure || completed || aborted) return;
failure = err;
ready = false;
connectTimer = clearTimer(connectTimer);
finishTimer = clearTimer(finishTimer);
partialTimer = clearTimer(partialTimer);
pendingAudio.length = 0;
_logger.warn('asr session failed', {
requestId,
phase: finishing ? 'finish' : 'stream',
error: err.message
});
rejectReady?.(err);
resolveReady = null;
rejectReady = null;
if (rejectFinish) {
rejectFinish(err);
resolveFinish = null;
rejectFinish = null;
}
// finish 阶段由 finish() 的 await 抛出;采集阶段由 onError 让调用方销毁本轮
if (!finishing) onError?.(err);
closeSocket();
};
/** 中间结果按固定间隔节流,避免逐字占用 WS 带宽 */
const emitPartial = (text) => {
if (!onPartial || !text) return;
const now = Date.now();
if (now - lastPartialAt >= partialIntervalMs) {
lastPartialAt = now;
onPartial(text);
return;
}
pendingPartial = text;
if (partialTimer) return;
partialTimer = setTimeout(() => {
partialTimer = null;
if (!pendingPartial) return;
lastPartialAt = Date.now();
onPartial(pendingPartial);
pendingPartial = '';
}, partialIntervalMs - (now - lastPartialAt));
};
const sendJson = (payload) => {
if (!ws || ws.readyState !== WS_OPEN) return false;
ws.send(JSON.stringify(payload));
return true;
};
const sendAudio = (buffer) => {
if (!ws || ws.readyState !== WS_OPEN) return false;
ws.send(buffer);
return true;
};
/**
* 等发送队列排空再收尾
*
* 整段识别会把一秒以上的音频连续推入,socket 的发送队列往往还没排空;
* 此时立刻发 finish-task,供应商会在音频到齐前结束任务,实测直接断流
* (日志表现为 finish 阶段 asr socket closed)。这里等到 bufferedAmount 归零,
* 最多等 timeoutMs,避免慢网下一直挂着。
*/
const drainOutgoing = async (socket, timeoutMs = 3000) => {
if (!socket || typeof socket.bufferedAmount !== 'number') return;
const deadline = Date.now() + timeoutMs;
while (socket === ws && socket.bufferedAmount > 0 && Date.now() < deadline) {
await new Promise((resolve) => setTimeout(resolve, 20));
}
};
const flushPendingAudio = () => {
while (pendingAudio.length && ws && ws.readyState === WS_OPEN) {
sendAudio(pendingAudio.shift());
}
};
/** 处理供应商事件:task-started / result-generated / task-finished / task-failed */
const handleMessage = (raw) => {
let msg = null;
try {
msg = JSON.parse(typeof raw === 'string' ? raw : raw.toString());
} catch {
_logger.warn('asr message is not json', { requestId });
return;
}
const event = msg?.header?.event;
if (!event) return;
if (event === 'task-started') {
connectTimer = clearTimer(connectTimer);
ready = true;
resolveReady?.();
resolveReady = null;
rejectReady = null;
flushPendingAudio();
return;
}
if (event === 'result-generated') {
if (completed) return;
const output = msg.payload?.output || {};
const sentence = output.sentence;
if (sentence) {
const text = sentence.text || '';
if (sentence.sentence_end === true) {
if (text) sentences.push(text);
currentText = '';
} else {
currentText = text;
}
} else if (output.text) {
// 部分模型只给全量文本:先当当前句中间结果,末帧再以 task-finished 为准
currentText = output.text;
}
const text = composeText();
if (text) emitPartial(text);
return;
}
if (event === 'task-finished') {
completed = true;
finishing = false;
connectTimer = clearTimer(connectTimer);
finishTimer = clearTimer(finishTimer);
partialTimer = clearTimer(partialTimer);
pendingPartial = '';
finalText = (msg.payload?.output?.text || composeText()).trim();
_logger.debug('asr task finished', { requestId, chars: finalText.length });
if (resolveFinish) {
resolveFinish(finalText);
resolveFinish = null;
rejectFinish = null;
}
closeSocket();
return;
}
if (event === 'task-failed') {
const detail = msg.header?.error_message || msg.header?.error_code || 'asr task failed';
fail(new Error(detail));
}
};
/** 建立供应商连接(首块音频时懒触发) */
const start = () => {
if (started || aborted || failure) return;
started = true;
if (!key) {
fail(new Error('asr api key missing'));
return;
}
if (!/^wss?:\/\//i.test(endpoint)) {
fail(new Error(`invalid asr endpoint: ${endpoint}`));
return;
}
connectTimer = setTimeout(() => {
fail(new Error('asr connect timeout'));
}, connectTimeoutMs);
try {
ws = new WebSocketImpl(endpoint, {
headers: { Authorization: `Bearer ${key}` }
});
} catch (err) {
fail(err);
return;
}
ws.on('open', () => {
const taskParameters = { format };
// 压缩格式的采样率来自文件头,不再额外声明,避免供应商按错误采样率解码
if (FORMATS_WITH_SAMPLE_RATE.has(format)) {
taskParameters.sample_rate = rate;
}
sendJson({
header: {
action: 'run-task',
task_id: taskId,
streaming: 'duplex'
},
payload: {
task_group: 'audio',
task: 'asr',
function: 'recognition',
model,
parameters: {
...taskParameters,
...parameters
},
input: {}
}
});
});
ws.on('message', handleMessage);
ws.on('error', (err) => {
fail(err instanceof Error ? err : new Error(String(err)));
});
ws.on('close', () => {
if (completed || aborted || failure) return;
fail(new Error('asr socket closed'));
});
};
return {
/**
* 推入一个音频分片(原始字节)
* @param {Buffer} buffer - 分片
*/
push(buffer) {
if (aborted || finishing || failure || !buffer?.length) return;
start();
if (!ready) {
pendingAudio.push(buffer);
return;
}
sendAudio(buffer);
},
/**
* 结束推流并取定稿
* @returns {Promise<string>} 定稿(可能为空字符串)
*/
async finish() {
if (aborted) return '';
if (failure) throw failure;
if (completed) return finalText;
finishing = true;
start();
await readyPromise;
if (aborted || failure) throw failure || new Error('asr aborted');
if (completed) return finalText;
// 音频要先全部发出去,再告诉供应商「说完了」,否则会收到半截文本或直接断流
await drainOutgoing(ws);
if (aborted || failure) throw failure || new Error('asr aborted');
if (completed) return finalText;
// 先把 Promise 挂上再发 finish-task:同步失败(如 socket 恰好关闭)也能落到 reject
const done = new Promise((resolve, reject) => {
resolveFinish = resolve;
rejectFinish = reject;
});
finishTimer = setTimeout(() => {
fail(new Error('asr finish timeout'));
}, finishTimeoutMs);
const sent = sendJson({
header: {
action: 'finish-task',
task_id: taskId,
streaming: 'duplex'
},
payload: { input: {} }
});
if (!sent) {
fail(new Error('asr socket closed before finish'));
}
const text = await done;
return (text || '').trim();
},
/** 中止本轮识别:丢弃缓冲并关闭供应商连接,之后 finish 返回空串 */
abort() {
if (aborted || completed) return;
aborted = true;
connectTimer = clearTimer(connectTimer);
finishTimer = clearTimer(finishTimer);
partialTimer = clearTimer(partialTimer);
pendingAudio.length = 0;
// 卡在「等建连」的 finish 也要立即退出,否则调用方会永久等待
rejectReady?.(new Error('asr aborted'));
resolveReady = null;
rejectReady = null;
if (rejectFinish) {
rejectFinish(new Error('asr aborted'));
resolveFinish = null;
rejectFinish = null;
}
closeSocket();
}
};
}
export default createDashscopeRecognizer;