识别只返回文本,不投递、不落库:语音支持 mp3/aac/wav/pcm,WebM 先按文件头识别再转 PCM;图片走 qwen-vl-ocr。新增 /health/ai/asr 与 /health/ai/ocr,登录后按用户限流 20 次/分钟,并对 /health/ai/ 前缀放宽请求体上限。
450 lines
14 KiB
JavaScript
450 lines
14 KiB
JavaScript
/**
|
||
* 语音识别适配器:DashScope 实时 WebSocket(Fun-ASR)
|
||
*
|
||
* 与 asr.js 的批式适配器保持同一会话接口:
|
||
* { push(buffer), finish() -> Promise<text>, abort() }
|
||
*
|
||
* 协议(DashScope WebSocket):
|
||
* - 首帧 run-task:task_group=audio / task=asr / function=recognition,
|
||
* 模型默认 fun-asr-realtime
|
||
* - 之后发送音频二进制帧(pcm / wav / mp3 / aac 等由参数 format 指定)
|
||
* - 末帧 finish-task;收到 task-finished 后取累计文本
|
||
* - result-generated 的 payload.output.sentence 里 sentence_end=false 为中间结果,
|
||
* true 为当前句定稿;onPartial 用于实时回显(HTTP 整段上传时可不传)
|
||
*
|
||
* 密钥只从配置或环境变量读取,不回显给调用方。
|
||
*/
|
||
import WebSocket from 'ws';
|
||
import { randomUUID } from 'node:crypto';
|
||
|
||
/** 通义千问实时 ASR 的固定入口(可在配置里覆盖) */
|
||
export const DASHSCOPE_REALTIME_ENDPOINT = 'wss://dashscope.aliyuncs.com/api-ws/v1/inference';
|
||
|
||
/** 缺省模型:Fun-ASR 实时版 */
|
||
export const DASHSCOPE_DEFAULT_MODEL = 'fun-asr-realtime';
|
||
|
||
/** 内部编码名 → DashScope format 参数 */
|
||
const FORMAT_BY_CODEC = {
|
||
pcm16: 'pcm',
|
||
pcm: 'pcm',
|
||
wav: 'wav',
|
||
mp3: 'mp3',
|
||
aac: 'aac',
|
||
opus: 'opus',
|
||
amr: 'amr'
|
||
};
|
||
|
||
/** 只有裸 PCM / WAV 需要显式采样率;压缩格式由文件头自带 */
|
||
const FORMATS_WITH_SAMPLE_RATE = new Set(['pcm', 'wav']);
|
||
|
||
/** WebSocket 已连接状态(避免依赖注入实现里的常量) */
|
||
const WS_OPEN = 1;
|
||
|
||
/** 无 logger 时的空实现,避免调用方必须注入 */
|
||
const NOOP_LOGGER = { info() {}, warn() {}, debug() {} };
|
||
|
||
/**
|
||
* 创建一轮实时识别会话
|
||
*
|
||
* 连接懒建立:首块音频到达时才连供应商,避免空轮次占用供应商连接。
|
||
*
|
||
* @param {Object} [options]
|
||
* @param {string} [options.endpoint] - DashScope WebSocket 地址
|
||
* @param {string} [options.apiKey] - 显式密钥(优先于环境变量)
|
||
* @param {string} [options.apiKeyEnv] - 密钥环境变量名
|
||
* @param {string} [options.model] - 模型名(默认 fun-asr-realtime)
|
||
* @param {string} [options.codec] - 内部编码(默认 pcm16)
|
||
* @param {number} [options.rate] - 采样率(默认 16000,仅 pcm/wav 下发)
|
||
* @param {number} [options.channels] - 声道数(实时协议按单声道)
|
||
* @param {Object} [options.parameters] - 追加到 parameters 的供应商参数
|
||
* @param {string} [options.requestId] - 请求标识(仅日志关联用)
|
||
* @param {Object} [options.logger] - 日志实例
|
||
* @param {Function} [options.onPartial] - 中间结果回调 (text) => void
|
||
* @param {Function} [options.onError] - 采集阶段异步失败回调
|
||
* @param {number} [options.connectTimeoutMs] - 建连 + task-started 超时
|
||
* @param {number} [options.finishTimeoutMs] - finish-task 到 task-finished 的超时
|
||
* @param {number} [options.partialIntervalMs] - 中间结果节流间隔
|
||
* @param {Function} [options.WebSocketImpl] - 测试注入的 WebSocket 实现
|
||
* @returns {{ push: Function, finish: Function, abort: Function }}
|
||
*/
|
||
export function createDashscopeRecognizer({
|
||
endpoint = DASHSCOPE_REALTIME_ENDPOINT,
|
||
apiKey = '',
|
||
apiKeyEnv = '',
|
||
model = DASHSCOPE_DEFAULT_MODEL,
|
||
codec = 'pcm16',
|
||
rate = 16000,
|
||
channels = 1,
|
||
parameters = {},
|
||
requestId = '',
|
||
logger = null,
|
||
onPartial = null,
|
||
onError = null,
|
||
connectTimeoutMs = 10000,
|
||
finishTimeoutMs = 60000,
|
||
partialIntervalMs = 120,
|
||
WebSocketImpl = WebSocket
|
||
} = {}) {
|
||
const _logger = logger || NOOP_LOGGER;
|
||
const key = apiKey || (apiKeyEnv ? process.env[apiKeyEnv] : '') || '';
|
||
const format = FORMAT_BY_CODEC[codec] || codec;
|
||
const taskId = randomUUID().replace(/-/g, '');
|
||
|
||
/** 已结束的句子(sentence_end=true) */
|
||
const sentences = [];
|
||
/** 当前句中间结果(sentence_end=false) */
|
||
let currentText = '';
|
||
/** task-finished 兜底文本(部分供应商只在末帧给全量) */
|
||
let finalText = '';
|
||
|
||
/** 建连前到达的音频(供应商连接未就绪时先缓存) */
|
||
const pendingAudio = [];
|
||
let ws = null;
|
||
let started = false;
|
||
let ready = false;
|
||
let finishing = false;
|
||
let completed = false;
|
||
let aborted = false;
|
||
let failure = null;
|
||
|
||
let connectTimer = null;
|
||
let finishTimer = null;
|
||
let partialTimer = null;
|
||
let pendingPartial = '';
|
||
let lastPartialAt = 0;
|
||
|
||
let resolveReady = null;
|
||
let rejectReady = null;
|
||
let resolveFinish = null;
|
||
let rejectFinish = null;
|
||
|
||
const readyPromise = new Promise((resolve, reject) => {
|
||
resolveReady = resolve;
|
||
rejectReady = reject;
|
||
});
|
||
// 调用方可能不 await(例如采集阶段就失败,由 onError 接管),先挂处理器避免未捕获拒绝
|
||
readyPromise.catch(() => {});
|
||
|
||
const clearTimer = (timer) => {
|
||
if (timer) clearTimeout(timer);
|
||
return null;
|
||
};
|
||
|
||
const composeText = () => `${sentences.join('')}${currentText}`.trim();
|
||
|
||
/** 关闭 socket,不触发失败回调(正常收尾与主动中止都走这里) */
|
||
const closeSocket = () => {
|
||
if (!ws) return;
|
||
const socket = ws;
|
||
ws = null;
|
||
try {
|
||
socket.close();
|
||
} catch {
|
||
// 关闭失败无需处理
|
||
}
|
||
};
|
||
|
||
/** 统一失败出口:先 reject 等待中的 Promise,再按阶段决定是否通知调用方 */
|
||
const fail = (err) => {
|
||
if (failure || completed || aborted) return;
|
||
failure = err;
|
||
ready = false;
|
||
connectTimer = clearTimer(connectTimer);
|
||
finishTimer = clearTimer(finishTimer);
|
||
partialTimer = clearTimer(partialTimer);
|
||
pendingAudio.length = 0;
|
||
|
||
_logger.warn('asr session failed', {
|
||
requestId,
|
||
phase: finishing ? 'finish' : 'stream',
|
||
error: err.message
|
||
});
|
||
|
||
rejectReady?.(err);
|
||
resolveReady = null;
|
||
rejectReady = null;
|
||
|
||
if (rejectFinish) {
|
||
rejectFinish(err);
|
||
resolveFinish = null;
|
||
rejectFinish = null;
|
||
}
|
||
|
||
// finish 阶段由 finish() 的 await 抛出;采集阶段由 onError 让调用方销毁本轮
|
||
if (!finishing) onError?.(err);
|
||
closeSocket();
|
||
};
|
||
|
||
/** 中间结果按固定间隔节流,避免逐字占用 WS 带宽 */
|
||
const emitPartial = (text) => {
|
||
if (!onPartial || !text) return;
|
||
const now = Date.now();
|
||
if (now - lastPartialAt >= partialIntervalMs) {
|
||
lastPartialAt = now;
|
||
onPartial(text);
|
||
return;
|
||
}
|
||
pendingPartial = text;
|
||
if (partialTimer) return;
|
||
partialTimer = setTimeout(() => {
|
||
partialTimer = null;
|
||
if (!pendingPartial) return;
|
||
lastPartialAt = Date.now();
|
||
onPartial(pendingPartial);
|
||
pendingPartial = '';
|
||
}, partialIntervalMs - (now - lastPartialAt));
|
||
};
|
||
|
||
const sendJson = (payload) => {
|
||
if (!ws || ws.readyState !== WS_OPEN) return false;
|
||
ws.send(JSON.stringify(payload));
|
||
return true;
|
||
};
|
||
|
||
const sendAudio = (buffer) => {
|
||
if (!ws || ws.readyState !== WS_OPEN) return false;
|
||
ws.send(buffer);
|
||
return true;
|
||
};
|
||
|
||
/**
|
||
* 等发送队列排空再收尾
|
||
*
|
||
* 整段识别会把一秒以上的音频连续推入,socket 的发送队列往往还没排空;
|
||
* 此时立刻发 finish-task,供应商会在音频到齐前结束任务,实测直接断流
|
||
* (日志表现为 finish 阶段 asr socket closed)。这里等到 bufferedAmount 归零,
|
||
* 最多等 timeoutMs,避免慢网下一直挂着。
|
||
*/
|
||
const drainOutgoing = async (socket, timeoutMs = 3000) => {
|
||
if (!socket || typeof socket.bufferedAmount !== 'number') return;
|
||
const deadline = Date.now() + timeoutMs;
|
||
while (socket === ws && socket.bufferedAmount > 0 && Date.now() < deadline) {
|
||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||
}
|
||
};
|
||
|
||
const flushPendingAudio = () => {
|
||
while (pendingAudio.length && ws && ws.readyState === WS_OPEN) {
|
||
sendAudio(pendingAudio.shift());
|
||
}
|
||
};
|
||
|
||
/** 处理供应商事件:task-started / result-generated / task-finished / task-failed */
|
||
const handleMessage = (raw) => {
|
||
let msg = null;
|
||
try {
|
||
msg = JSON.parse(typeof raw === 'string' ? raw : raw.toString());
|
||
} catch {
|
||
_logger.warn('asr message is not json', { requestId });
|
||
return;
|
||
}
|
||
|
||
const event = msg?.header?.event;
|
||
if (!event) return;
|
||
|
||
if (event === 'task-started') {
|
||
connectTimer = clearTimer(connectTimer);
|
||
ready = true;
|
||
resolveReady?.();
|
||
resolveReady = null;
|
||
rejectReady = null;
|
||
flushPendingAudio();
|
||
return;
|
||
}
|
||
|
||
if (event === 'result-generated') {
|
||
if (completed) return;
|
||
const output = msg.payload?.output || {};
|
||
const sentence = output.sentence;
|
||
if (sentence) {
|
||
const text = sentence.text || '';
|
||
if (sentence.sentence_end === true) {
|
||
if (text) sentences.push(text);
|
||
currentText = '';
|
||
} else {
|
||
currentText = text;
|
||
}
|
||
} else if (output.text) {
|
||
// 部分模型只给全量文本:先当当前句中间结果,末帧再以 task-finished 为准
|
||
currentText = output.text;
|
||
}
|
||
const text = composeText();
|
||
if (text) emitPartial(text);
|
||
return;
|
||
}
|
||
|
||
if (event === 'task-finished') {
|
||
completed = true;
|
||
finishing = false;
|
||
connectTimer = clearTimer(connectTimer);
|
||
finishTimer = clearTimer(finishTimer);
|
||
partialTimer = clearTimer(partialTimer);
|
||
pendingPartial = '';
|
||
finalText = (msg.payload?.output?.text || composeText()).trim();
|
||
_logger.debug('asr task finished', { requestId, chars: finalText.length });
|
||
if (resolveFinish) {
|
||
resolveFinish(finalText);
|
||
resolveFinish = null;
|
||
rejectFinish = null;
|
||
}
|
||
closeSocket();
|
||
return;
|
||
}
|
||
|
||
if (event === 'task-failed') {
|
||
const detail = msg.header?.error_message || msg.header?.error_code || 'asr task failed';
|
||
fail(new Error(detail));
|
||
}
|
||
};
|
||
|
||
/** 建立供应商连接(首块音频时懒触发) */
|
||
const start = () => {
|
||
if (started || aborted || failure) return;
|
||
started = true;
|
||
|
||
if (!key) {
|
||
fail(new Error('asr api key missing'));
|
||
return;
|
||
}
|
||
if (!/^wss?:\/\//i.test(endpoint)) {
|
||
fail(new Error(`invalid asr endpoint: ${endpoint}`));
|
||
return;
|
||
}
|
||
|
||
connectTimer = setTimeout(() => {
|
||
fail(new Error('asr connect timeout'));
|
||
}, connectTimeoutMs);
|
||
|
||
try {
|
||
ws = new WebSocketImpl(endpoint, {
|
||
headers: { Authorization: `Bearer ${key}` }
|
||
});
|
||
} catch (err) {
|
||
fail(err);
|
||
return;
|
||
}
|
||
|
||
ws.on('open', () => {
|
||
const taskParameters = { format };
|
||
// 压缩格式的采样率来自文件头,不再额外声明,避免供应商按错误采样率解码
|
||
if (FORMATS_WITH_SAMPLE_RATE.has(format)) {
|
||
taskParameters.sample_rate = rate;
|
||
}
|
||
sendJson({
|
||
header: {
|
||
action: 'run-task',
|
||
task_id: taskId,
|
||
streaming: 'duplex'
|
||
},
|
||
payload: {
|
||
task_group: 'audio',
|
||
task: 'asr',
|
||
function: 'recognition',
|
||
model,
|
||
parameters: {
|
||
...taskParameters,
|
||
...parameters
|
||
},
|
||
input: {}
|
||
}
|
||
});
|
||
});
|
||
|
||
ws.on('message', handleMessage);
|
||
|
||
ws.on('error', (err) => {
|
||
fail(err instanceof Error ? err : new Error(String(err)));
|
||
});
|
||
|
||
ws.on('close', () => {
|
||
if (completed || aborted || failure) return;
|
||
fail(new Error('asr socket closed'));
|
||
});
|
||
};
|
||
|
||
return {
|
||
/**
|
||
* 推入一个音频分片(原始字节)
|
||
* @param {Buffer} buffer - 分片
|
||
*/
|
||
push(buffer) {
|
||
if (aborted || finishing || failure || !buffer?.length) return;
|
||
start();
|
||
if (!ready) {
|
||
pendingAudio.push(buffer);
|
||
return;
|
||
}
|
||
sendAudio(buffer);
|
||
},
|
||
|
||
/**
|
||
* 结束推流并取定稿
|
||
* @returns {Promise<string>} 定稿(可能为空字符串)
|
||
*/
|
||
async finish() {
|
||
if (aborted) return '';
|
||
if (failure) throw failure;
|
||
if (completed) return finalText;
|
||
|
||
finishing = true;
|
||
start();
|
||
|
||
await readyPromise;
|
||
|
||
if (aborted || failure) throw failure || new Error('asr aborted');
|
||
if (completed) return finalText;
|
||
|
||
// 音频要先全部发出去,再告诉供应商「说完了」,否则会收到半截文本或直接断流
|
||
await drainOutgoing(ws);
|
||
if (aborted || failure) throw failure || new Error('asr aborted');
|
||
if (completed) return finalText;
|
||
|
||
// 先把 Promise 挂上再发 finish-task:同步失败(如 socket 恰好关闭)也能落到 reject
|
||
const done = new Promise((resolve, reject) => {
|
||
resolveFinish = resolve;
|
||
rejectFinish = reject;
|
||
});
|
||
finishTimer = setTimeout(() => {
|
||
fail(new Error('asr finish timeout'));
|
||
}, finishTimeoutMs);
|
||
|
||
const sent = sendJson({
|
||
header: {
|
||
action: 'finish-task',
|
||
task_id: taskId,
|
||
streaming: 'duplex'
|
||
},
|
||
payload: { input: {} }
|
||
});
|
||
if (!sent) {
|
||
fail(new Error('asr socket closed before finish'));
|
||
}
|
||
|
||
const text = await done;
|
||
return (text || '').trim();
|
||
},
|
||
|
||
/** 中止本轮识别:丢弃缓冲并关闭供应商连接,之后 finish 返回空串 */
|
||
abort() {
|
||
if (aborted || completed) return;
|
||
aborted = true;
|
||
connectTimer = clearTimer(connectTimer);
|
||
finishTimer = clearTimer(finishTimer);
|
||
partialTimer = clearTimer(partialTimer);
|
||
pendingAudio.length = 0;
|
||
// 卡在「等建连」的 finish 也要立即退出,否则调用方会永久等待
|
||
rejectReady?.(new Error('asr aborted'));
|
||
resolveReady = null;
|
||
rejectReady = null;
|
||
if (rejectFinish) {
|
||
rejectFinish(new Error('asr aborted'));
|
||
resolveFinish = null;
|
||
rejectFinish = null;
|
||
}
|
||
closeSocket();
|
||
}
|
||
};
|
||
}
|
||
|
||
export default createDashscopeRecognizer;
|