Files
api_health/websocket.js
T
lik c5415c2c9f feat: /chat 支持实时语音识别,WebM 分片服务端流式转码
上行 asr_start/asr_chunk/asr_stop/asr_cancel,下行 asr_partial/asr_final/asr_error。首帧判容器:mp3 直接透传,WebM 边收边解码成 16bit 单声道 PCM(48k 抽取到 16k)再推供应商;认不出容器先缓存 16KB 再判,失败回 asr_error 让客户端整段兜底。分片串行处理,断开连接时释放会话与解码器。
2026-10-31 17:02:48 +08:00

550 lines
18 KiB
JavaScript

import WebSocket, { WebSocketServer } from 'ws';
import http from 'http';
import { DBModel } from "./models/index.js";
import { chatTask } from "./agent/bots/bot-escort/task.js";
import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js"
import { getUserByToken } from "./utils/user_service.js";
import { isAdmin } from "./utils/permissions.js";
import { createUserLimiter } from "./middleware/ratelimit.js";
import {
createAudioStreamRecognizer,
isAsrConfigured,
normalizeAudioMeta,
decodeBase64Media
} from "./agent/infra/media/index.js";
import {
sniffAudioFormat,
createStreamingWebmDecoder,
downsamplePcm16
} from "./agent/infra/media/audio.js";
let systemMessageSequence = 0;
// 单连接上行帧上限:超限会被 ws 直接断开,避免超大 JSON 占用内存与 CPU
const MAX_PAYLOAD_BYTES = 256 * 1024;
// 单用户聊天频次限制:AI 调用有成本,且同一用户的会话本就串行处理
const chatLimiter = createUserLimiter(20);
// 实时语音识别同样按次计费:每轮 asr_start 计一次,与聊天额度分开
const asrLimiter = createUserLimiter(20);
// 实时识别分片上限:单帧按客户端 frameSize=8KB 留足余量,
// 整轮按 60 秒录音的一倍余量收口,超过直接中止本轮,避免连接内存被刷满
const ASR_LIMITS = {
chunkBytes: 128 * 1024,
totalBytes: 8 * 1024 * 1024,
// 首帧认不出容器时先缓存再判(有些平台的首帧只是 cluster,没有文件头)
sniffBytes: 16 * 1024
};
/**
* WebM 解码出的 PCM 采样率归一
*
* 实时协议对裸 PCM 只稳定支持 16k:48k/24k 这类整数倍能整除到 16k,
* 直接降到 16k;其他采样率保持原值,避免声明与数据不一致。
*/
const resolveAsrRate = (sourceRate) => {
const rate = Number(sourceRate) || 48000;
return Number.isInteger(rate / 16000) && rate > 16000 ? 16000 : rate;
};
const sendFrame = (ws, source, type, content = '', id = '') => {
if (ws.readyState !== WebSocket.OPEN) return;
ws.send(JSON.stringify({
source,
type,
content,
id: id || `system_${Date.now()}_${++systemMessageSequence}`
}));
};
export default class WebSocketServerManager {
constructor(port = 8080) {
this.port = port;
this.wss = null;
this.server = null;
}
start() {
this.server = http.createServer((req, res) => {
res.writeHead(426, { 'Content-Type': 'text/plain' });
res.end('Upgrade Required: WebSocket expected');
});
this.wss = new WebSocketServer({
server: this.server,
maxPayload: MAX_PAYLOAD_BYTES,
perMessageDeflate: {
zlibDeflateOptions: {
chunkSize: 1024,
memLevel: 7,
level: 3
},
zlibInflateOptions: {
chunkSize: 10 * 1024
},
clientNoContextTakeover: true,
serverNoContextTakeover: true,
serverMaxWindowBits: 10,
concurrencyLimit: 10,
threshold: 1024
}
});
this.wss.on('connection', (ws, req) => {
const path = req.url;
console.log(`WebSocket connected: ${req.socket.remoteAddress}, path: ${path}`);
if (path === '/chat') {
this._handleChatConnection(ws);
} else {
sendFrame(ws, 'system', 'error', 'Unknown path');
ws.close(1000, 'Unknown path');
}
});
this.server.listen(this.port, () => {
console.log(`WebSocket server started on ws://localhost:${this.port}`);
});
}
_handleChatConnection(ws) {
sendFrame(ws, 'system', 'connected', 'Connected to chat server');
// 同一连接的消息串行处理:agent 的 messages 数组是共享状态,
// 并发跑同一个 agent 会让上下文交错、回复错乱
ws.messageQueue = Promise.resolve();
ws.on('message', (data) => {
ws.messageQueue = ws.messageQueue
.then(() => this._handleMessage(ws, data))
.catch((err) => console.error('WebSocket message handling error:', err));
});
ws.on('close', () => {
// 断开后释放在途的识别会话与 WebM 解码器:不释放会让供应商连接与缓冲悬空
this._abortAsrSession(ws);
// 断开后释放队列引用,避免连接对象被闭包长期持有
ws.messageQueue = null;
console.log('Chat client disconnected');
});
ws.on('error', (err) => {
console.error('WebSocket error:', err);
});
}
async _handleMessage(ws, data) {
try {
const raw = data.toString();
const msg = JSON.parse(raw);
// 只记录消息类型:正文含健康描述、token/userId 属敏感信息,不进日志。
// 实时识别的分片按秒级频率到达,逐帧打日志会把日志刷满,这里跳过
if (msg.type !== 'asr_chunk') {
console.log('Received message type:', msg.type);
}
if (msg.type === 'ping') {
sendFrame(ws, 'system', 'pong', '', msg.id);
return;
}
// 实时语音识别:与聊天共用一条连接,一轮识别 = start → 若干 chunk → stop/cancel
if (msg.type === 'asr_start') {
await this._handleAsrStart(ws, msg);
return;
}
if (msg.type === 'asr_chunk') {
this._handleAsrChunk(ws, msg);
return;
}
if (msg.type === 'asr_stop') {
await this._handleAsrStop(ws, msg);
return;
}
if (msg.type === 'asr_cancel') {
this._handleAsrCancel(ws, msg);
return;
}
if (msg.type === 'chat' || msg.type === 'clear') {
// 客户端断开后不再处理消息
if (ws.readyState !== WebSocket.OPEN) return;
const send = (source, type, content, id) => {
sendFrame(ws, source, type, content, id);
};
try {
// 身份只认 token:不再接受凭 userId 明文接入,防止冒充他人聊天
if (!msg.token) {
send('system', 'error', '请先登录', msg.id);
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
send('system', 'error', '请先登录', msg.id);
return;
}
// 频次限制按用户维度:防止单账号刷 AI 调用(同一账号多连接共享额度)
if (msg.type === 'chat') {
const limit = chatLimiter(String(userInfo._id));
if (!limit.allowed) {
send('system', 'error', `操作过于频繁,请 ${limit.retryAfter} 秒后重试`, msg.id);
return;
}
}
if (msg.agent === 'escort-admin') {
// 管理端 agent 含写库/环境变量等高危工具,必须校验管理员角色,防止普通用户越权
if (!isAdmin(userInfo)) {
send('system', 'error', '无管理员权限', msg.id);
return;
}
await adminAgent.streamChat(userInfo, [msg], (source, type, content, id) => {
send(source, type, content, id);
});
} else {
await chatTask.streamChat(userInfo, msg, (source, type, content, id) => {
send(source, type, content, id);
});
}
// 流式 chunk 可能携带不同的模型消息 ID,结束帧统一关联本次客户端请求。
send('system', 'end', '', msg.id);
} catch (err) {
console.error('streamChat error:', err);
send('system', 'error', '处理消息失败,请稍后重试', msg.id);
}
return;
}
if (msg.type === 'system') {
console.log('[System]', msg.content);
return;
}
console.log('[Unknown message type]', msg.type);
} catch (e) {
console.error('Message parse error:', e);
sendFrame(ws, 'system', 'error', '消息格式错误');
}
}
/** 释放当前连接的识别会话(主动中止,不会让供应商再回结果) */
_abortAsrSession(ws) {
const state = ws.asrSession;
ws.asrSession = null;
if (state) {
state.session?.abort();
state.decoder?.free();
}
return state;
}
/**
* 建供应商会话(首帧确认真实容器后才建,避免空轮次占用连接)
*
* 采集阶段失败统一走 onError:客户端收到 asr_error 后继续录完整句,
* 结束时改走整段 /health/ai/asr。
*/
_createAsrSession(ws, state, { codec, rate }) {
const session = createAudioStreamRecognizer({
codec,
rate,
channels: state.meta.channels,
requestId: `asr_stream_${state.id}`,
// 中间结果即时回显;客户端按 id 过滤,只认本轮
onPartial: (text) => {
if (ws.asrSession !== state) return;
sendFrame(ws, 'system', 'asr_partial', text, state.id);
},
onError: (err) => {
if (ws.asrSession !== state) return;
state.failed = true;
state.session = null;
console.warn('[ASR] stream failed:', err.message, { requestId: state.id });
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
}
});
state.session = session;
return session;
}
/** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */
_failAsrStream(ws, state, message) {
if (ws.asrSession === state) ws.asrSession = null;
state.failed = true;
state.session?.abort();
state.session = null;
state.decoder?.free();
state.decoder = null;
sendFrame(ws, 'system', 'asr_error', message, state.id);
}
/**
* 开始一轮实时识别
* 上行:{ type: 'asr_start', id, token, format?, sampleRate?, channels? }
* 身份只认 token,与聊天同一套鉴权;每轮按用户限流一次
*/
async _handleAsrStart(ws, msg) {
// 同一连接不叠加两轮识别:先中止上一轮,避免两份供应商会话并行计费
this._abortAsrSession(ws);
const fail = (content) => sendFrame(ws, 'system', 'asr_error', content, msg.id);
try {
if (!msg.token) {
fail('请先登录');
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
fail('请先登录');
return;
}
if (!isAsrConfigured()) {
fail('语音识别服务暂不可用');
return;
}
const limit = asrLimiter(String(userInfo._id));
if (!limit.allowed) {
fail(`操作过于频繁,请 ${limit.retryAfter} 秒后重试`);
return;
}
let meta;
try {
meta = normalizeAudioMeta({
format: msg.format || 'mp3',
sampleRate: msg.sampleRate,
channels: msg.channels
});
} catch (err) {
fail(err.message || '语音参数不支持');
return;
}
// 供应商会话按首帧确认真实容器后再建:真机是真 mp3,浏览器(开发者工具)是 WebM,
// 后者要先流式解码成 PCM 才能推给供应商,两种编码不能共用一份 run-task 参数
const state = {
id: msg.id || `asr_${Date.now()}`,
userId: String(userInfo._id),
meta,
bytes: 0,
failed: false,
format: '',
pending: [],
decoder: null,
// 推给供应商的 PCM 采样率(WebM 解码后归一,mp3 链路不用)
rate: 0,
session: null,
// 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱
queue: Promise.resolve()
};
ws.asrSession = state;
} catch (err) {
console.error('[ASR] start failed:', err);
fail('语音识别服务暂不可用');
}
}
/**
* 推入一帧音频(base64)
* 单帧解码失败只丢这一帧:录音仍在继续,不该因一帧异常打断整轮
*
* 分片串行排队处理:WebM 解码和供应商推送都是异步的,
* 并发处理会让 PCM 顺序错乱,也会把供应商会话的 run-task 竞态化。
*/
_handleAsrChunk(ws, msg) {
const state = ws.asrSession;
if (!state || state.failed) return;
if (msg.id && msg.id !== state.id) return;
let buffer;
try {
buffer = decodeBase64Media(msg.audio, {
maxBytes: ASR_LIMITS.chunkBytes,
label: '语音分片'
});
} catch (err) {
return;
}
if (state.bytes + buffer.length > ASR_LIMITS.totalBytes) {
this._failAsrStream(ws, state, '录音过长,请缩短语音后重试');
return;
}
state.bytes += buffer.length;
state.queue = state.queue
.then(() => this._consumeAsrChunk(ws, state, buffer))
.catch((err) => {
console.warn('[ASR] chunk failed:', err?.message ?? err, { requestId: state.id });
});
}
/**
* 处理一帧音频:首帧先判真实容器,再决定直接推 mp3 还是先流式解码 WebM
*
* 真机录的是 mp3 分片,容器直接交给供应商;
* 开发者工具录的是 WebM/Opus,供应商不认这个容器,必须边收边解码成 PCM。
*/
async _consumeAsrChunk(ws, state, buffer) {
if (ws.asrSession !== state || state.failed) return;
if (!state.format) {
state.pending.push(buffer);
const head = Buffer.concat(state.pending);
const sniffed = sniffAudioFormat(head);
// 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判
if (sniffed === 'unknown') {
if (head.length < ASR_LIMITS.sniffBytes) return;
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
if (sniffed !== 'mp3' && sniffed !== 'webm') {
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
state.format = sniffed;
state.pending = [];
if (sniffed === 'webm') {
try {
state.decoder = await createStreamingWebmDecoder();
} catch (err) {
console.warn('[ASR] webm decoder init failed:', err?.message ?? err, { requestId: state.id });
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
return;
}
// 供应商会话等第一段 PCM 解出来再建:采样率以解码器实际输出为准
await this._pushAsrAudio(ws, state, head);
return;
}
this._createAsrSession(ws, state, { codec: state.meta.codec, rate: state.meta.rate });
await this._pushAsrAudio(ws, state, head);
return;
}
await this._pushAsrAudio(ws, state, buffer);
}
/** 把一帧音频推给供应商:WebM 先解码成 PCM,mp3 原样推 */
async _pushAsrAudio(ws, state, buffer) {
if (ws.asrSession !== state || state.failed) return;
try {
if (state.decoder) {
const pcm = await state.decoder.push(buffer);
if (!state.session) {
// 采样率以首帧解码结果为准:Opus 是 48k,整数倍抽取到 16k
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
const out = pcm.length ? downsamplePcm16(pcm, state.decoder.rate, state.rate) : pcm;
if (out.length) state.session.push(out);
return;
}
if (state.session) state.session.push(buffer);
} catch (err) {
// 解码失败说明这一轮的分片无法拼成完整音频,交给客户端结束后的整段兜底
console.warn('[ASR] audio decode failed:', err?.message ?? err, { requestId: state.id });
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
}
}
/**
* 结束本轮并回定稿
* 上行:{ type: 'asr_stop', id }
* 下行:asr_final(可能为空串,由客户端提示重录)
*/
async _handleAsrStop(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
// 采集阶段的失败已经回过 asr_error,这里只需收尾
if (state.failed) {
this._abortAsrSession(ws);
return;
}
try {
// 先把排队的最后一帧处理完,否则会漏掉句尾(也会让 WebM 少解码一段)
await state.queue;
if (ws.asrSession !== state || state.failed) return;
// WebM 流式解码:把解码器里剩余的帧冲出来再收尾
if (state.decoder) {
const tail = await state.decoder.flush();
if (!state.session && tail.length) {
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
if (tail.length && state.session) {
const out = downsamplePcm16(tail, state.decoder.rate, state.rate);
if (out.length) state.session.push(out);
}
}
const text = state.session ? (await state.session.finish()) || '' : '';
if (ws.asrSession === state) ws.asrSession = null;
sendFrame(ws, 'system', 'asr_final', text, state.id);
} catch (err) {
if (ws.asrSession === state) ws.asrSession = null;
state.session?.abort();
console.warn('[ASR] finish failed:', err.message, { requestId: state.id, bytes: state.bytes });
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
} finally {
state.decoder?.free();
state.decoder = null;
}
}
/**
* 取消本轮(页面隐藏、录音中断、客户端兜底失败)
* 只中止会话,不回任何帧,也不产生定稿
*/
_handleAsrCancel(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
this._abortAsrSession(ws);
}
broadcast(message) {
if (!this.wss) return;
this.wss.clients.forEach((client) => {
if (client.readyState === WebSocket.OPEN) {
client.send(message);
}
});
}
stop() {
if (this.wss) {
this.wss.close();
this.wss = null;
}
if (this.server) {
this.server.close();
this.server = null;
}
console.log('WebSocket server stopped');
}
getClientsCount() {
if (!this.wss) return 0;
return this.wss.clients.size;
}
}