Files
api_health/websocket.js
T
lik 47814f4c24 fix: 修复实时语音 asr_start 直接失败,并补齐全链路诊断与 EBML 头裁剪
- websocket.js 补上缺失的 logger 导入:asr_start 里 logger.info 抛 ReferenceError 被 catch 兜住,
  客户端只会收到「语音识别服务暂不可用」,实时回显整条链路直接失效
- asr_start 失败时中止本次会话,避免遗留半截 asrSession 让后续分片误判
- onError / 降级 / 空音频收尾统一走 logger,记录 format/bytes/partialSeen,便于区分
  「没发分片」「解码失败」「供应商失败」三种情况
- 首帧嗅探失败时记录前 48 字节 hex;新增 findEbmlHeader,在头 4KB 内定位 EBML 魔数,
  命中后裁掉前导字节按 WebM 继续流式解码(开发者工具分片头不一定在偏移 0)
- WebM 首帧只有容器头、解不出 PCM 时不提前建供应商会话
2026-10-31 18:50:02 +08:00

636 lines
21 KiB
JavaScript

import WebSocket, { WebSocketServer } from 'ws';
import http from 'http';
import { DBModel } from "./models/index.js";
import { chatTask } from "./agent/bots/bot-escort/task.js";
import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js"
import { getUserByToken } from "./utils/user_service.js";
import { isAdmin } from "./utils/permissions.js";
import logger from "./utils/logger.js";
import { createUserLimiter } from "./middleware/ratelimit.js";
import {
createAudioStreamRecognizer,
isAsrConfigured,
normalizeAudioMeta,
decodeBase64Media
} from "./agent/infra/media/index.js";
import {
sniffAudioFormat,
findEbmlHeader,
createStreamingWebmDecoder,
downsamplePcm16
} from "./agent/infra/media/audio.js";
let systemMessageSequence = 0;
// 单连接上行帧上限:超限会被 ws 直接断开,避免超大 JSON 占用内存与 CPU
const MAX_PAYLOAD_BYTES = 256 * 1024;
// 单用户聊天频次限制:AI 调用有成本,且同一用户的会话本就串行处理
const chatLimiter = createUserLimiter(20);
// 实时语音识别同样按次计费:每轮 asr_start 计一次,与聊天额度分开
const asrLimiter = createUserLimiter(20);
// 实时识别分片上限:单帧按客户端 frameSize=8KB 留足余量,
// 整轮按 60 秒录音的一倍余量收口,超过直接中止本轮,避免连接内存被刷满
const ASR_LIMITS = {
chunkBytes: 128 * 1024,
totalBytes: 8 * 1024 * 1024,
// 首帧认不出容器时先缓存再判(有些平台的首帧只是 cluster,没有文件头)
sniffBytes: 16 * 1024
};
/**
* WebM 解码出的 PCM 采样率归一
*
* 实时协议对裸 PCM 只稳定支持 16k:48k/24k 这类整数倍能整除到 16k,
* 直接降到 16k;其他采样率保持原值,避免声明与数据不一致。
*/
const resolveAsrRate = (sourceRate) => {
const rate = Number(sourceRate) || 48000;
return Number.isInteger(rate / 16000) && rate > 16000 ? 16000 : rate;
};
const sendFrame = (ws, source, type, content = '', id = '') => {
if (ws.readyState !== WebSocket.OPEN) return;
ws.send(JSON.stringify({
source,
type,
content,
id: id || `system_${Date.now()}_${++systemMessageSequence}`
}));
};
export default class WebSocketServerManager {
constructor(port = 8080) {
this.port = port;
this.wss = null;
this.server = null;
}
start() {
this.server = http.createServer((req, res) => {
res.writeHead(426, { 'Content-Type': 'text/plain' });
res.end('Upgrade Required: WebSocket expected');
});
this.wss = new WebSocketServer({
server: this.server,
maxPayload: MAX_PAYLOAD_BYTES,
perMessageDeflate: {
zlibDeflateOptions: {
chunkSize: 1024,
memLevel: 7,
level: 3
},
zlibInflateOptions: {
chunkSize: 10 * 1024
},
clientNoContextTakeover: true,
serverNoContextTakeover: true,
serverMaxWindowBits: 10,
concurrencyLimit: 10,
threshold: 1024
}
});
this.wss.on('connection', (ws, req) => {
const path = req.url;
console.log(`WebSocket connected: ${req.socket.remoteAddress}, path: ${path}`);
if (path === '/chat') {
this._handleChatConnection(ws);
} else {
sendFrame(ws, 'system', 'error', 'Unknown path');
ws.close(1000, 'Unknown path');
}
});
this.server.listen(this.port, () => {
console.log(`WebSocket server started on ws://localhost:${this.port}`);
});
}
_handleChatConnection(ws) {
sendFrame(ws, 'system', 'connected', 'Connected to chat server');
// 同一连接的消息串行处理:agent 的 messages 数组是共享状态,
// 并发跑同一个 agent 会让上下文交错、回复错乱
ws.messageQueue = Promise.resolve();
ws.on('message', (data) => {
ws.messageQueue = ws.messageQueue
.then(() => this._handleMessage(ws, data))
.catch((err) => console.error('WebSocket message handling error:', err));
});
ws.on('close', () => {
// 断开后释放在途的识别会话与 WebM 解码器:不释放会让供应商连接与缓冲悬空
this._abortAsrSession(ws);
// 断开后释放队列引用,避免连接对象被闭包长期持有
ws.messageQueue = null;
console.log('Chat client disconnected');
});
ws.on('error', (err) => {
console.error('WebSocket error:', err);
});
}
async _handleMessage(ws, data) {
try {
const raw = data.toString();
const msg = JSON.parse(raw);
// 只记录消息类型:正文含健康描述、token/userId 属敏感信息,不进日志。
// 实时识别的分片按秒级频率到达,逐帧打日志会把日志刷满,这里跳过
if (msg.type !== 'asr_chunk') {
console.log('Received message type:', msg.type);
}
if (msg.type === 'ping') {
sendFrame(ws, 'system', 'pong', '', msg.id);
return;
}
// 实时语音识别:与聊天共用一条连接,一轮识别 = start → 若干 chunk → stop/cancel
if (msg.type === 'asr_start') {
await this._handleAsrStart(ws, msg);
return;
}
if (msg.type === 'asr_chunk') {
this._handleAsrChunk(ws, msg);
return;
}
if (msg.type === 'asr_stop') {
await this._handleAsrStop(ws, msg);
return;
}
if (msg.type === 'asr_cancel') {
this._handleAsrCancel(ws, msg);
return;
}
if (msg.type === 'chat' || msg.type === 'clear') {
// 客户端断开后不再处理消息
if (ws.readyState !== WebSocket.OPEN) return;
const send = (source, type, content, id) => {
sendFrame(ws, source, type, content, id);
};
try {
// 身份只认 token:不再接受凭 userId 明文接入,防止冒充他人聊天
if (!msg.token) {
send('system', 'error', '请先登录', msg.id);
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
send('system', 'error', '请先登录', msg.id);
return;
}
// 频次限制按用户维度:防止单账号刷 AI 调用(同一账号多连接共享额度)
if (msg.type === 'chat') {
const limit = chatLimiter(String(userInfo._id));
if (!limit.allowed) {
send('system', 'error', `操作过于频繁,请 ${limit.retryAfter} 秒后重试`, msg.id);
return;
}
}
if (msg.agent === 'escort-admin') {
// 管理端 agent 含写库/环境变量等高危工具,必须校验管理员角色,防止普通用户越权
if (!isAdmin(userInfo)) {
send('system', 'error', '无管理员权限', msg.id);
return;
}
await adminAgent.streamChat(userInfo, [msg], (source, type, content, id) => {
send(source, type, content, id);
});
} else {
await chatTask.streamChat(userInfo, msg, (source, type, content, id) => {
send(source, type, content, id);
});
}
// 流式 chunk 可能携带不同的模型消息 ID,结束帧统一关联本次客户端请求。
send('system', 'end', '', msg.id);
} catch (err) {
console.error('streamChat error:', err);
send('system', 'error', '处理消息失败,请稍后重试', msg.id);
}
return;
}
if (msg.type === 'system') {
console.log('[System]', msg.content);
return;
}
console.log('[Unknown message type]', msg.type);
} catch (e) {
console.error('Message parse error:', e);
sendFrame(ws, 'system', 'error', '消息格式错误');
}
}
/** 释放当前连接的识别会话(主动中止,不会让供应商再回结果) */
_abortAsrSession(ws) {
const state = ws.asrSession;
ws.asrSession = null;
if (state) {
state.session?.abort();
state.decoder?.free();
}
return state;
}
/**
* 建供应商会话(首帧确认真实容器后才建,避免空轮次占用连接)
*
* 采集阶段失败统一走 onError:客户端收到 asr_error 后继续录完整句,
* 结束时改走整段 /health/ai/asr。
*/
_createAsrSession(ws, state, { codec, rate }) {
const session = createAudioStreamRecognizer({
codec,
rate,
channels: state.meta.channels,
requestId: `asr_stream_${state.id}`,
// 中间结果即时回显;客户端按 id 过滤,只认本轮
onPartial: (text) => {
if (ws.asrSession !== state) return;
// 每个会话只记一次:用来区分「实时链路通了」与「一直没出中间结果」
if (!state.partialSeen) {
state.partialSeen = true;
logger.info('asr stream first result', {
requestId: state.id,
format: state.format,
chars: text.length
});
}
sendFrame(ws, 'system', 'asr_partial', text, state.id);
},
onError: (err) => {
if (ws.asrSession !== state) return;
state.failed = true;
state.session = null;
logger.warn('asr stream failed', {
requestId: state.id,
partialSeen: state.partialSeen,
bytes: state.bytes,
error: err?.message ?? String(err)
});
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
}
});
state.session = session;
return session;
}
/** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */
_failAsrStream(ws, state, message) {
logger.warn('asr stream degraded', {
requestId: state.id,
format: state.format || 'unknown',
bytes: state.bytes,
partialSeen: state.partialSeen,
reason: message
});
if (ws.asrSession === state) ws.asrSession = null;
state.failed = true;
state.session?.abort();
state.session = null;
state.decoder?.free();
state.decoder = null;
sendFrame(ws, 'system', 'asr_error', message, state.id);
}
/**
* 开始一轮实时识别
* 上行:{ type: 'asr_start', id, token, format?, sampleRate?, channels? }
* 身份只认 token,与聊天同一套鉴权;每轮按用户限流一次
*/
async _handleAsrStart(ws, msg) {
// 同一连接不叠加两轮识别:先中止上一轮,避免两份供应商会话并行计费
this._abortAsrSession(ws);
const fail = (content) => sendFrame(ws, 'system', 'asr_error', content, msg.id);
try {
if (!msg.token) {
fail('请先登录');
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
fail('请先登录');
return;
}
if (!isAsrConfigured()) {
fail('语音识别服务暂不可用');
return;
}
const limit = asrLimiter(String(userInfo._id));
if (!limit.allowed) {
fail(`操作过于频繁,请 ${limit.retryAfter} 秒后重试`);
return;
}
let meta;
try {
meta = normalizeAudioMeta({
format: msg.format || 'mp3',
sampleRate: msg.sampleRate,
channels: msg.channels
});
} catch (err) {
fail(err.message || '语音参数不支持');
return;
}
// 供应商会话按首帧确认真实容器后再建:真机是真 mp3,浏览器(开发者工具)是 WebM,
// 后者要先流式解码成 PCM 才能推给供应商,两种编码不能共用一份 run-task 参数
const state = {
id: msg.id || `asr_${Date.now()}`,
userId: String(userInfo._id),
meta,
bytes: 0,
failed: false,
format: '',
pending: [],
decoder: null,
// 帧头诊断只打一次:确认客户端分片是否带完整容器头
headLogged: false,
// 推给供应商的 PCM 采样率(WebM 解码后归一,mp3 链路不用)
rate: 0,
// 是否已收到过中间结果(仅用于诊断日志)
partialSeen: false,
session: null,
// 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱
queue: Promise.resolve()
};
ws.asrSession = state;
// 只在开头记一条:用来区分「客户端根本没发分片」与「分片到了但解码/供应商失败」
logger.info('asr stream started', {
requestId: state.id,
declaredFormat: meta.format,
rate: meta.rate,
channels: meta.channels
});
} catch (err) {
console.error('[ASR] start failed:', err);
// 失败时不能留下半截状态:后续分片会误以为本轮还在进行
// 进入本方法时已中止上一轮,所以这里的 session 必然是本次新建的
if (ws.asrSession) this._abortAsrSession(ws);
fail('语音识别服务暂不可用');
}
}
/**
* 推入一帧音频(base64)
* 单帧解码失败只丢这一帧:录音仍在继续,不该因一帧异常打断整轮
*
* 分片串行排队处理:WebM 解码和供应商推送都是异步的,
* 并发处理会让 PCM 顺序错乱,也会把供应商会话的 run-task 竞态化。
*/
_handleAsrChunk(ws, msg) {
const state = ws.asrSession;
if (!state || state.failed) return;
if (msg.id && msg.id !== state.id) return;
let buffer;
try {
buffer = decodeBase64Media(msg.audio, {
maxBytes: ASR_LIMITS.chunkBytes,
label: '语音分片'
});
} catch (err) {
return;
}
if (state.bytes + buffer.length > ASR_LIMITS.totalBytes) {
this._failAsrStream(ws, state, '录音过长,请缩短语音后重试');
return;
}
state.bytes += buffer.length;
state.queue = state.queue
.then(() => this._consumeAsrChunk(ws, state, buffer))
.catch((err) => {
console.warn('[ASR] chunk failed:', err?.message ?? err, { requestId: state.id });
});
}
/**
* 处理一帧音频:首帧先判真实容器,再决定直接推 mp3 还是先流式解码 WebM
*
* 真机录的是 mp3 分片,容器直接交给供应商;
* 开发者工具录的是 WebM/Opus,供应商不认这个容器,必须边收边解码成 PCM。
*/
async _consumeAsrChunk(ws, state, buffer) {
if (ws.asrSession !== state || state.failed) return;
if (!state.format) {
state.pending.push(buffer);
let head = Buffer.concat(state.pending);
let sniffed = sniffAudioFormat(head);
// 首片前可能带少量前导字节,容器头不在偏移 0:定位 EBML 头后裁掉再按 WebM 处理
if (sniffed === 'unknown') {
const ebmlOffset = findEbmlHeader(head);
if (ebmlOffset > 0) {
logger.info('asr stream trimmed to ebml header', {
requestId: state.id,
offset: ebmlOffset,
headBytes: head.length
});
head = head.subarray(ebmlOffset);
sniffed = 'webm';
}
}
// 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判
if (sniffed === 'unknown') {
// 只记一次帧头:用来确认客户端分片到底是不是完整容器(开发者工具常缺 EBML 头)
if (!state.headLogged) {
state.headLogged = true;
logger.info('asr stream head unrecognized', {
requestId: state.id,
declaredFormat: state.meta.format,
headBytes: head.length,
firstBytes: head.subarray(0, Math.min(head.length, 48)).toString('hex')
});
}
if (head.length < ASR_LIMITS.sniffBytes) return;
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
if (sniffed !== 'mp3' && sniffed !== 'webm') {
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
state.format = sniffed;
state.pending = [];
// 排查实时链路时先看这一行:确认服务端认出的真实容器
logger.info('asr stream format detected', {
requestId: state.id,
format: sniffed,
headBytes: head.length
});
if (sniffed === 'webm') {
try {
state.decoder = await createStreamingWebmDecoder();
} catch (err) {
console.warn('[ASR] webm decoder init failed:', err?.message ?? err, { requestId: state.id });
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
return;
}
// 供应商会话等第一段 PCM 解出来再建:采样率以解码器实际输出为准
await this._pushAsrAudio(ws, state, head);
return;
}
this._createAsrSession(ws, state, { codec: state.meta.codec, rate: state.meta.rate });
await this._pushAsrAudio(ws, state, head);
return;
}
await this._pushAsrAudio(ws, state, buffer);
}
/** 把一帧音频推给供应商:WebM 先解码成 PCM,mp3 原样推 */
async _pushAsrAudio(ws, state, buffer) {
if (ws.asrSession !== state || state.failed) return;
try {
if (state.decoder) {
const pcm = await state.decoder.push(buffer);
// 首帧往往只有 WebM 头,解不出音频:等第一段 PCM 出来再建供应商会话
if (!pcm.length) return;
if (!state.session) {
// 采样率以首帧解码结果为准:Opus 是 48k,整数倍抽取到 16k
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
const out = downsamplePcm16(pcm, state.decoder.rate, state.rate);
if (out.length) state.session.push(out);
return;
}
if (state.session) state.session.push(buffer);
} catch (err) {
// 解码失败说明这一轮的分片无法拼成完整音频,交给客户端结束后的整段兜底
console.warn('[ASR] audio decode failed:', err?.message ?? err, { requestId: state.id });
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
}
}
/**
* 结束本轮并回定稿
* 上行:{ type: 'asr_stop', id }
* 下行:asr_final(可能为空串,由客户端提示重录)
*/
async _handleAsrStop(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
// 采集阶段的失败已经回过 asr_error,这里只需收尾
if (state.failed) {
this._abortAsrSession(ws);
return;
}
try {
// 先把排队的最后一帧处理完,否则会漏掉句尾(也会让 WebM 少解码一段)
await state.queue;
if (ws.asrSession !== state || state.failed) return;
// 一帧都没收到:说明客户端没发分片(录音分片回调未触发 / 还在用旧包),
// 这里记一条,配合 asr stream started 可直接定位
if (!state.format) {
logger.warn('asr stream finished without audio', {
requestId: state.id,
declaredFormat: state.meta.format,
bytes: state.bytes
});
}
// WebM 流式解码:把解码器里剩余的帧冲出来再收尾
if (state.decoder) {
const tail = await state.decoder.flush();
if (!state.session && tail.length) {
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
if (tail.length && state.session) {
const out = downsamplePcm16(tail, state.decoder.rate, state.rate);
if (out.length) state.session.push(out);
}
}
const text = state.session ? (await state.session.finish()) || '' : '';
if (ws.asrSession === state) ws.asrSession = null;
logger.info('asr stream finished', {
requestId: state.id,
format: state.format || 'unknown',
bytes: state.bytes,
partialSeen: state.partialSeen,
chars: text.length
});
sendFrame(ws, 'system', 'asr_final', text, state.id);
} catch (err) {
if (ws.asrSession === state) ws.asrSession = null;
state.session?.abort();
console.warn('[ASR] finish failed:', err.message, { requestId: state.id, bytes: state.bytes });
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
} finally {
state.decoder?.free();
state.decoder = null;
}
}
/**
* 取消本轮(页面隐藏、录音中断、客户端兜底失败)
* 只中止会话,不回任何帧,也不产生定稿
*/
_handleAsrCancel(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
this._abortAsrSession(ws);
}
broadcast(message) {
if (!this.wss) return;
this.wss.clients.forEach((client) => {
if (client.readyState === WebSocket.OPEN) {
client.send(message);
}
});
}
stop() {
if (this.wss) {
this.wss.close();
this.wss = null;
}
if (this.server) {
this.server.close();
this.server = null;
}
console.log('WebSocket server stopped');
}
getClientsCount() {
if (!this.wss) return 0;
return this.wss.clients.size;
}
}