import WebSocket, { WebSocketServer } from 'ws'; import http from 'http'; import { DBModel } from "./models/index.js"; import { chatTask } from "./agent/bots/bot-escort/task.js"; import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js" import { getUserByToken } from "./utils/user_service.js"; import { isAdmin } from "./utils/permissions.js"; import logger from "./utils/logger.js"; import { createUserLimiter } from "./middleware/ratelimit.js"; import { createAudioStreamRecognizer, isAsrConfigured, normalizeAudioMeta, decodeBase64Media } from "./agent/infra/media/index.js"; import { sniffAudioFormat, findEbmlHeader, createStreamingAudioDecoder, downsamplePcm16 } from "./agent/infra/media/audio.js"; let systemMessageSequence = 0; // 单连接上行帧上限:超限会被 ws 直接断开,避免超大 JSON 占用内存与 CPU const MAX_PAYLOAD_BYTES = 256 * 1024; // 单用户聊天频次限制:AI 调用有成本,且同一用户的会话本就串行处理 const chatLimiter = createUserLimiter(20); // 实时语音识别同样按次计费:每轮 asr_start 计一次,与聊天额度分开 const asrLimiter = createUserLimiter(20); // 实时识别分片上限:单帧按客户端 frameSize=8KB 留足余量, // 整轮按 60 秒录音的一倍余量收口,超过直接中止本轮,避免连接内存被刷满 const ASR_LIMITS = { chunkBytes: 128 * 1024, totalBytes: 8 * 1024 * 1024, // 首帧认不出容器时先缓存再判(有些平台的首帧只是 cluster,没有文件头) sniffBytes: 16 * 1024 }; /** * WebM 解码出的 PCM 采样率归一 * * 实时协议对裸 PCM 只稳定支持 16k:48k/24k 这类整数倍能整除到 16k, * 直接降到 16k;其他采样率保持原值,避免声明与数据不一致。 */ const resolveAsrRate = (sourceRate) => { const rate = Number(sourceRate) || 48000; return Number.isInteger(rate / 16000) && rate > 16000 ? 16000 : rate; }; const sendFrame = (ws, source, type, content = '', id = '') => { if (ws.readyState !== WebSocket.OPEN) return; ws.send(JSON.stringify({ source, type, content, id: id || `system_${Date.now()}_${++systemMessageSequence}` })); }; export default class WebSocketServerManager { constructor(port = 8080) { this.port = port; this.wss = null; this.server = null; } start() { this.server = http.createServer((req, res) => { res.writeHead(426, { 'Content-Type': 'text/plain' }); res.end('Upgrade Required: WebSocket expected'); }); this.wss = new WebSocketServer({ server: this.server, maxPayload: MAX_PAYLOAD_BYTES, perMessageDeflate: { zlibDeflateOptions: { chunkSize: 1024, memLevel: 7, level: 3 }, zlibInflateOptions: { chunkSize: 10 * 1024 }, clientNoContextTakeover: true, serverNoContextTakeover: true, serverMaxWindowBits: 10, concurrencyLimit: 10, threshold: 1024 } }); this.wss.on('connection', (ws, req) => { const path = req.url; console.log(`WebSocket connected: ${req.socket.remoteAddress}, path: ${path}`); if (path === '/chat') { this._handleChatConnection(ws); } else { sendFrame(ws, 'system', 'error', 'Unknown path'); ws.close(1000, 'Unknown path'); } }); this.server.listen(this.port, () => { console.log(`WebSocket server started on ws://localhost:${this.port}`); }); } _handleChatConnection(ws) { sendFrame(ws, 'system', 'connected', 'Connected to chat server'); // 同一连接的消息串行处理:agent 的 messages 数组是共享状态, // 并发跑同一个 agent 会让上下文交错、回复错乱 ws.messageQueue = Promise.resolve(); ws.on('message', (data) => { ws.messageQueue = ws.messageQueue .then(() => this._handleMessage(ws, data)) .catch((err) => console.error('WebSocket message handling error:', err)); }); ws.on('close', () => { // 断开后释放在途的识别会话与 WebM 解码器:不释放会让供应商连接与缓冲悬空 this._abortAsrSession(ws); // 断开后释放队列引用,避免连接对象被闭包长期持有 ws.messageQueue = null; console.log('Chat client disconnected'); }); ws.on('error', (err) => { console.error('WebSocket error:', err); }); } async _handleMessage(ws, data) { try { const raw = data.toString(); const msg = JSON.parse(raw); // 只记录消息类型:正文含健康描述、token/userId 属敏感信息,不进日志。 // 实时识别的分片按秒级频率到达,逐帧打日志会把日志刷满,这里跳过 if (msg.type !== 'asr_chunk') { console.log('Received message type:', msg.type); } if (msg.type === 'ping') { sendFrame(ws, 'system', 'pong', '', msg.id); return; } // 实时语音识别:与聊天共用一条连接,一轮识别 = start → 若干 chunk → stop/cancel if (msg.type === 'asr_start') { await this._handleAsrStart(ws, msg); return; } if (msg.type === 'asr_chunk') { this._handleAsrChunk(ws, msg); return; } if (msg.type === 'asr_stop') { await this._handleAsrStop(ws, msg); return; } if (msg.type === 'asr_cancel') { this._handleAsrCancel(ws, msg); return; } if (msg.type === 'chat' || msg.type === 'clear') { // 客户端断开后不再处理消息 if (ws.readyState !== WebSocket.OPEN) return; const send = (source, type, content, id) => { sendFrame(ws, source, type, content, id); }; try { // 身份只认 token:不再接受凭 userId 明文接入,防止冒充他人聊天 if (!msg.token) { send('system', 'error', '请先登录', msg.id); return; } const userInfo = await getUserByToken(msg.token); if (!userInfo) { send('system', 'error', '请先登录', msg.id); return; } // 频次限制按用户维度:防止单账号刷 AI 调用(同一账号多连接共享额度) if (msg.type === 'chat') { const limit = chatLimiter(String(userInfo._id)); if (!limit.allowed) { send('system', 'error', `操作过于频繁,请 ${limit.retryAfter} 秒后重试`, msg.id); return; } } if (msg.agent === 'escort-admin') { // 管理端 agent 含写库/环境变量等高危工具,必须校验管理员角色,防止普通用户越权 if (!isAdmin(userInfo)) { send('system', 'error', '无管理员权限', msg.id); return; } await adminAgent.streamChat(userInfo, [msg], (source, type, content, id) => { send(source, type, content, id); }); } else { await chatTask.streamChat(userInfo, msg, (source, type, content, id) => { send(source, type, content, id); }); } // 流式 chunk 可能携带不同的模型消息 ID,结束帧统一关联本次客户端请求。 send('system', 'end', '', msg.id); } catch (err) { console.error('streamChat error:', err); send('system', 'error', '处理消息失败,请稍后重试', msg.id); } return; } if (msg.type === 'system') { console.log('[System]', msg.content); return; } console.log('[Unknown message type]', msg.type); } catch (e) { console.error('Message parse error:', e); sendFrame(ws, 'system', 'error', '消息格式错误'); } } /** 释放当前连接的识别会话(主动中止,不会让供应商再回结果) */ _abortAsrSession(ws) { const state = ws.asrSession; ws.asrSession = null; if (state) { state.session?.abort(); state.decoder?.free(); } return state; } /** * 建供应商会话(首帧确认真实容器后才建,避免空轮次占用连接) * * 采集阶段失败统一走 onError:客户端收到 asr_error 后继续录完整句, * 结束时改走整段 /health/ai/asr。 */ _createAsrSession(ws, state, { codec, rate }) { const session = createAudioStreamRecognizer({ codec, rate, channels: state.meta.channels, requestId: `asr_stream_${state.id}`, // 中间结果即时回显;客户端按 id 过滤,只认本轮 onPartial: (text) => { if (ws.asrSession !== state) return; // 每个会话只记一次:用来区分「实时链路通了」与「一直没出中间结果」 if (!state.partialSeen) { state.partialSeen = true; logger.info('asr stream first result', { requestId: state.id, format: state.format, chars: text.length }); } sendFrame(ws, 'system', 'asr_partial', text, state.id); }, onError: (err) => { if (ws.asrSession !== state) return; state.failed = true; state.session = null; logger.warn('asr stream failed', { requestId: state.id, partialSeen: state.partialSeen, bytes: state.bytes, error: err?.message ?? String(err) }); sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id); } }); state.session = session; return session; } /** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */ _failAsrStream(ws, state, message) { logger.warn('asr stream degraded', { requestId: state.id, format: state.format || 'unknown', bytes: state.bytes, partialSeen: state.partialSeen, reason: message }); if (ws.asrSession === state) ws.asrSession = null; state.failed = true; state.session?.abort(); state.session = null; state.decoder?.free(); state.decoder = null; sendFrame(ws, 'system', 'asr_error', message, state.id); } /** * 开始一轮实时识别 * 上行:{ type: 'asr_start', id, token, format?, sampleRate?, channels? } * 身份只认 token,与聊天同一套鉴权;每轮按用户限流一次 */ async _handleAsrStart(ws, msg) { // 同一连接不叠加两轮识别:先中止上一轮,避免两份供应商会话并行计费 this._abortAsrSession(ws); const fail = (content) => sendFrame(ws, 'system', 'asr_error', content, msg.id); try { if (!msg.token) { fail('请先登录'); return; } const userInfo = await getUserByToken(msg.token); if (!userInfo) { fail('请先登录'); return; } if (!isAsrConfigured()) { fail('语音识别服务暂不可用'); return; } const limit = asrLimiter(String(userInfo._id)); if (!limit.allowed) { fail(`操作过于频繁,请 ${limit.retryAfter} 秒后重试`); return; } let meta; try { meta = normalizeAudioMeta({ format: msg.format || 'mp3', sampleRate: msg.sampleRate, channels: msg.channels }); } catch (err) { fail(err.message || '语音参数不支持'); return; } // 供应商会话按首帧确认真实容器后再建:真机是 16k mp3,浏览器(开发者工具)是 WebM。 // 两种容器供应商都解不了(实测报 DecodePost 解码失败),都要先在服务端流式解成 PCM, // 因此不能把原始分片直接透传,也不能按声明格式共用一份 run-task 参数 const state = { id: msg.id || `asr_${Date.now()}`, userId: String(userInfo._id), meta, bytes: 0, failed: false, format: '', pending: [], decoder: null, // 帧头诊断只打一次:确认客户端分片是否带完整容器头 headLogged: false, // 推给供应商的 PCM 采样率(解码后按实际输出归一) rate: 0, // 是否已收到过中间结果(仅用于诊断日志) partialSeen: false, session: null, // 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱 queue: Promise.resolve() }; ws.asrSession = state; // 只在开头记一条:用来区分「客户端根本没发分片」与「分片到了但解码/供应商失败」 logger.info('asr stream started', { requestId: state.id, declaredFormat: meta.format, rate: meta.rate, channels: meta.channels }); } catch (err) { console.error('[ASR] start failed:', err); // 失败时不能留下半截状态:后续分片会误以为本轮还在进行 // 进入本方法时已中止上一轮,所以这里的 session 必然是本次新建的 if (ws.asrSession) this._abortAsrSession(ws); fail('语音识别服务暂不可用'); } } /** * 推入一帧音频(base64) * 单帧解码失败只丢这一帧:录音仍在继续,不该因一帧异常打断整轮 * * 分片串行排队处理:WebM 解码和供应商推送都是异步的, * 并发处理会让 PCM 顺序错乱,也会把供应商会话的 run-task 竞态化。 */ _handleAsrChunk(ws, msg) { const state = ws.asrSession; if (!state || state.failed) return; if (msg.id && msg.id !== state.id) return; let buffer; try { buffer = decodeBase64Media(msg.audio, { maxBytes: ASR_LIMITS.chunkBytes, label: '语音分片' }); } catch (err) { return; } if (state.bytes + buffer.length > ASR_LIMITS.totalBytes) { this._failAsrStream(ws, state, '录音过长,请缩短语音后重试'); return; } state.bytes += buffer.length; state.queue = state.queue .then(() => this._consumeAsrChunk(ws, state, buffer)) .catch((err) => { console.warn('[ASR] chunk failed:', err?.message ?? err, { requestId: state.id }); }); } /** * 处理一帧音频:首帧先判真实容器,再决定解码方式 * * 真机录的是 16k mp3,开发者工具录的是 WebM/Opus, * 两种容器供应商都解不了,统一在服务端边收边解码成 PCM 再推。 */ async _consumeAsrChunk(ws, state, buffer) { if (ws.asrSession !== state || state.failed) return; if (!state.format) { state.pending.push(buffer); let head = Buffer.concat(state.pending); let sniffed = sniffAudioFormat(head); // 首片前可能带少量前导字节,容器头不在偏移 0:定位 EBML 头后裁掉再按 WebM 处理 if (sniffed === 'unknown') { const ebmlOffset = findEbmlHeader(head); if (ebmlOffset > 0) { logger.info('asr stream trimmed to ebml header', { requestId: state.id, offset: ebmlOffset, headBytes: head.length }); head = head.subarray(ebmlOffset); sniffed = 'webm'; } } // 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判 if (sniffed === 'unknown') { // 只记一次帧头:用来确认客户端分片到底是不是完整容器(开发者工具常缺 EBML 头) if (!state.headLogged) { state.headLogged = true; logger.info('asr stream head unrecognized', { requestId: state.id, declaredFormat: state.meta.format, headBytes: head.length, firstBytes: head.subarray(0, Math.min(head.length, 48)).toString('hex') }); } if (head.length < ASR_LIMITS.sniffBytes) return; this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别'); return; } if (sniffed !== 'mp3' && sniffed !== 'webm') { this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别'); return; } state.format = sniffed; state.pending = []; // 排查实时链路时先看这一行:确认服务端认出的真实容器与帧头 logger.info('asr stream format detected', { requestId: state.id, format: sniffed, headBytes: head.length, firstBytes: head.subarray(0, Math.min(head.length, 16)).toString('hex') }); // webm(开发者工具)与 mp3(真机)都先流式解码成 PCM: // 供应商对这两种容器的解码都不可靠,透传会整轮失败 try { state.decoder = await createStreamingAudioDecoder(sniffed); } catch (err) { console.warn('[ASR] audio decoder init failed:', err?.message ?? err, { requestId: state.id, format: sniffed }); this._failAsrStream(ws, state, '语音识别失败,请稍后重试'); return; } // 供应商会话等第一段 PCM 解出来再建:采样率以解码器实际输出为准 await this._pushAsrAudio(ws, state, head); return; } await this._pushAsrAudio(ws, state, buffer); } /** 把一帧音频推给供应商:容器分片先解码成 PCM,再按 16k 归一推流 */ async _pushAsrAudio(ws, state, buffer) { if (ws.asrSession !== state || state.failed || !state.decoder) return; try { const pcm = await state.decoder.push(buffer); // 首帧往往只有容器头(WebM 的 EBML / mp3 的元信息),解不出音频: // 等第一段 PCM 出来再建供应商会话 if (!pcm.length) return; if (!state.session) { // 采样率以首帧解码结果为准:Opus 是 48k,真机 mp3 是 16k state.rate = resolveAsrRate(state.decoder.rate); this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate }); } const out = downsamplePcm16(pcm, state.decoder.rate, state.rate); if (out.length) state.session.push(out); } catch (err) { // 解码失败说明这一轮的分片无法拼成完整音频,交给客户端结束后的整段兜底 console.warn('[ASR] audio decode failed:', err?.message ?? err, { requestId: state.id }); this._failAsrStream(ws, state, '语音识别失败,请稍后重试'); } } /** * 结束本轮并回定稿 * 上行:{ type: 'asr_stop', id } * 下行:asr_final(可能为空串,由客户端提示重录) */ async _handleAsrStop(ws, msg) { const state = ws.asrSession; if (!state) return; if (msg.id && msg.id !== state.id) return; // 采集阶段的失败已经回过 asr_error,这里只需收尾 if (state.failed) { this._abortAsrSession(ws); return; } try { // 先把排队的最后一帧处理完,否则会漏掉句尾(也会让 WebM 少解码一段) await state.queue; if (ws.asrSession !== state || state.failed) return; // 一帧都没收到:说明客户端没发分片(录音分片回调未触发 / 还在用旧包), // 这里记一条,配合 asr stream started 可直接定位 if (!state.format) { logger.warn('asr stream finished without audio', { requestId: state.id, declaredFormat: state.meta.format, bytes: state.bytes }); } // 流式解码收尾:把解码器里剩余的音频冲出来再收(mp3 无尾部缓冲,返回空) if (state.decoder) { const tail = await state.decoder.flush(); if (!state.session && tail.length) { state.rate = resolveAsrRate(state.decoder.rate); this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate }); } if (tail.length && state.session) { const out = downsamplePcm16(tail, state.decoder.rate, state.rate); if (out.length) state.session.push(out); } } const text = state.session ? (await state.session.finish()) || '' : ''; if (ws.asrSession === state) ws.asrSession = null; logger.info('asr stream finished', { requestId: state.id, format: state.format || 'unknown', bytes: state.bytes, partialSeen: state.partialSeen, chars: text.length }); sendFrame(ws, 'system', 'asr_final', text, state.id); } catch (err) { if (ws.asrSession === state) ws.asrSession = null; state.session?.abort(); console.warn('[ASR] finish failed:', err.message, { requestId: state.id, bytes: state.bytes }); sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id); } finally { state.decoder?.free(); state.decoder = null; } } /** * 取消本轮(页面隐藏、录音中断、客户端兜底失败) * 只中止会话,不回任何帧,也不产生定稿 */ _handleAsrCancel(ws, msg) { const state = ws.asrSession; if (!state) return; if (msg.id && msg.id !== state.id) return; this._abortAsrSession(ws); } broadcast(message) { if (!this.wss) return; this.wss.clients.forEach((client) => { if (client.readyState === WebSocket.OPEN) { client.send(message); } }); } stop() { if (this.wss) { this.wss.close(); this.wss = null; } if (this.server) { this.server.close(); this.server = null; } console.log('WebSocket server stopped'); } getClientsCount() { if (!this.wss) return 0; return this.wss.clients.size; } }