上行 asr_start/asr_chunk/asr_stop/asr_cancel,下行 asr_partial/asr_final/asr_error。首帧判容器:mp3 直接透传,WebM 边收边解码成 16bit 单声道 PCM(48k 抽取到 16k)再推供应商;认不出容器先缓存 16KB 再判,失败回 asr_error 让客户端整段兜底。分片串行处理,断开连接时释放会话与解码器。
550 lines
18 KiB
JavaScript
550 lines
18 KiB
JavaScript
import WebSocket, { WebSocketServer } from 'ws';
|
|
import http from 'http';
|
|
import { DBModel } from "./models/index.js";
|
|
import { chatTask } from "./agent/bots/bot-escort/task.js";
|
|
import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js"
|
|
import { getUserByToken } from "./utils/user_service.js";
|
|
import { isAdmin } from "./utils/permissions.js";
|
|
import { createUserLimiter } from "./middleware/ratelimit.js";
|
|
import {
|
|
createAudioStreamRecognizer,
|
|
isAsrConfigured,
|
|
normalizeAudioMeta,
|
|
decodeBase64Media
|
|
} from "./agent/infra/media/index.js";
|
|
import {
|
|
sniffAudioFormat,
|
|
createStreamingWebmDecoder,
|
|
downsamplePcm16
|
|
} from "./agent/infra/media/audio.js";
|
|
|
|
let systemMessageSequence = 0;
|
|
|
|
// 单连接上行帧上限:超限会被 ws 直接断开,避免超大 JSON 占用内存与 CPU
|
|
const MAX_PAYLOAD_BYTES = 256 * 1024;
|
|
// 单用户聊天频次限制:AI 调用有成本,且同一用户的会话本就串行处理
|
|
const chatLimiter = createUserLimiter(20);
|
|
// 实时语音识别同样按次计费:每轮 asr_start 计一次,与聊天额度分开
|
|
const asrLimiter = createUserLimiter(20);
|
|
|
|
// 实时识别分片上限:单帧按客户端 frameSize=8KB 留足余量,
|
|
// 整轮按 60 秒录音的一倍余量收口,超过直接中止本轮,避免连接内存被刷满
|
|
const ASR_LIMITS = {
|
|
chunkBytes: 128 * 1024,
|
|
totalBytes: 8 * 1024 * 1024,
|
|
// 首帧认不出容器时先缓存再判(有些平台的首帧只是 cluster,没有文件头)
|
|
sniffBytes: 16 * 1024
|
|
};
|
|
|
|
/**
|
|
* WebM 解码出的 PCM 采样率归一
|
|
*
|
|
* 实时协议对裸 PCM 只稳定支持 16k:48k/24k 这类整数倍能整除到 16k,
|
|
* 直接降到 16k;其他采样率保持原值,避免声明与数据不一致。
|
|
*/
|
|
const resolveAsrRate = (sourceRate) => {
|
|
const rate = Number(sourceRate) || 48000;
|
|
return Number.isInteger(rate / 16000) && rate > 16000 ? 16000 : rate;
|
|
};
|
|
|
|
const sendFrame = (ws, source, type, content = '', id = '') => {
|
|
if (ws.readyState !== WebSocket.OPEN) return;
|
|
ws.send(JSON.stringify({
|
|
source,
|
|
type,
|
|
content,
|
|
id: id || `system_${Date.now()}_${++systemMessageSequence}`
|
|
}));
|
|
};
|
|
|
|
export default class WebSocketServerManager {
|
|
constructor(port = 8080) {
|
|
this.port = port;
|
|
this.wss = null;
|
|
this.server = null;
|
|
}
|
|
|
|
start() {
|
|
this.server = http.createServer((req, res) => {
|
|
res.writeHead(426, { 'Content-Type': 'text/plain' });
|
|
res.end('Upgrade Required: WebSocket expected');
|
|
});
|
|
|
|
this.wss = new WebSocketServer({
|
|
server: this.server,
|
|
maxPayload: MAX_PAYLOAD_BYTES,
|
|
perMessageDeflate: {
|
|
zlibDeflateOptions: {
|
|
chunkSize: 1024,
|
|
memLevel: 7,
|
|
level: 3
|
|
},
|
|
zlibInflateOptions: {
|
|
chunkSize: 10 * 1024
|
|
},
|
|
clientNoContextTakeover: true,
|
|
serverNoContextTakeover: true,
|
|
serverMaxWindowBits: 10,
|
|
concurrencyLimit: 10,
|
|
threshold: 1024
|
|
}
|
|
});
|
|
|
|
this.wss.on('connection', (ws, req) => {
|
|
const path = req.url;
|
|
console.log(`WebSocket connected: ${req.socket.remoteAddress}, path: ${path}`);
|
|
|
|
if (path === '/chat') {
|
|
this._handleChatConnection(ws);
|
|
} else {
|
|
sendFrame(ws, 'system', 'error', 'Unknown path');
|
|
ws.close(1000, 'Unknown path');
|
|
}
|
|
});
|
|
|
|
this.server.listen(this.port, () => {
|
|
console.log(`WebSocket server started on ws://localhost:${this.port}`);
|
|
});
|
|
}
|
|
|
|
_handleChatConnection(ws) {
|
|
sendFrame(ws, 'system', 'connected', 'Connected to chat server');
|
|
|
|
// 同一连接的消息串行处理:agent 的 messages 数组是共享状态,
|
|
// 并发跑同一个 agent 会让上下文交错、回复错乱
|
|
ws.messageQueue = Promise.resolve();
|
|
ws.on('message', (data) => {
|
|
ws.messageQueue = ws.messageQueue
|
|
.then(() => this._handleMessage(ws, data))
|
|
.catch((err) => console.error('WebSocket message handling error:', err));
|
|
});
|
|
|
|
ws.on('close', () => {
|
|
// 断开后释放在途的识别会话与 WebM 解码器:不释放会让供应商连接与缓冲悬空
|
|
this._abortAsrSession(ws);
|
|
// 断开后释放队列引用,避免连接对象被闭包长期持有
|
|
ws.messageQueue = null;
|
|
console.log('Chat client disconnected');
|
|
});
|
|
|
|
ws.on('error', (err) => {
|
|
console.error('WebSocket error:', err);
|
|
});
|
|
}
|
|
|
|
async _handleMessage(ws, data) {
|
|
try {
|
|
const raw = data.toString();
|
|
|
|
const msg = JSON.parse(raw);
|
|
// 只记录消息类型:正文含健康描述、token/userId 属敏感信息,不进日志。
|
|
// 实时识别的分片按秒级频率到达,逐帧打日志会把日志刷满,这里跳过
|
|
if (msg.type !== 'asr_chunk') {
|
|
console.log('Received message type:', msg.type);
|
|
}
|
|
|
|
if (msg.type === 'ping') {
|
|
sendFrame(ws, 'system', 'pong', '', msg.id);
|
|
return;
|
|
}
|
|
|
|
// 实时语音识别:与聊天共用一条连接,一轮识别 = start → 若干 chunk → stop/cancel
|
|
if (msg.type === 'asr_start') {
|
|
await this._handleAsrStart(ws, msg);
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'asr_chunk') {
|
|
this._handleAsrChunk(ws, msg);
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'asr_stop') {
|
|
await this._handleAsrStop(ws, msg);
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'asr_cancel') {
|
|
this._handleAsrCancel(ws, msg);
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'chat' || msg.type === 'clear') {
|
|
// 客户端断开后不再处理消息
|
|
if (ws.readyState !== WebSocket.OPEN) return;
|
|
|
|
const send = (source, type, content, id) => {
|
|
sendFrame(ws, source, type, content, id);
|
|
};
|
|
|
|
try {
|
|
// 身份只认 token:不再接受凭 userId 明文接入,防止冒充他人聊天
|
|
if (!msg.token) {
|
|
send('system', 'error', '请先登录', msg.id);
|
|
return;
|
|
}
|
|
const userInfo = await getUserByToken(msg.token);
|
|
if (!userInfo) {
|
|
send('system', 'error', '请先登录', msg.id);
|
|
return;
|
|
}
|
|
|
|
// 频次限制按用户维度:防止单账号刷 AI 调用(同一账号多连接共享额度)
|
|
if (msg.type === 'chat') {
|
|
const limit = chatLimiter(String(userInfo._id));
|
|
if (!limit.allowed) {
|
|
send('system', 'error', `操作过于频繁,请 ${limit.retryAfter} 秒后重试`, msg.id);
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (msg.agent === 'escort-admin') {
|
|
// 管理端 agent 含写库/环境变量等高危工具,必须校验管理员角色,防止普通用户越权
|
|
if (!isAdmin(userInfo)) {
|
|
send('system', 'error', '无管理员权限', msg.id);
|
|
return;
|
|
}
|
|
await adminAgent.streamChat(userInfo, [msg], (source, type, content, id) => {
|
|
send(source, type, content, id);
|
|
});
|
|
} else {
|
|
await chatTask.streamChat(userInfo, msg, (source, type, content, id) => {
|
|
send(source, type, content, id);
|
|
});
|
|
}
|
|
// 流式 chunk 可能携带不同的模型消息 ID,结束帧统一关联本次客户端请求。
|
|
send('system', 'end', '', msg.id);
|
|
} catch (err) {
|
|
console.error('streamChat error:', err);
|
|
send('system', 'error', '处理消息失败,请稍后重试', msg.id);
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (msg.type === 'system') {
|
|
console.log('[System]', msg.content);
|
|
return;
|
|
}
|
|
|
|
console.log('[Unknown message type]', msg.type);
|
|
} catch (e) {
|
|
console.error('Message parse error:', e);
|
|
sendFrame(ws, 'system', 'error', '消息格式错误');
|
|
}
|
|
}
|
|
|
|
/** 释放当前连接的识别会话(主动中止,不会让供应商再回结果) */
|
|
_abortAsrSession(ws) {
|
|
const state = ws.asrSession;
|
|
ws.asrSession = null;
|
|
if (state) {
|
|
state.session?.abort();
|
|
state.decoder?.free();
|
|
}
|
|
return state;
|
|
}
|
|
|
|
/**
|
|
* 建供应商会话(首帧确认真实容器后才建,避免空轮次占用连接)
|
|
*
|
|
* 采集阶段失败统一走 onError:客户端收到 asr_error 后继续录完整句,
|
|
* 结束时改走整段 /health/ai/asr。
|
|
*/
|
|
_createAsrSession(ws, state, { codec, rate }) {
|
|
const session = createAudioStreamRecognizer({
|
|
codec,
|
|
rate,
|
|
channels: state.meta.channels,
|
|
requestId: `asr_stream_${state.id}`,
|
|
// 中间结果即时回显;客户端按 id 过滤,只认本轮
|
|
onPartial: (text) => {
|
|
if (ws.asrSession !== state) return;
|
|
sendFrame(ws, 'system', 'asr_partial', text, state.id);
|
|
},
|
|
onError: (err) => {
|
|
if (ws.asrSession !== state) return;
|
|
state.failed = true;
|
|
state.session = null;
|
|
console.warn('[ASR] stream failed:', err.message, { requestId: state.id });
|
|
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
|
|
}
|
|
});
|
|
state.session = session;
|
|
return session;
|
|
}
|
|
|
|
/** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */
|
|
_failAsrStream(ws, state, message) {
|
|
if (ws.asrSession === state) ws.asrSession = null;
|
|
state.failed = true;
|
|
state.session?.abort();
|
|
state.session = null;
|
|
state.decoder?.free();
|
|
state.decoder = null;
|
|
sendFrame(ws, 'system', 'asr_error', message, state.id);
|
|
}
|
|
|
|
/**
|
|
* 开始一轮实时识别
|
|
* 上行:{ type: 'asr_start', id, token, format?, sampleRate?, channels? }
|
|
* 身份只认 token,与聊天同一套鉴权;每轮按用户限流一次
|
|
*/
|
|
async _handleAsrStart(ws, msg) {
|
|
// 同一连接不叠加两轮识别:先中止上一轮,避免两份供应商会话并行计费
|
|
this._abortAsrSession(ws);
|
|
|
|
const fail = (content) => sendFrame(ws, 'system', 'asr_error', content, msg.id);
|
|
|
|
try {
|
|
if (!msg.token) {
|
|
fail('请先登录');
|
|
return;
|
|
}
|
|
const userInfo = await getUserByToken(msg.token);
|
|
if (!userInfo) {
|
|
fail('请先登录');
|
|
return;
|
|
}
|
|
|
|
if (!isAsrConfigured()) {
|
|
fail('语音识别服务暂不可用');
|
|
return;
|
|
}
|
|
|
|
const limit = asrLimiter(String(userInfo._id));
|
|
if (!limit.allowed) {
|
|
fail(`操作过于频繁,请 ${limit.retryAfter} 秒后重试`);
|
|
return;
|
|
}
|
|
|
|
let meta;
|
|
try {
|
|
meta = normalizeAudioMeta({
|
|
format: msg.format || 'mp3',
|
|
sampleRate: msg.sampleRate,
|
|
channels: msg.channels
|
|
});
|
|
} catch (err) {
|
|
fail(err.message || '语音参数不支持');
|
|
return;
|
|
}
|
|
|
|
// 供应商会话按首帧确认真实容器后再建:真机是真 mp3,浏览器(开发者工具)是 WebM,
|
|
// 后者要先流式解码成 PCM 才能推给供应商,两种编码不能共用一份 run-task 参数
|
|
const state = {
|
|
id: msg.id || `asr_${Date.now()}`,
|
|
userId: String(userInfo._id),
|
|
meta,
|
|
bytes: 0,
|
|
failed: false,
|
|
format: '',
|
|
pending: [],
|
|
decoder: null,
|
|
// 推给供应商的 PCM 采样率(WebM 解码后归一,mp3 链路不用)
|
|
rate: 0,
|
|
session: null,
|
|
// 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱
|
|
queue: Promise.resolve()
|
|
};
|
|
|
|
ws.asrSession = state;
|
|
} catch (err) {
|
|
console.error('[ASR] start failed:', err);
|
|
fail('语音识别服务暂不可用');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 推入一帧音频(base64)
|
|
* 单帧解码失败只丢这一帧:录音仍在继续,不该因一帧异常打断整轮
|
|
*
|
|
* 分片串行排队处理:WebM 解码和供应商推送都是异步的,
|
|
* 并发处理会让 PCM 顺序错乱,也会把供应商会话的 run-task 竞态化。
|
|
*/
|
|
_handleAsrChunk(ws, msg) {
|
|
const state = ws.asrSession;
|
|
if (!state || state.failed) return;
|
|
if (msg.id && msg.id !== state.id) return;
|
|
|
|
let buffer;
|
|
try {
|
|
buffer = decodeBase64Media(msg.audio, {
|
|
maxBytes: ASR_LIMITS.chunkBytes,
|
|
label: '语音分片'
|
|
});
|
|
} catch (err) {
|
|
return;
|
|
}
|
|
|
|
if (state.bytes + buffer.length > ASR_LIMITS.totalBytes) {
|
|
this._failAsrStream(ws, state, '录音过长,请缩短语音后重试');
|
|
return;
|
|
}
|
|
state.bytes += buffer.length;
|
|
|
|
state.queue = state.queue
|
|
.then(() => this._consumeAsrChunk(ws, state, buffer))
|
|
.catch((err) => {
|
|
console.warn('[ASR] chunk failed:', err?.message ?? err, { requestId: state.id });
|
|
});
|
|
}
|
|
|
|
/**
|
|
* 处理一帧音频:首帧先判真实容器,再决定直接推 mp3 还是先流式解码 WebM
|
|
*
|
|
* 真机录的是 mp3 分片,容器直接交给供应商;
|
|
* 开发者工具录的是 WebM/Opus,供应商不认这个容器,必须边收边解码成 PCM。
|
|
*/
|
|
async _consumeAsrChunk(ws, state, buffer) {
|
|
if (ws.asrSession !== state || state.failed) return;
|
|
|
|
if (!state.format) {
|
|
state.pending.push(buffer);
|
|
const head = Buffer.concat(state.pending);
|
|
const sniffed = sniffAudioFormat(head);
|
|
|
|
// 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判
|
|
if (sniffed === 'unknown') {
|
|
if (head.length < ASR_LIMITS.sniffBytes) return;
|
|
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
|
|
return;
|
|
}
|
|
if (sniffed !== 'mp3' && sniffed !== 'webm') {
|
|
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
|
|
return;
|
|
}
|
|
|
|
state.format = sniffed;
|
|
state.pending = [];
|
|
|
|
if (sniffed === 'webm') {
|
|
try {
|
|
state.decoder = await createStreamingWebmDecoder();
|
|
} catch (err) {
|
|
console.warn('[ASR] webm decoder init failed:', err?.message ?? err, { requestId: state.id });
|
|
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
|
|
return;
|
|
}
|
|
// 供应商会话等第一段 PCM 解出来再建:采样率以解码器实际输出为准
|
|
await this._pushAsrAudio(ws, state, head);
|
|
return;
|
|
}
|
|
|
|
this._createAsrSession(ws, state, { codec: state.meta.codec, rate: state.meta.rate });
|
|
await this._pushAsrAudio(ws, state, head);
|
|
return;
|
|
}
|
|
|
|
await this._pushAsrAudio(ws, state, buffer);
|
|
}
|
|
|
|
/** 把一帧音频推给供应商:WebM 先解码成 PCM,mp3 原样推 */
|
|
async _pushAsrAudio(ws, state, buffer) {
|
|
if (ws.asrSession !== state || state.failed) return;
|
|
|
|
try {
|
|
if (state.decoder) {
|
|
const pcm = await state.decoder.push(buffer);
|
|
if (!state.session) {
|
|
// 采样率以首帧解码结果为准:Opus 是 48k,整数倍抽取到 16k
|
|
state.rate = resolveAsrRate(state.decoder.rate);
|
|
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
|
|
}
|
|
const out = pcm.length ? downsamplePcm16(pcm, state.decoder.rate, state.rate) : pcm;
|
|
if (out.length) state.session.push(out);
|
|
return;
|
|
}
|
|
if (state.session) state.session.push(buffer);
|
|
} catch (err) {
|
|
// 解码失败说明这一轮的分片无法拼成完整音频,交给客户端结束后的整段兜底
|
|
console.warn('[ASR] audio decode failed:', err?.message ?? err, { requestId: state.id });
|
|
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 结束本轮并回定稿
|
|
* 上行:{ type: 'asr_stop', id }
|
|
* 下行:asr_final(可能为空串,由客户端提示重录)
|
|
*/
|
|
async _handleAsrStop(ws, msg) {
|
|
const state = ws.asrSession;
|
|
if (!state) return;
|
|
if (msg.id && msg.id !== state.id) return;
|
|
|
|
// 采集阶段的失败已经回过 asr_error,这里只需收尾
|
|
if (state.failed) {
|
|
this._abortAsrSession(ws);
|
|
return;
|
|
}
|
|
|
|
try {
|
|
// 先把排队的最后一帧处理完,否则会漏掉句尾(也会让 WebM 少解码一段)
|
|
await state.queue;
|
|
if (ws.asrSession !== state || state.failed) return;
|
|
|
|
// WebM 流式解码:把解码器里剩余的帧冲出来再收尾
|
|
if (state.decoder) {
|
|
const tail = await state.decoder.flush();
|
|
if (!state.session && tail.length) {
|
|
state.rate = resolveAsrRate(state.decoder.rate);
|
|
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
|
|
}
|
|
if (tail.length && state.session) {
|
|
const out = downsamplePcm16(tail, state.decoder.rate, state.rate);
|
|
if (out.length) state.session.push(out);
|
|
}
|
|
}
|
|
|
|
const text = state.session ? (await state.session.finish()) || '' : '';
|
|
if (ws.asrSession === state) ws.asrSession = null;
|
|
sendFrame(ws, 'system', 'asr_final', text, state.id);
|
|
} catch (err) {
|
|
if (ws.asrSession === state) ws.asrSession = null;
|
|
state.session?.abort();
|
|
console.warn('[ASR] finish failed:', err.message, { requestId: state.id, bytes: state.bytes });
|
|
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
|
|
} finally {
|
|
state.decoder?.free();
|
|
state.decoder = null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* 取消本轮(页面隐藏、录音中断、客户端兜底失败)
|
|
* 只中止会话,不回任何帧,也不产生定稿
|
|
*/
|
|
_handleAsrCancel(ws, msg) {
|
|
const state = ws.asrSession;
|
|
if (!state) return;
|
|
if (msg.id && msg.id !== state.id) return;
|
|
this._abortAsrSession(ws);
|
|
}
|
|
|
|
broadcast(message) {
|
|
if (!this.wss) return;
|
|
this.wss.clients.forEach((client) => {
|
|
if (client.readyState === WebSocket.OPEN) {
|
|
client.send(message);
|
|
}
|
|
});
|
|
}
|
|
|
|
stop() {
|
|
if (this.wss) {
|
|
this.wss.close();
|
|
this.wss = null;
|
|
}
|
|
if (this.server) {
|
|
this.server.close();
|
|
this.server = null;
|
|
}
|
|
console.log('WebSocket server stopped');
|
|
}
|
|
|
|
getClientsCount() {
|
|
if (!this.wss) return 0;
|
|
return this.wss.clients.size;
|
|
}
|
|
}
|