Files
api_health/websocket.js
T
lik 47fdb91dbe fix(agent): 图片识别原文直接回显,避免长报告被护栏截断
- 新增 WS format 帧:识别原文只并入会话上下文并分片原样回显,不再经过模型二次改写

- 原方案走 chat 会被咨询提示词压缩成 1-3 句,并被 OutputGuard 的 400 字上限截断

- 补充分片回显测试与 README 协议说明
2026-11-01 12:31:56 +08:00

637 lines
22 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import WebSocket, { WebSocketServer } from 'ws';
import http from 'http';
import { DBModel } from "./models/index.js";
import { chatTask } from "./agent/bots/bot-escort/task.js";
import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js"
import { getUserByToken } from "./utils/user_service.js";
import { resolveEscortAdminRole } from "./utils/permissions.js";
import logger from "./utils/logger.js";
import { createUserLimiter } from "./middleware/ratelimit.js";
import {
createAudioStreamRecognizer,
isAsrConfigured,
normalizeAudioMeta,
decodeBase64Media
} from "./agent/infra/media/index.js";
import {
sniffAudioFormat,
findEbmlHeader,
createStreamingAudioDecoder,
downsamplePcm16
} from "./agent/infra/media/audio.js";
let systemMessageSequence = 0;
// 单连接上行帧上限:超限会被 ws 直接断开,避免超大 JSON 占用内存与 CPU
const MAX_PAYLOAD_BYTES = 256 * 1024;
// 单用户聊天频次限制:AI 调用有成本,且同一用户的会话本就串行处理
const chatLimiter = createUserLimiter(20);
// 实时语音识别同样按次计费:每轮 asr_start 计一次,与聊天额度分开
const asrLimiter = createUserLimiter(20);
// 实时识别分片上限:单帧按客户端 frameSize=8KB 留足余量,
// 整轮按 60 秒录音的一倍余量收口,超过直接中止本轮,避免连接内存被刷满
const ASR_LIMITS = {
chunkBytes: 128 * 1024,
totalBytes: 8 * 1024 * 1024,
// 首帧认不出容器时先缓存再判(有些平台的首帧只是 cluster,没有文件头)
sniffBytes: 16 * 1024
};
/**
* WebM 解码出的 PCM 采样率归一
*
* 实时协议对裸 PCM 只稳定支持 16k:48k/24k 这类整数倍能整除到 16k,
* 直接降到 16k;其他采样率保持原值,避免声明与数据不一致。
*/
const resolveAsrRate = (sourceRate) => {
const rate = Number(sourceRate) || 48000;
return Number.isInteger(rate / 16000) && rate > 16000 ? 16000 : rate;
};
const sendFrame = (ws, source, type, content = '', id = '') => {
if (ws.readyState !== WebSocket.OPEN) return;
ws.send(JSON.stringify({
source,
type,
content,
id: id || `system_${Date.now()}_${++systemMessageSequence}`
}));
};
export default class WebSocketServerManager {
constructor(port = 8080) {
this.port = port;
this.wss = null;
this.server = null;
}
start() {
this.server = http.createServer((req, res) => {
res.writeHead(426, { 'Content-Type': 'text/plain' });
res.end('Upgrade Required: WebSocket expected');
});
this.wss = new WebSocketServer({
server: this.server,
maxPayload: MAX_PAYLOAD_BYTES,
perMessageDeflate: {
zlibDeflateOptions: {
chunkSize: 1024,
memLevel: 7,
level: 3
},
zlibInflateOptions: {
chunkSize: 10 * 1024
},
clientNoContextTakeover: true,
serverNoContextTakeover: true,
serverMaxWindowBits: 10,
concurrencyLimit: 10,
threshold: 1024
}
});
this.wss.on('connection', (ws, req) => {
const path = req.url;
console.log(`WebSocket connected: ${req.socket.remoteAddress}, path: ${path}`);
if (path === '/chat') {
this._handleChatConnection(ws);
} else {
sendFrame(ws, 'system', 'error', 'Unknown path');
ws.close(1000, 'Unknown path');
}
});
this.server.listen(this.port, () => {
console.log(`WebSocket server started on ws://localhost:${this.port}`);
});
}
_handleChatConnection(ws) {
sendFrame(ws, 'system', 'connected', 'Connected to chat server');
// 同一连接的消息串行处理:agent 的 messages 数组是共享状态,
// 并发跑同一个 agent 会让上下文交错、回复错乱
ws.messageQueue = Promise.resolve();
ws.on('message', (data) => {
ws.messageQueue = ws.messageQueue
.then(() => this._handleMessage(ws, data))
.catch((err) => console.error('WebSocket message handling error:', err));
});
ws.on('close', () => {
// 断开后释放在途的识别会话与 WebM 解码器:不释放会让供应商连接与缓冲悬空
this._abortAsrSession(ws);
// 断开后释放队列引用,避免连接对象被闭包长期持有
ws.messageQueue = null;
console.log('Chat client disconnected');
});
ws.on('error', (err) => {
console.error('WebSocket error:', err);
});
}
async _handleMessage(ws, data) {
try {
const raw = data.toString();
const msg = JSON.parse(raw);
// 只记录消息类型:正文含健康描述、token/userId 属敏感信息,不进日志。
// 实时识别的分片按秒级频率到达,逐帧打日志会把日志刷满,这里跳过
if (msg.type !== 'asr_chunk') {
console.log('Received message type:', msg.type);
}
if (msg.type === 'ping') {
sendFrame(ws, 'system', 'pong', '', msg.id);
return;
}
// 实时语音识别:与聊天共用一条连接,一轮识别 = start → 若干 chunk → stop/cancel
if (msg.type === 'asr_start') {
await this._handleAsrStart(ws, msg);
return;
}
if (msg.type === 'asr_chunk') {
this._handleAsrChunk(ws, msg);
return;
}
if (msg.type === 'asr_stop') {
await this._handleAsrStop(ws, msg);
return;
}
if (msg.type === 'asr_cancel') {
this._handleAsrCancel(ws, msg);
return;
}
// format 是图片识别原文的投递帧:只并入会话上下文并回显原文,不触发模型生成
if (msg.type === 'chat' || msg.type === 'clear' || msg.type === 'format') {
// 客户端断开后不再处理消息
if (ws.readyState !== WebSocket.OPEN) return;
const send = (source, type, content, id) => {
sendFrame(ws, source, type, content, id);
};
try {
// 身份只认 token:不再接受凭 userId 明文接入,防止冒充他人聊天
if (!msg.token) {
send('system', 'error', '请先登录', msg.id);
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
send('system', 'error', '请先登录', msg.id);
return;
}
// 频次限制按用户维度:防止单账号刷 AI 调用(同一账号多连接共享额度)
if (msg.type === 'chat') {
const limit = chatLimiter(String(userInfo._id));
if (!limit.allowed) {
send('system', 'error', `操作过于频繁,请 ${limit.retryAfter} 秒后重试`, msg.id);
return;
}
}
if (msg.agent === 'escort-admin') {
// 管理端 agent 按角色收敛工具:管理员可跨用户/写库,陪诊员只能看本人订单。
// 身份以 Employee 记录为准,普通 C 端账号不能借 agent 升级权限。
const role = await resolveEscortAdminRole(userInfo);
if (!role) {
send('system', 'error', '当前账号未绑定在职陪诊员档案,请联系管理员开通', msg.id);
return;
}
await adminAgent.streamChat(userInfo, [msg], (source, type, content, id) => {
send(source, type, content, id);
}, role);
} else {
await chatTask.streamChat(userInfo, msg, (source, type, content, id) => {
send(source, type, content, id);
});
}
// 流式 chunk 可能携带不同的模型消息 ID,结束帧统一关联本次客户端请求。
send('system', 'end', '', msg.id);
} catch (err) {
console.error('streamChat error:', err);
send('system', 'error', '处理消息失败,请稍后重试', msg.id);
}
return;
}
if (msg.type === 'system') {
console.log('[System]', msg.content);
return;
}
console.log('[Unknown message type]', msg.type);
} catch (e) {
console.error('Message parse error:', e);
sendFrame(ws, 'system', 'error', '消息格式错误');
}
}
/** 释放当前连接的识别会话(主动中止,不会让供应商再回结果) */
_abortAsrSession(ws) {
const state = ws.asrSession;
ws.asrSession = null;
if (state) {
state.session?.abort();
state.decoder?.free();
}
return state;
}
/**
* 建供应商会话(首帧确认真实容器后才建,避免空轮次占用连接)
*
* 采集阶段失败统一走 onError:客户端收到 asr_error 后继续录完整句,
* 结束时改走整段 /health/ai/asr。
*/
_createAsrSession(ws, state, { codec, rate }) {
const session = createAudioStreamRecognizer({
codec,
rate,
channels: state.meta.channels,
requestId: `asr_stream_${state.id}`,
// 中间结果即时回显;客户端按 id 过滤,只认本轮
onPartial: (text) => {
if (ws.asrSession !== state) return;
// 每个会话只记一次:用来区分「实时链路通了」与「一直没出中间结果」
if (!state.partialSeen) {
state.partialSeen = true;
logger.info('asr stream first result', {
requestId: state.id,
format: state.format,
chars: text.length
});
}
sendFrame(ws, 'system', 'asr_partial', text, state.id);
},
onError: (err) => {
if (ws.asrSession !== state) return;
state.failed = true;
state.session = null;
logger.warn('asr stream failed', {
requestId: state.id,
partialSeen: state.partialSeen,
bytes: state.bytes,
error: err?.message ?? String(err)
});
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
}
});
state.session = session;
return session;
}
/** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */
_failAsrStream(ws, state, message) {
logger.warn('asr stream degraded', {
requestId: state.id,
format: state.format || 'unknown',
bytes: state.bytes,
partialSeen: state.partialSeen,
reason: message
});
if (ws.asrSession === state) ws.asrSession = null;
state.failed = true;
state.session?.abort();
state.session = null;
state.decoder?.free();
state.decoder = null;
sendFrame(ws, 'system', 'asr_error', message, state.id);
}
/**
* 开始一轮实时识别
* 上行:{ type: 'asr_start', id, token, format?, sampleRate?, channels? }
* 身份只认 token,与聊天同一套鉴权;每轮按用户限流一次
*/
async _handleAsrStart(ws, msg) {
// 同一连接不叠加两轮识别:先中止上一轮,避免两份供应商会话并行计费
this._abortAsrSession(ws);
const fail = (content) => sendFrame(ws, 'system', 'asr_error', content, msg.id);
try {
if (!msg.token) {
fail('请先登录');
return;
}
const userInfo = await getUserByToken(msg.token);
if (!userInfo) {
fail('请先登录');
return;
}
if (!isAsrConfigured()) {
fail('语音识别服务暂不可用');
return;
}
const limit = asrLimiter(String(userInfo._id));
if (!limit.allowed) {
fail(`操作过于频繁,请 ${limit.retryAfter} 秒后重试`);
return;
}
let meta;
try {
meta = normalizeAudioMeta({
format: msg.format || 'mp3',
sampleRate: msg.sampleRate,
channels: msg.channels
});
} catch (err) {
fail(err.message || '语音参数不支持');
return;
}
// 供应商会话按首帧确认真实容器后再建:真机是 16k mp3,浏览器(开发者工具)是 WebM。
// 两种容器供应商都解不了(实测报 DecodePost 解码失败),都要先在服务端流式解成 PCM,
// 因此不能把原始分片直接透传,也不能按声明格式共用一份 run-task 参数
const state = {
id: msg.id || `asr_${Date.now()}`,
userId: String(userInfo._id),
meta,
bytes: 0,
failed: false,
format: '',
pending: [],
decoder: null,
// 帧头诊断只打一次:确认客户端分片是否带完整容器头
headLogged: false,
// 推给供应商的 PCM 采样率(解码后按实际输出归一)
rate: 0,
// 是否已收到过中间结果(仅用于诊断日志)
partialSeen: false,
session: null,
// 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱
queue: Promise.resolve()
};
ws.asrSession = state;
// 只在开头记一条:用来区分「客户端根本没发分片」与「分片到了但解码/供应商失败」
logger.info('asr stream started', {
requestId: state.id,
declaredFormat: meta.format,
rate: meta.rate,
channels: meta.channels
});
} catch (err) {
console.error('[ASR] start failed:', err);
// 失败时不能留下半截状态:后续分片会误以为本轮还在进行
// 进入本方法时已中止上一轮,所以这里的 session 必然是本次新建的
if (ws.asrSession) this._abortAsrSession(ws);
fail('语音识别服务暂不可用');
}
}
/**
* 推入一帧音频(base64)
* 单帧解码失败只丢这一帧:录音仍在继续,不该因一帧异常打断整轮
*
* 分片串行排队处理:WebM 解码和供应商推送都是异步的,
* 并发处理会让 PCM 顺序错乱,也会把供应商会话的 run-task 竞态化。
*/
_handleAsrChunk(ws, msg) {
const state = ws.asrSession;
if (!state || state.failed) return;
if (msg.id && msg.id !== state.id) return;
let buffer;
try {
buffer = decodeBase64Media(msg.audio, {
maxBytes: ASR_LIMITS.chunkBytes,
label: '语音分片'
});
} catch (err) {
return;
}
if (state.bytes + buffer.length > ASR_LIMITS.totalBytes) {
this._failAsrStream(ws, state, '录音过长,请缩短语音后重试');
return;
}
state.bytes += buffer.length;
state.queue = state.queue
.then(() => this._consumeAsrChunk(ws, state, buffer))
.catch((err) => {
console.warn('[ASR] chunk failed:', err?.message ?? err, { requestId: state.id });
});
}
/**
* 处理一帧音频:首帧先判真实容器,再决定解码方式
*
* 真机录的是 16k mp3,开发者工具录的是 WebM/Opus,
* 两种容器供应商都解不了,统一在服务端边收边解码成 PCM 再推。
*/
async _consumeAsrChunk(ws, state, buffer) {
if (ws.asrSession !== state || state.failed) return;
if (!state.format) {
state.pending.push(buffer);
let head = Buffer.concat(state.pending);
let sniffed = sniffAudioFormat(head);
// 首片前可能带少量前导字节,容器头不在偏移 0:定位 EBML 头后裁掉再按 WebM 处理
if (sniffed === 'unknown') {
const ebmlOffset = findEbmlHeader(head);
if (ebmlOffset > 0) {
logger.info('asr stream trimmed to ebml header', {
requestId: state.id,
offset: ebmlOffset,
headBytes: head.length
});
head = head.subarray(ebmlOffset);
sniffed = 'webm';
}
}
// 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判
if (sniffed === 'unknown') {
// 只记一次帧头:用来确认客户端分片到底是不是完整容器(开发者工具常缺 EBML 头)
if (!state.headLogged) {
state.headLogged = true;
logger.info('asr stream head unrecognized', {
requestId: state.id,
declaredFormat: state.meta.format,
headBytes: head.length,
firstBytes: head.subarray(0, Math.min(head.length, 48)).toString('hex')
});
}
if (head.length < ASR_LIMITS.sniffBytes) return;
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
if (sniffed !== 'mp3' && sniffed !== 'webm') {
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
return;
}
state.format = sniffed;
state.pending = [];
// 排查实时链路时先看这一行:确认服务端认出的真实容器与帧头
logger.info('asr stream format detected', {
requestId: state.id,
format: sniffed,
headBytes: head.length,
firstBytes: head.subarray(0, Math.min(head.length, 16)).toString('hex')
});
// webm(开发者工具)与 mp3(真机)都先流式解码成 PCM:
// 供应商对这两种容器的解码都不可靠,透传会整轮失败
try {
state.decoder = await createStreamingAudioDecoder(sniffed);
} catch (err) {
console.warn('[ASR] audio decoder init failed:', err?.message ?? err, {
requestId: state.id,
format: sniffed
});
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
return;
}
// 供应商会话等第一段 PCM 解出来再建:采样率以解码器实际输出为准
await this._pushAsrAudio(ws, state, head);
return;
}
await this._pushAsrAudio(ws, state, buffer);
}
/** 把一帧音频推给供应商:容器分片先解码成 PCM,再按 16k 归一推流 */
async _pushAsrAudio(ws, state, buffer) {
if (ws.asrSession !== state || state.failed || !state.decoder) return;
try {
const pcm = await state.decoder.push(buffer);
// 首帧往往只有容器头(WebM 的 EBML / mp3 的元信息),解不出音频:
// 等第一段 PCM 出来再建供应商会话
if (!pcm.length) return;
if (!state.session) {
// 采样率以首帧解码结果为准:Opus 是 48k,真机 mp3 是 16k
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
const out = downsamplePcm16(pcm, state.decoder.rate, state.rate);
if (out.length) state.session.push(out);
} catch (err) {
// 解码失败说明这一轮的分片无法拼成完整音频,交给客户端结束后的整段兜底
console.warn('[ASR] audio decode failed:', err?.message ?? err, { requestId: state.id });
this._failAsrStream(ws, state, '语音识别失败,请稍后重试');
}
}
/**
* 结束本轮并回定稿
* 上行:{ type: 'asr_stop', id }
* 下行:asr_final(可能为空串,由客户端提示重录)
*/
async _handleAsrStop(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
// 采集阶段的失败已经回过 asr_error,这里只需收尾
if (state.failed) {
this._abortAsrSession(ws);
return;
}
try {
// 先把排队的最后一帧处理完,否则会漏掉句尾(也会让 WebM 少解码一段)
await state.queue;
if (ws.asrSession !== state || state.failed) return;
// 一帧都没收到:说明客户端没发分片(录音分片回调未触发 / 还在用旧包),
// 这里记一条,配合 asr stream started 可直接定位
if (!state.format) {
logger.warn('asr stream finished without audio', {
requestId: state.id,
declaredFormat: state.meta.format,
bytes: state.bytes
});
}
// 流式解码收尾:把解码器里剩余的音频冲出来再收(mp3 无尾部缓冲,返回空)
if (state.decoder) {
const tail = await state.decoder.flush();
if (!state.session && tail.length) {
state.rate = resolveAsrRate(state.decoder.rate);
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
}
if (tail.length && state.session) {
const out = downsamplePcm16(tail, state.decoder.rate, state.rate);
if (out.length) state.session.push(out);
}
}
const text = state.session ? (await state.session.finish()) || '' : '';
if (ws.asrSession === state) ws.asrSession = null;
logger.info('asr stream finished', {
requestId: state.id,
format: state.format || 'unknown',
bytes: state.bytes,
partialSeen: state.partialSeen,
chars: text.length
});
sendFrame(ws, 'system', 'asr_final', text, state.id);
} catch (err) {
if (ws.asrSession === state) ws.asrSession = null;
state.session?.abort();
console.warn('[ASR] finish failed:', err.message, { requestId: state.id, bytes: state.bytes });
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
} finally {
state.decoder?.free();
state.decoder = null;
}
}
/**
* 取消本轮(页面隐藏、录音中断、客户端兜底失败)
* 只中止会话,不回任何帧,也不产生定稿
*/
_handleAsrCancel(ws, msg) {
const state = ws.asrSession;
if (!state) return;
if (msg.id && msg.id !== state.id) return;
this._abortAsrSession(ws);
}
broadcast(message) {
if (!this.wss) return;
this.wss.clients.forEach((client) => {
if (client.readyState === WebSocket.OPEN) {
client.send(message);
}
});
}
stop() {
if (this.wss) {
this.wss.close();
this.wss = null;
}
if (this.server) {
this.server.close();
this.server = null;
}
console.log('WebSocket server stopped');
}
getClientsCount() {
if (!this.wss) return 0;
return this.wss.clients.size;
}
}