fix: 修复实时语音 asr_start 直接失败,并补齐全链路诊断与 EBML 头裁剪
- websocket.js 补上缺失的 logger 导入:asr_start 里 logger.info 抛 ReferenceError 被 catch 兜住, 客户端只会收到「语音识别服务暂不可用」,实时回显整条链路直接失效 - asr_start 失败时中止本次会话,避免遗留半截 asrSession 让后续分片误判 - onError / 降级 / 空音频收尾统一走 logger,记录 format/bytes/partialSeen,便于区分 「没发分片」「解码失败」「供应商失败」三种情况 - 首帧嗅探失败时记录前 48 字节 hex;新增 findEbmlHeader,在头 4KB 内定位 EBML 魔数, 命中后裁掉前导字节按 WebM 继续流式解码(开发者工具分片头不一定在偏移 0) - WebM 首帧只有容器头、解不出 PCM 时不提前建供应商会话
This commit is contained in:
@@ -13,6 +13,26 @@ import decodeWebm, { decoder as createWebmDecoder } from '@audio/decode-webm';
|
||||
/** WebM / Matroska 的 EBML 魔数:1A 45 DF A3 */
|
||||
const EBML_MAGIC = 0x1a45dfa3;
|
||||
|
||||
/**
|
||||
* 在头部窗口内定位 EBML 头
|
||||
*
|
||||
* 有些平台的 onFrameRecorded 首片前面会带少量前导字节(半帧 / padding),
|
||||
* 容器头不在偏移 0,直接嗅探会判成 unknown;EBML 魔数是 4 字节常量,
|
||||
* 在小窗口内扫描不会误判,找到后裁掉前导字节即可继续按 WebM 流式解码。
|
||||
*
|
||||
* @param {Buffer|Uint8Array} audio - 已累积的帧头字节
|
||||
* @param {number} limit - 最多扫描多少字节
|
||||
* @returns {number} 魔数偏移,找不到返回 -1
|
||||
*/
|
||||
export function findEbmlHeader(audio, limit = 4096) {
|
||||
const buf = Buffer.isBuffer(audio) ? audio : Buffer.from(audio);
|
||||
const last = Math.min(buf.length - 4, limit);
|
||||
for (let offset = 0; offset <= last; offset += 1) {
|
||||
if (buf.readUInt32BE(offset) === EBML_MAGIC) return offset;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* 按文件头判断音频容器
|
||||
* @param {Buffer|Uint8Array} audio - 原始音频字节
|
||||
|
||||
+90
-4
@@ -5,6 +5,7 @@ import { chatTask } from "./agent/bots/bot-escort/task.js";
|
||||
import { adminAgent } from "./agent/bots/bot-escort-admin/agent.js"
|
||||
import { getUserByToken } from "./utils/user_service.js";
|
||||
import { isAdmin } from "./utils/permissions.js";
|
||||
import logger from "./utils/logger.js";
|
||||
import { createUserLimiter } from "./middleware/ratelimit.js";
|
||||
import {
|
||||
createAudioStreamRecognizer,
|
||||
@@ -14,6 +15,7 @@ import {
|
||||
} from "./agent/infra/media/index.js";
|
||||
import {
|
||||
sniffAudioFormat,
|
||||
findEbmlHeader,
|
||||
createStreamingWebmDecoder,
|
||||
downsamplePcm16
|
||||
} from "./agent/infra/media/audio.js";
|
||||
@@ -259,13 +261,27 @@ export default class WebSocketServerManager {
|
||||
// 中间结果即时回显;客户端按 id 过滤,只认本轮
|
||||
onPartial: (text) => {
|
||||
if (ws.asrSession !== state) return;
|
||||
// 每个会话只记一次:用来区分「实时链路通了」与「一直没出中间结果」
|
||||
if (!state.partialSeen) {
|
||||
state.partialSeen = true;
|
||||
logger.info('asr stream first result', {
|
||||
requestId: state.id,
|
||||
format: state.format,
|
||||
chars: text.length
|
||||
});
|
||||
}
|
||||
sendFrame(ws, 'system', 'asr_partial', text, state.id);
|
||||
},
|
||||
onError: (err) => {
|
||||
if (ws.asrSession !== state) return;
|
||||
state.failed = true;
|
||||
state.session = null;
|
||||
console.warn('[ASR] stream failed:', err.message, { requestId: state.id });
|
||||
logger.warn('asr stream failed', {
|
||||
requestId: state.id,
|
||||
partialSeen: state.partialSeen,
|
||||
bytes: state.bytes,
|
||||
error: err?.message ?? String(err)
|
||||
});
|
||||
sendFrame(ws, 'system', 'asr_error', '语音识别失败,请稍后重试', state.id);
|
||||
}
|
||||
});
|
||||
@@ -275,6 +291,13 @@ export default class WebSocketServerManager {
|
||||
|
||||
/** 流式失败:中止会话、释放解码器,并通知客户端改走整段转写 */
|
||||
_failAsrStream(ws, state, message) {
|
||||
logger.warn('asr stream degraded', {
|
||||
requestId: state.id,
|
||||
format: state.format || 'unknown',
|
||||
bytes: state.bytes,
|
||||
partialSeen: state.partialSeen,
|
||||
reason: message
|
||||
});
|
||||
if (ws.asrSession === state) ws.asrSession = null;
|
||||
state.failed = true;
|
||||
state.session?.abort();
|
||||
@@ -340,16 +363,30 @@ export default class WebSocketServerManager {
|
||||
format: '',
|
||||
pending: [],
|
||||
decoder: null,
|
||||
// 帧头诊断只打一次:确认客户端分片是否带完整容器头
|
||||
headLogged: false,
|
||||
// 推给供应商的 PCM 采样率(WebM 解码后归一,mp3 链路不用)
|
||||
rate: 0,
|
||||
// 是否已收到过中间结果(仅用于诊断日志)
|
||||
partialSeen: false,
|
||||
session: null,
|
||||
// 分片串行处理:解码与推送是异步的,乱序会把 PCM 顺序打乱
|
||||
queue: Promise.resolve()
|
||||
};
|
||||
|
||||
ws.asrSession = state;
|
||||
// 只在开头记一条:用来区分「客户端根本没发分片」与「分片到了但解码/供应商失败」
|
||||
logger.info('asr stream started', {
|
||||
requestId: state.id,
|
||||
declaredFormat: meta.format,
|
||||
rate: meta.rate,
|
||||
channels: meta.channels
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[ASR] start failed:', err);
|
||||
// 失败时不能留下半截状态:后续分片会误以为本轮还在进行
|
||||
// 进入本方法时已中止上一轮,所以这里的 session 必然是本次新建的
|
||||
if (ws.asrSession) this._abortAsrSession(ws);
|
||||
fail('语音识别服务暂不可用');
|
||||
}
|
||||
}
|
||||
@@ -400,11 +437,35 @@ export default class WebSocketServerManager {
|
||||
|
||||
if (!state.format) {
|
||||
state.pending.push(buffer);
|
||||
const head = Buffer.concat(state.pending);
|
||||
const sniffed = sniffAudioFormat(head);
|
||||
let head = Buffer.concat(state.pending);
|
||||
let sniffed = sniffAudioFormat(head);
|
||||
|
||||
// 首片前可能带少量前导字节,容器头不在偏移 0:定位 EBML 头后裁掉再按 WebM 处理
|
||||
if (sniffed === 'unknown') {
|
||||
const ebmlOffset = findEbmlHeader(head);
|
||||
if (ebmlOffset > 0) {
|
||||
logger.info('asr stream trimmed to ebml header', {
|
||||
requestId: state.id,
|
||||
offset: ebmlOffset,
|
||||
headBytes: head.length
|
||||
});
|
||||
head = head.subarray(ebmlOffset);
|
||||
sniffed = 'webm';
|
||||
}
|
||||
}
|
||||
|
||||
// 认不出容器(有些平台首帧只有 cluster、没有文件头):先攒着,够量再判
|
||||
if (sniffed === 'unknown') {
|
||||
// 只记一次帧头:用来确认客户端分片到底是不是完整容器(开发者工具常缺 EBML 头)
|
||||
if (!state.headLogged) {
|
||||
state.headLogged = true;
|
||||
logger.info('asr stream head unrecognized', {
|
||||
requestId: state.id,
|
||||
declaredFormat: state.meta.format,
|
||||
headBytes: head.length,
|
||||
firstBytes: head.subarray(0, Math.min(head.length, 48)).toString('hex')
|
||||
});
|
||||
}
|
||||
if (head.length < ASR_LIMITS.sniffBytes) return;
|
||||
this._failAsrStream(ws, state, '实时识别不支持当前录音格式,将在结束后整段识别');
|
||||
return;
|
||||
@@ -416,6 +477,12 @@ export default class WebSocketServerManager {
|
||||
|
||||
state.format = sniffed;
|
||||
state.pending = [];
|
||||
// 排查实时链路时先看这一行:确认服务端认出的真实容器
|
||||
logger.info('asr stream format detected', {
|
||||
requestId: state.id,
|
||||
format: sniffed,
|
||||
headBytes: head.length
|
||||
});
|
||||
|
||||
if (sniffed === 'webm') {
|
||||
try {
|
||||
@@ -445,12 +512,14 @@ export default class WebSocketServerManager {
|
||||
try {
|
||||
if (state.decoder) {
|
||||
const pcm = await state.decoder.push(buffer);
|
||||
// 首帧往往只有 WebM 头,解不出音频:等第一段 PCM 出来再建供应商会话
|
||||
if (!pcm.length) return;
|
||||
if (!state.session) {
|
||||
// 采样率以首帧解码结果为准:Opus 是 48k,整数倍抽取到 16k
|
||||
state.rate = resolveAsrRate(state.decoder.rate);
|
||||
this._createAsrSession(ws, state, { codec: 'pcm16', rate: state.rate });
|
||||
}
|
||||
const out = pcm.length ? downsamplePcm16(pcm, state.decoder.rate, state.rate) : pcm;
|
||||
const out = downsamplePcm16(pcm, state.decoder.rate, state.rate);
|
||||
if (out.length) state.session.push(out);
|
||||
return;
|
||||
}
|
||||
@@ -483,6 +552,16 @@ export default class WebSocketServerManager {
|
||||
await state.queue;
|
||||
if (ws.asrSession !== state || state.failed) return;
|
||||
|
||||
// 一帧都没收到:说明客户端没发分片(录音分片回调未触发 / 还在用旧包),
|
||||
// 这里记一条,配合 asr stream started 可直接定位
|
||||
if (!state.format) {
|
||||
logger.warn('asr stream finished without audio', {
|
||||
requestId: state.id,
|
||||
declaredFormat: state.meta.format,
|
||||
bytes: state.bytes
|
||||
});
|
||||
}
|
||||
|
||||
// WebM 流式解码:把解码器里剩余的帧冲出来再收尾
|
||||
if (state.decoder) {
|
||||
const tail = await state.decoder.flush();
|
||||
@@ -498,6 +577,13 @@ export default class WebSocketServerManager {
|
||||
|
||||
const text = state.session ? (await state.session.finish()) || '' : '';
|
||||
if (ws.asrSession === state) ws.asrSession = null;
|
||||
logger.info('asr stream finished', {
|
||||
requestId: state.id,
|
||||
format: state.format || 'unknown',
|
||||
bytes: state.bytes,
|
||||
partialSeen: state.partialSeen,
|
||||
chars: text.length
|
||||
});
|
||||
sendFrame(ws, 'system', 'asr_final', text, state.id);
|
||||
} catch (err) {
|
||||
if (ws.asrSession === state) ws.asrSession = null;
|
||||
|
||||
Reference in New Issue
Block a user