DashScope 无法解码微信真机录制的 16k MPEG-2 Layer III mp3(报 DecodePost resample audio from 0 to 16000 failed)。按文件头识别容器后,实时与整段两条链路统一先解码为 16bit 单声道 PCM 再推供应商。
369 lines
14 KiB
JavaScript
369 lines
14 KiB
JavaScript
/**
|
||
* 语音 / 图像识别入口(agent 侧多模态输入能力)
|
||
*
|
||
* 设计来源:xagent 的设备数据通道(asr.js / asr_dashscope.js / ocr.js / ocr_qwen.js),
|
||
* 这里保留「识别与传输解耦」的会话契约,把传输方式换成适合小程序的 HTTP 整段上传:
|
||
* - 上游只负责把音频/图片字节交给 recognizeAudio / recognizeImage
|
||
* - 识别结果只返回文本,不投递、不落库;是否交给 agent 由用户确认后走原聊天链路
|
||
* - 音频/图片只在本轮内存中存在,不写磁盘、不进日志
|
||
*
|
||
* 配置位于 conf.json 的 agent.media 段;密钥支持 apiKey 或 apiKeyEnv(默认与 xagent 一致的
|
||
* ASR_KEY,DASHSCOPE_API_KEY 作为兜底)。
|
||
*/
|
||
import config from '../../../config.js';
|
||
import logger from '../../../utils/logger.js';
|
||
import { createRecognizer, DEFAULT_AUDIO_FORMAT } from './asr.js';
|
||
import { sniffAudioFormat, webmToPcm, mp3ToPcm, downsamplePcm16 } from './audio.js';
|
||
import { createOcrSession, DEFAULT_IMAGE_FORMAT } from './ocr.js';
|
||
|
||
/** conf.json 的 agent.media 段(缺失时用内置默认值) */
|
||
const MEDIA_CONFIG = config.agent?.media ?? {};
|
||
|
||
/** 单轮媒体大小上限(默认与参考实现一致) */
|
||
export const MEDIA_LIMITS = {
|
||
maxAudioBytes: 10 * 1024 * 1024,
|
||
maxImageBytes: 8 * 1024 * 1024,
|
||
...(MEDIA_CONFIG.limits ?? {})
|
||
};
|
||
|
||
/** 支持的音频格式 → 识别供应商编码 */
|
||
export const AUDIO_FORMATS = Object.freeze({
|
||
mp3: { codec: 'mp3', mime: 'audio/mpeg' },
|
||
aac: { codec: 'aac', mime: 'audio/aac' },
|
||
wav: { codec: 'wav', mime: 'audio/wav' },
|
||
pcm: { codec: 'pcm16', mime: 'audio/L16' },
|
||
pcm16: { codec: 'pcm16', mime: 'audio/L16' }
|
||
});
|
||
|
||
/** 采样率白名单:实时识别协议支持 8k/16k/24k/48k */
|
||
const SUPPORTED_RATES = [8000, 16000, 24000, 48000];
|
||
|
||
/** 缺省采样率:与 asr.js 的 DEFAULT_AUDIO_FORMAT.rate 保持一致 */
|
||
const DEVICE_DEFAULT_RATE = DEFAULT_AUDIO_FORMAT.rate;
|
||
|
||
/** 图片 MIME 白名单(默认值;可由 conf.json 的 media.image.mimes 覆盖) */
|
||
export const DEFAULT_IMAGE_MIMES = [...DEFAULT_IMAGE_FORMAT.mimes];
|
||
|
||
/** 输入参数错误:调用方按 400 回复,不重试 */
|
||
export class MediaInputError extends Error {
|
||
constructor(message) {
|
||
super(message);
|
||
this.name = 'MediaInputError';
|
||
this.status = 400;
|
||
}
|
||
}
|
||
|
||
function readEnv(name) {
|
||
const value = name ? process.env[name] : '';
|
||
return value ? String(value).trim() : '';
|
||
}
|
||
|
||
/**
|
||
* 解析识别密钥:显式配置 > apiKeyEnv > 参考项目的 ASR_KEY > DashScope 通用变量
|
||
* 变量名与 xagent 保持一致,便于两个项目共用同一把 key
|
||
* @param {Object} cfg 供应商配置
|
||
* @returns {string}
|
||
*/
|
||
function resolveApiKey(cfg = {}) {
|
||
return String(cfg.apiKey || '').trim()
|
||
|| readEnv(cfg.apiKeyEnv)
|
||
|| readEnv('ASR_KEY')
|
||
|| readEnv('DASHSCOPE_API_KEY');
|
||
}
|
||
|
||
/** 语音识别是否已配置(用于快速回「服务暂不可用」,避免先解码大文件) */
|
||
export function isAsrConfigured() {
|
||
const cfg = MEDIA_CONFIG.asr ?? {};
|
||
const provider = cfg.provider || (/^wss?:\/\//i.test(cfg.endpoint || '') ? 'dashscope' : '');
|
||
if (provider === 'dashscope' || !cfg.endpoint) {
|
||
return Boolean(resolveApiKey(cfg));
|
||
}
|
||
return true;
|
||
}
|
||
|
||
/** 图像识别是否已配置(qwen 需要密钥,自建 HTTP endpoint 视为已配置) */
|
||
export function isOcrConfigured() {
|
||
const cfg = MEDIA_CONFIG.ocr ?? {};
|
||
const provider = cfg.provider ?? 'qwen';
|
||
if (provider === 'qwen' || provider === '') {
|
||
return Boolean(resolveApiKey(cfg) || cfg.endpoint);
|
||
}
|
||
return Boolean(cfg.endpoint || resolveApiKey(cfg));
|
||
}
|
||
|
||
/**
|
||
* 音频元信息规范化:格式、采样率、声道都按白名单校验
|
||
* @param {Object} [input]
|
||
* @param {string} [input.format] - mp3 / aac / wav / pcm / pcm16
|
||
* @param {number} [input.sampleRate] - 采样率
|
||
* @param {number} [input.channels] - 声道数(实时协议仅支持单声道)
|
||
* @returns {{ format: string, codec: string, mime: string, rate: number, channels: number }}
|
||
*/
|
||
export function normalizeAudioMeta({ format, sampleRate, channels } = {}) {
|
||
const key = String(format || 'mp3').trim().toLowerCase();
|
||
const spec = AUDIO_FORMATS[key];
|
||
if (!spec) {
|
||
throw new MediaInputError(`不支持的音频格式:${format}(仅支持 mp3、aac、wav、pcm)`);
|
||
}
|
||
|
||
const audioCfg = MEDIA_CONFIG.audio ?? {};
|
||
const rate = Number(sampleRate ?? audioCfg.rate ?? DEFAULT_AUDIO_FORMAT.rate);
|
||
if (!SUPPORTED_RATES.includes(rate)) {
|
||
throw new MediaInputError(`不支持的采样率:${sampleRate}(仅支持 8000、16000、24000、48000)`);
|
||
}
|
||
|
||
const channelCount = Number(channels ?? audioCfg.channels ?? DEFAULT_AUDIO_FORMAT.channels);
|
||
if (channelCount !== 1) {
|
||
throw new MediaInputError('语音识别仅支持单声道音频');
|
||
}
|
||
|
||
return { format: key, codec: spec.codec, mime: spec.mime, rate, channels: channelCount };
|
||
}
|
||
|
||
/**
|
||
* 图片元信息规范化:MIME 白名单,类型首版固定 auto
|
||
* @param {Object} [input]
|
||
* @param {string} [input.mime] - image/jpeg / png / webp
|
||
* @param {number} [input.width] - 像素宽(仅用于模型判断清晰度)
|
||
* @param {number} [input.height] - 像素高
|
||
* @param {string} [input.kind] - 首版只接受 auto
|
||
* @returns {{ mime: string, width: number, height: number, kind: string }}
|
||
*/
|
||
export function normalizeImageMeta({ mime, width, height, kind } = {}) {
|
||
const mimes = MEDIA_CONFIG.image?.mimes ?? DEFAULT_IMAGE_MIMES;
|
||
const picked = String(mime || 'image/jpeg').trim().toLowerCase();
|
||
if (!mimes.includes(picked)) {
|
||
throw new MediaInputError(`不支持的图片格式:${mime}(仅支持 ${mimes.join('、')})`);
|
||
}
|
||
|
||
if (kind !== undefined && kind !== 'auto') {
|
||
throw new MediaInputError(`不支持的图片类型:${kind}(首版仅支持 auto)`);
|
||
}
|
||
|
||
const size = (value) => {
|
||
const num = Number(value);
|
||
return Number.isFinite(num) && num > 0 && num <= 100000 ? Math.round(num) : 0;
|
||
};
|
||
|
||
return { mime: picked, width: size(width), height: size(height), kind: 'auto' };
|
||
}
|
||
|
||
/**
|
||
* 解码 base64 媒体内容(兼容 data URL 前缀),并按上限校验
|
||
* @param {string} value - base64 字符串或 data URL
|
||
* @param {Object} [options]
|
||
* @param {number} [options.maxBytes] - 解码后字节上限
|
||
* @param {string} [options.label] - 错误文案里的名词(音频 / 图片)
|
||
* @returns {Buffer}
|
||
*/
|
||
export function decodeBase64Media(value, { maxBytes = 0, label = '内容' } = {}) {
|
||
if (typeof value !== 'string' || !value.trim()) {
|
||
throw new MediaInputError(`${label}内容为空`);
|
||
}
|
||
|
||
const raw = /^data:/i.test(value) && value.includes(',')
|
||
? value.slice(value.indexOf(',') + 1)
|
||
: value;
|
||
const base64 = raw.replace(/\s+/g, '');
|
||
if (!base64 || !/^[A-Za-z0-9+/]+={0,2}$/.test(base64)) {
|
||
throw new MediaInputError(`${label}不是有效的 base64 数据`);
|
||
}
|
||
|
||
// 先按字符数粗判,避免为超限内容分配大块内存
|
||
if (maxBytes > 0 && base64.length > Math.ceil(maxBytes * 4 / 3) + 8) {
|
||
throw new MediaInputError(`${label}过大(最大 ${Math.round(maxBytes / 1024 / 1024)}MB)`);
|
||
}
|
||
|
||
const buffer = Buffer.from(base64, 'base64');
|
||
if (buffer.length === 0) {
|
||
throw new MediaInputError(`${label}内容为空`);
|
||
}
|
||
if (maxBytes > 0 && buffer.length > maxBytes) {
|
||
throw new MediaInputError(`${label}过大(最大 ${Math.round(maxBytes / 1024 / 1024)}MB)`);
|
||
}
|
||
return buffer;
|
||
}
|
||
|
||
/** 整段识别按 300ms 音频切帧(压缩格式按 64KB 切),与「分片上传」的节奏接近 */
|
||
const BATCH_CHUNK_MS = 300;
|
||
const BATCH_COMPRESSED_CHUNK_BYTES = 64 * 1024;
|
||
|
||
/**
|
||
* 把整段音频切成适合供应商的分片
|
||
*
|
||
* 一次性推入几秒 PCM(48k 单声道约 30KB/百毫秒)会让单帧过大、发送队列迅速积压,
|
||
* 供应商侧表现为直接断流(asr socket closed)。切成小块更接近实时分片的节奏。
|
||
*/
|
||
function chunkAudio(input, meta = {}) {
|
||
const codec = meta.codec || '';
|
||
const isPcm = codec === 'pcm16' || codec === 'pcm';
|
||
const size = isPcm
|
||
? Math.max(3200, Math.round(((meta.rate || DEVICE_DEFAULT_RATE) * 2 * BATCH_CHUNK_MS) / 1000))
|
||
: BATCH_COMPRESSED_CHUNK_BYTES;
|
||
|
||
const chunks = [];
|
||
for (let offset = 0; offset < input.length; offset += size) {
|
||
chunks.push(input.subarray(offset, offset + size));
|
||
}
|
||
return chunks;
|
||
}
|
||
|
||
/**
|
||
* 顺序推入分片,并在分片之间让出事件循环
|
||
*
|
||
* 同步 for 循环发送会把整段音频瞬间塞进 socket 发送队列,供应商来不及消费;
|
||
* 每片之间让出一次宏任务,既保持连续又不会积压。
|
||
*/
|
||
async function pushChunks(session, chunks) {
|
||
for (let index = 0; index < chunks.length; index += 1) {
|
||
session.push(chunks[index]);
|
||
if (index + 1 < chunks.length) {
|
||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||
}
|
||
}
|
||
}
|
||
|
||
/**
|
||
* 整段音频识别(一轮一段,失败即抛错)
|
||
* @param {Buffer} audio - 原始音频字节
|
||
* @param {Object} meta - normalizeAudioMeta 的返回值,另可带 requestId
|
||
* @returns {Promise<string>} 识别文本(可能为空串)
|
||
*/
|
||
export async function recognizeAudio(audio, meta = {}) {
|
||
const cfg = MEDIA_CONFIG.asr ?? {};
|
||
|
||
// 两条录音链路的容器供应商都解不了(开发者工具是 WebM/Opus,真机是 16k MP3,
|
||
// 实测都会报 DecodePost 解码失败,见 audio.js),这里按文件头识别后统一转成 PCM。
|
||
let input = audio;
|
||
let inputMeta = meta;
|
||
const container = sniffAudioFormat(audio);
|
||
if (container === 'webm' || container === 'mp3') {
|
||
try {
|
||
const decoded = container === 'webm' ? await webmToPcm(audio) : await mp3ToPcm(audio);
|
||
// 实时协议对裸 PCM 只稳定支持 16k:48k 先整数倍抽取,与实时链路保持一致
|
||
const rate = Number.isInteger(decoded.rate / 16000) && decoded.rate > 16000 ? 16000 : decoded.rate;
|
||
input = rate === decoded.rate ? decoded.pcm : downsamplePcm16(decoded.pcm, decoded.rate, rate);
|
||
inputMeta = {
|
||
...meta,
|
||
format: 'pcm',
|
||
codec: 'pcm16',
|
||
mime: 'audio/L16',
|
||
rate,
|
||
channels: 1
|
||
};
|
||
logger.info('audio container decoded for asr', {
|
||
requestId: meta.requestId || '',
|
||
container,
|
||
bytes: audio.length,
|
||
sourceRate: decoded.rate,
|
||
pcmBytes: input.length,
|
||
rate
|
||
});
|
||
} catch (err) {
|
||
throw new MediaInputError('录音内容无法解析,请重新录制');
|
||
}
|
||
}
|
||
|
||
const sessionOptions = {
|
||
...cfg,
|
||
apiKey: resolveApiKey(cfg),
|
||
codec: inputMeta.codec || DEFAULT_AUDIO_FORMAT.codec,
|
||
rate: inputMeta.rate || DEVICE_DEFAULT_RATE,
|
||
channels: inputMeta.channels || DEFAULT_AUDIO_FORMAT.channels,
|
||
requestId: meta.requestId || '',
|
||
logger
|
||
};
|
||
|
||
// 供应商偶发断流(socket closed / 建连超时)不值得让用户重录:同一条音频再试一次。
|
||
// 只重试传输层错误;参数/密钥/解码类错误重试也不会成功
|
||
const TRANSIENT_ERROR = /socket closed|socket hang up|connect timeout|ECONNRESET|ETIMEDOUT/i;
|
||
const MAX_ATTEMPTS = 2;
|
||
let lastError = null;
|
||
const chunks = chunkAudio(input, inputMeta);
|
||
|
||
for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt += 1) {
|
||
const session = createRecognizer(sessionOptions);
|
||
try {
|
||
await pushChunks(session, chunks);
|
||
return await session.finish();
|
||
} catch (err) {
|
||
session.abort();
|
||
lastError = err;
|
||
if (attempt + 1 < MAX_ATTEMPTS && TRANSIENT_ERROR.test(err?.message ?? '')) {
|
||
logger.warn('asr transient failure, retrying', {
|
||
requestId: meta.requestId || '',
|
||
attempt: attempt + 1,
|
||
error: err.message
|
||
});
|
||
await new Promise((resolve) => setTimeout(resolve, 600));
|
||
continue;
|
||
}
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
throw lastError;
|
||
}
|
||
|
||
/**
|
||
* 创建一轮流式识别会话(供 WS 分片推流使用)
|
||
*
|
||
* 与 recognizeAudio 共用同一份供应商配置与密钥解析,区别是不预先缓冲整段音频:
|
||
* 调用方按分片持续 push,中间文本经 onPartial 实时回调,最后 finish() 取定稿。
|
||
* 只用于实时供应商(dashscope);批式配置下 push 的分片会在 finish 时整段转写。
|
||
*
|
||
* @param {Object} [meta]
|
||
* @param {string} [meta.codec] - 内部编码(默认 pcm16,按客户端录制格式传入)
|
||
* @param {number} [meta.rate] - 采样率
|
||
* @param {number} [meta.channels] - 声道数
|
||
* @param {string} [meta.requestId] - 请求标识(仅日志关联用)
|
||
* @param {Function} [meta.onPartial] - 中间结果回调 (text) => void
|
||
* @param {Function} [meta.onError] - 采集阶段异步失败回调
|
||
* @returns {{ push: Function, finish: Function, abort: Function }}
|
||
*/
|
||
export function createAudioStreamRecognizer(meta = {}) {
|
||
const cfg = MEDIA_CONFIG.asr ?? {};
|
||
return createRecognizer({
|
||
...cfg,
|
||
apiKey: resolveApiKey(cfg),
|
||
codec: meta.codec || 'pcm16',
|
||
rate: meta.rate || DEVICE_DEFAULT_RATE,
|
||
channels: meta.channels || DEFAULT_AUDIO_FORMAT.channels,
|
||
requestId: meta.requestId || '',
|
||
onPartial: meta.onPartial || null,
|
||
onError: meta.onError || null,
|
||
logger
|
||
});
|
||
}
|
||
|
||
/**
|
||
* 整张图片识别(一轮一张,失败即抛错)
|
||
* @param {Buffer} image - 原始图片字节
|
||
* @param {Object} meta - normalizeImageMeta 的返回值,另可带 requestId
|
||
* @returns {Promise<string>} 可读文本(含类型、原文、质量与免责说明)
|
||
*/
|
||
export async function recognizeImage(image, meta = {}) {
|
||
const cfg = MEDIA_CONFIG.ocr ?? {};
|
||
const session = createOcrSession({
|
||
...cfg,
|
||
apiKey: resolveApiKey(cfg),
|
||
mime: meta.mime || DEFAULT_IMAGE_FORMAT.mimes[0],
|
||
width: meta.width || 0,
|
||
height: meta.height || 0,
|
||
kind: meta.kind || 'auto',
|
||
turnId: meta.requestId || '',
|
||
logger
|
||
});
|
||
|
||
try {
|
||
session.push(image);
|
||
return await session.finish();
|
||
} catch (err) {
|
||
session.abort();
|
||
throw err;
|
||
}
|
||
}
|
||
|
||
/** 会话工厂与生效配置一并导出,便于上层自定义或单测注入 */
|
||
export { createRecognizer, createOcrSession, MEDIA_CONFIG };
|