Files
api_health/test/media.test.js
T
lik 13981c047b fix: 真机 16k mp3 服务端解码为 PCM,修复 ASR 实时回显失败
DashScope 无法解码微信真机录制的 16k MPEG-2 Layer III mp3(报 DecodePost resample audio from 0 to 16000 failed)。按文件头识别容器后,实时与整段两条链路统一先解码为 16bit 单声道 PCM 再推供应商。
2026-10-31 19:54:37 +08:00

553 lines
20 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { test } from "node:test";
import assert from "node:assert/strict";
import { createRecognizer } from "../agent/infra/media/asr.js";
import { createOcrSession } from "../agent/infra/media/ocr.js";
import {
buildMessages,
createQwenVisionSession,
formatTranscription,
parseVisionResult,
} from "../agent/infra/media/ocr_qwen.js";
import {
decodeBase64Media,
normalizeAudioMeta,
normalizeImageMeta,
MediaInputError,
} from "../agent/infra/media/index.js";
import {
createStreamingAudioDecoder,
createStreamingMp3Decoder,
createStreamingWebmDecoder,
downsamplePcm16,
findEbmlHeader,
mp3ToPcm,
sniffAudioFormat,
webmToPcm,
} from "../agent/infra/media/audio.js";
/** 测试用静默 logger:不写日志文件,也不污染输出 */
const silentLogger = { info() {}, warn() {}, debug() {} };
test("音频元信息按白名单规范化", () => {
assert.deepEqual(
normalizeAudioMeta({ format: "mp3", sampleRate: 16000, channels: 1 }),
{ format: "mp3", codec: "mp3", mime: "audio/mpeg", rate: 16000, channels: 1 }
);
// 大小写与 pcm 别名都归一到内部编码
assert.equal(normalizeAudioMeta({ format: "PCM" }).codec, "pcm16");
assert.equal(normalizeAudioMeta({}).codec, "mp3");
assert.throws(() => normalizeAudioMeta({ format: "ogg" }), /不支持的音频格式/);
assert.throws(() => normalizeAudioMeta({ format: "mp3", sampleRate: 44100 }), /不支持的采样率/);
assert.throws(() => normalizeAudioMeta({ format: "mp3", channels: 2 }), /单声道/);
});
test("按文件头识别音频容器,开发者工具 WebM 不会被当成 mp3", () => {
assert.equal(sniffAudioFormat(Buffer.from([0x1a, 0x45, 0xdf, 0xa3, 0x01])), "webm");
assert.equal(sniffAudioFormat(Buffer.from("RIFF\0\0\0\0WAVEfmt ", "binary")), "wav");
assert.equal(sniffAudioFormat(Buffer.from("ID3\x03\x00", "binary")), "mp3");
assert.equal(sniffAudioFormat(Buffer.from([0x49, 0x44, 0x33])), "mp3");
// 0xFFF1 是 ADTS AAC(layer 位为 00);0xFFFB 是 MPEG 音频
assert.equal(sniffAudioFormat(Buffer.from([0xff, 0xf1, 0x50, 0x80])), "aac");
assert.equal(sniffAudioFormat(Buffer.from([0xff, 0xfb, 0x90, 0x64])), "mp3");
assert.equal(sniffAudioFormat(Buffer.from([1, 2, 3])), "unknown");
assert.equal(sniffAudioFormat(Buffer.alloc(0)), "unknown");
});
test("首片带前导字节时能在头部窗口内定位 EBML 头", () => {
const magic = Buffer.from([0x1a, 0x45, 0xdf, 0xa3]);
const padded = Buffer.concat([Buffer.from([0, 0, 0]), magic, Buffer.alloc(8)]);
assert.equal(findEbmlHeader(padded), 3);
assert.equal(findEbmlHeader(Buffer.concat([magic, Buffer.alloc(8)]), 4096), 0);
// 纯编码数据里不该出现魔数,扫描结果必须是 -1
assert.equal(findEbmlHeader(Buffer.from([0xff, 0xf3, 0x84, 0xc0, 0x00, 0x11]), 4096), -1);
});
test("WebM 解码结果下混为单声道 16bit PCM", async () => {
const decodeImpl = async () => ({
channelData: [new Float32Array([0.5, -0.5, 1.5, -1.5])],
sampleRate: 48000,
});
const { pcm, rate, samples } = await webmToPcm(Buffer.from("webm"), decodeImpl);
assert.equal(rate, 48000);
assert.equal(samples, 4);
assert.equal(pcm.length, 8);
assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767));
assert.equal(pcm.readInt16LE(2), Math.round(-0.5 * 32767));
// 浮点越界裁剪到满量程,避免回绕成爆音
assert.equal(pcm.readInt16LE(4), 32767);
assert.equal(pcm.readInt16LE(6), -32767);
// 立体声取平均下混
const stereo = await webmToPcm(Buffer.from("webm"), async () => ({
channelData: [new Float32Array([1, -1]), new Float32Array([0, 0])],
sampleRate: 48000,
}));
assert.equal(stereo.pcm.readInt16LE(0), Math.round(0.5 * 32767));
assert.equal(stereo.pcm.readInt16LE(2), Math.round(-0.5 * 32767));
await assert.rejects(
() => webmToPcm(Buffer.from("webm"), async () => ({ channelData: [] })),
/decoded audio is empty/
);
});
test("WebM 流式解码器按分片连续产出 PCM", async () => {
const received = [];
const fakeDecoder = {
decode(chunk) {
received.push(chunk.length);
// 每个分片解出一帧,实时链路按片回填
return { channelData: [new Float32Array([0.5, -0.5])], sampleRate: 48000 };
},
flush() {
return { channelData: [new Float32Array([0.25])], sampleRate: 48000 };
},
free() {},
};
const decoder = await createStreamingWebmDecoder(async () => fakeDecoder);
assert.equal(decoder.rate, 48000);
const first = await decoder.push(Buffer.from([1, 2, 3]));
const second = await decoder.push(Buffer.from([4, 5]));
const tail = await decoder.flush();
// 分片原样交给解码器,解码结果按 16bit 单声道输出
assert.deepEqual(received, [3, 2]);
assert.equal(first.length, 4);
assert.equal(second.length, 4);
assert.equal(tail.length, 2);
decoder.free();
});
test("MP3 解码结果下混为 16bit 单声道 PCM", async () => {
// mpg123 对单声道也输出左右两路相同的通道数据,下混后仍是原样本
const { pcm, rate, samples } = await mp3ToPcm(Buffer.from("mp3"), async () => ({
channelData: [
new Float32Array([0.5, -0.5]),
new Float32Array([0.5, -0.5]),
],
sampleRate: 16000,
}));
assert.equal(rate, 16000);
assert.equal(samples, 2);
assert.equal(pcm.length, 4);
assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767));
assert.equal(pcm.readInt16LE(2), Math.round(-0.5 * 32767));
await assert.rejects(
() => mp3ToPcm(Buffer.from("mp3"), async () => ({ channelData: [] })),
/decoded audio is empty/
);
});
test("MP3 流式解码器按分片产出 PCM,并以解码器实际采样率为准", async () => {
const fakeDecoder = {
decode(chunk) {
// 不足一帧时 mpg123 返回空通道数据和 0 采样率,不能把 rate 写成 0
if (chunk.length < 4) return { channelData: [], sampleRate: 0 };
return {
channelData: [new Float32Array([0.5]), new Float32Array([0.5])],
sampleRate: 16000,
};
},
free() {},
};
const decoder = await createStreamingMp3Decoder(async () => fakeDecoder);
assert.equal(decoder.rate, 16000);
assert.equal((await decoder.push(Buffer.from([1, 2, 3]))).length, 0);
assert.equal(decoder.rate, 16000);
const pcm = await decoder.push(Buffer.from([4, 5, 6, 7]));
assert.equal(pcm.length, 2);
assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767));
// mp3 没有尾部缓冲,flush 必须是空结果而不是抛错
assert.equal((await decoder.flush()).length, 0);
decoder.free();
});
test("流式解码器按容器分发,未知容器直接报错", async () => {
// 只验证分发入口的契约,避免在单测里加载 WASM 解码器
await assert.rejects(
() => createStreamingAudioDecoder("aac"),
/unsupported streaming audio format/
);
});
test("48k PCM 按整数倍抽取到 16k,非整数倍保持原样", () => {
const pcm = Buffer.alloc(48 * 2);
for (let i = 0; i < 48; i += 1) {
pcm.writeInt16LE(100 * (i + 1), i * 2);
}
const out = downsamplePcm16(pcm, 48000);
assert.equal(out.length, 32);
// 每 3 个点取均值:(100+200+300)/3 = 200
assert.equal(out.readInt16LE(0), 200);
assert.equal(out.readInt16LE(2), 500);
// 44.1k 无法整数倍抽取,声明与数据必须保持一致
assert.equal(downsamplePcm16(pcm, 44100).length, pcm.length);
});
test("图片元信息只接受白名单 MIME 且类型固定 auto", () => {
assert.deepEqual(
normalizeImageMeta({ mime: "IMAGE/PNG", width: 800, height: 600 }),
{ mime: "image/png", width: 800, height: 600, kind: "auto" }
);
// 非法尺寸不阻断识别,仅丢弃
assert.equal(normalizeImageMeta({ mime: "image/jpeg", width: -1 }).width, 0);
assert.throws(() => normalizeImageMeta({ mime: "image/gif" }), /不支持的图片格式/);
assert.throws(() => normalizeImageMeta({ mime: "image/jpeg", kind: "report" }), /仅支持 auto/);
});
test("base64 媒体解码兼容 data URL 并校验大小", () => {
assert.equal(decodeBase64Media(Buffer.from("hello").toString("base64")).toString(), "hello");
assert.equal(
decodeBase64Media(`data:audio/mpeg;base64,${Buffer.from("hi").toString("base64")}`).toString(),
"hi"
);
assert.throws(() => decodeBase64Media(""), MediaInputError);
assert.throws(() => decodeBase64Media("***"), /不是有效的 base64/);
assert.throws(
() => decodeBase64Media(Buffer.alloc(16).toString("base64"), { maxBytes: 8, label: "图片" }),
/图片过大/
);
});
test("批式 ASR 会话累积分片并返回去空白文本", async () => {
let captured = null;
const session = createRecognizer({
transcribe: async (audio, meta) => {
captured = { bytes: audio.length, meta };
return " 你好,我是小橙 ";
},
requestId: "test-asr",
logger: silentLogger,
});
session.push(Buffer.from([1, 2]));
session.push(Buffer.from([3]));
assert.equal(await session.finish(), "你好,我是小橙");
assert.deepEqual(captured, {
bytes: 3,
meta: { codec: "mp3", rate: 16000, channels: 1 },
});
});
test("ASR 会话中止后返回空串且不再识别", async () => {
const session = createRecognizer({
transcribe: async () => "不应被调用",
logger: silentLogger,
});
session.push(Buffer.from("audio"));
session.abort();
assert.equal(await session.finish(), "");
});
test("未配置 ASR 供应商时给出可判断的错误", async () => {
const session = createRecognizer({ logger: silentLogger });
session.push(Buffer.from("audio"));
await assert.rejects(() => session.finish(), /asr provider not configured/);
});
test("DashScope 会话缺密钥时在本地失败且不建连", async () => {
const session = createRecognizer({
provider: "dashscope",
endpoint: "wss://example.invalid/asr",
apiKey: "",
apiKeyEnv: "__TEST_MISSING_ASR_KEY__",
logger: silentLogger,
});
session.push(Buffer.from("audio"));
await assert.rejects(() => session.finish(), /api key missing/);
});
test("DashScope 实时会话把中间结果经 onPartial 回传,末帧定稿", async () => {
/** 假 WebSocket:只回放本用例关心的事件 */
class FakeWs {
constructor(url, options) {
this.url = url;
this.options = options;
this.readyState = 1;
this.bufferedAmount = 0;
this.sent = [];
this.listeners = {};
FakeWs.instances.push(this);
}
on(event, callback) {
this.listeners[event] = callback;
}
emit(event, data) {
this.listeners[event]?.(data);
}
send(data) {
this.sent.push(data);
}
close() {
this.readyState = 3;
}
}
FakeWs.instances = [];
const partials = [];
const session = createRecognizer({
provider: "dashscope",
endpoint: "wss://example.invalid/asr",
apiKey: "test-key",
logger: silentLogger,
// 关掉节流,断言每一个中间结果都回调
partialIntervalMs: 0,
WebSocketImpl: FakeWs,
onPartial: (text) => partials.push(text),
});
session.push(Buffer.from([1, 2, 3, 4]));
const ws = FakeWs.instances[0];
assert.ok(ws, "首帧应触发建连");
// 建连后先发 run-task,再在 task-started 时把缓存的音频冲出去
ws.emit("open");
const runTask = JSON.parse(ws.sent[0]);
assert.equal(runTask.header.action, "run-task");
assert.equal(runTask.header.task_id.length > 0, true);
ws.emit("message", JSON.stringify({ header: { event: "task-started" }, payload: {} }));
assert.equal(Buffer.isBuffer(ws.sent[1]), true);
// sentence_end=false 是中间结果:必须实时回显(不再等整句说完)
ws.emit("message", JSON.stringify({
header: { event: "result-generated" },
payload: { output: { sentence: { text: "你好", sentence_end: false } } },
}));
assert.deepEqual(partials, ["你好"]);
ws.emit("message", JSON.stringify({
header: { event: "result-generated" },
payload: { output: { sentence: { text: "你好,我是小橙", sentence_end: false } } },
}));
assert.deepEqual(partials, ["你好", "你好,我是小橙"]);
// 定稿:句子结束 + finish-task 后的 task-finished 取全量
const finished = session.finish();
ws.emit("message", JSON.stringify({
header: { event: "task-finished" },
payload: { output: { text: "你好,我是小橙。" } },
}));
assert.match(await finished, /你好,我是小橙。/);
});
test("批式 OCR 会话累积分片并传递元信息", async () => {
let captured = null;
const session = createOcrSession({
recognize: async (image, meta) => {
captured = { bytes: image.length, meta };
return " 报告文本 ";
},
mime: "image/png",
width: 640,
height: 480,
logger: silentLogger,
});
session.push(Buffer.from([1, 2]));
session.push(Buffer.from([3]));
assert.equal(await session.finish(), "报告文本");
assert.deepEqual(captured, {
bytes: 3,
meta: { mime: "image/png", kind: "auto", width: 640, height: 480 },
});
});
test("未配置 OCR 供应商时给出可判断的错误", async () => {
// 缺省走 qwen 视觉:没有密钥时在本地失败
const session = createOcrSession({
apiKey: "",
apiKeyEnv: "__TEST_MISSING_OCR_KEY__",
logger: silentLogger,
});
session.push(Buffer.from("image"));
await assert.rejects(() => session.finish(), /ocr api key missing/);
// 显式指定批式供应商但没有 endpoint / recognize 时是「未配置」
const batch = createOcrSession({ provider: "http", logger: silentLogger });
batch.push(Buffer.from("image"));
await assert.rejects(() => batch.finish(), /ocr provider not configured/);
});
test("只转写模式只传图片,不附加会压缩原文的提示词", () => {
const messages = buildMessages({
mime: "image/jpeg",
base64: "AAAA",
width: 800,
height: 600,
profile: "transcribe",
});
// 专用 OCR 模型自带逐行模板,附加系统提示或 JSON 契约会把原文压成摘要
assert.equal(messages.length, 1);
assert.equal(messages[0].role, "user");
assert.equal(messages[0].content.length, 1);
assert.equal(messages[0].content[0].type, "image_url");
assert.equal(messages[0].content[0].image_url.url, "data:image/jpeg;base64,AAAA");
});
test("视觉结果解析兼容代码块并按内容纠正报告类型", () => {
const fenced = parseVisionResult(
'```json\n{"docType":"lab_report","title":"胸部CT","text":"双肺纹理清晰","quality":{"readable":true,"issues":[]}}\n```'
);
// CT 被模型误判成 lab_report 时按标题纠正为影像
assert.match(fenced.text, /【图像识别】影像|胸部CT/);
assert.match(fenced.text, /【原文转写】\n\n双肺纹理清晰/);
assert.match(fenced.text, /【说明】/);
// 非 JSON 输出按完整原文转写落盘,不再丢掉【原文转写】段落
const plain = parseVisionResult("这是一段无法结构化的识别文本");
assert.equal(plain.data.text, "这是一段无法结构化的识别文本");
assert.match(plain.text, /【图像识别】/);
assert.match(plain.text, /【原文转写】\n\n这是一段无法结构化的识别文本/);
});
test("原生 OCR 行结果还原为 markdown 表格并补齐空列", () => {
// 原生输出用多个空格分列;markdown 会折叠空格,因此要转成表格
const raw = [
'```json',
'[{"rotate_rect":[358,40,33,375,90],"text":"血常规检验报告单"},',
'{"rotate_rect":[268,96,25,453,90],"text":"项目 结果 单位 参考范围 提示"},',
'{"rotate_rect":[362,136,25,643,90],"text":"白细胞计数 WBC 6.2 10^9/L 3.5-9.5"},',
'{"rotate_rect":[362,176,25,643,90],"text":"血红蛋白 HGB 128 g/L 130-175 偏低"}]',
'```',
].join("\n");
const result = parseVisionResult(raw);
assert.equal(result.data.text.split("\n").length, 4);
assert.match(result.text, /【图像识别】检验报告|血常规检验报告单/);
assert.match(result.text, /【原文转写】\n\n血常规检验报告单\n\n/);
assert.match(result.text, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/);
assert.match(result.text, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| 3\.5-9\.5 \| \|/);
assert.match(result.text, /\| 血红蛋白 HGB \| 128 \| g\/L \| 130-175 \| 偏低 \|/);
});
test("原生 OCR 的制表符与全角空格分列同样还原成表格", () => {
const tabs = "项目\t结果\t单位\t参考范围\t提示\n白细胞计数 WBC\t6.2\t10^9/L\t3.5-9.5";
const tabTable = formatTranscription(tabs);
assert.match(tabTable, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/);
assert.match(tabTable, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| 3\.5-9\.5 \| \|/);
// qwen-vl-ocr 原生输出常用全角空格对齐列
const fullWidth = "项目\u3000\u3000结果\u3000\u3000单位\u3000\u3000参考范围\u3000\u3000提示\n白细胞计数 WBC\u3000\u30006.2\u3000\u300010^9/L";
const fullWidthTable = formatTranscription(fullWidth);
assert.match(fullWidthTable, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/);
assert.match(fullWidthTable, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| \| \|/);
});
test("识别行只有坐标没有文字时视为未识别", () => {
const result = parseVisionResult('[{"rotate_rect":[358,40,33,375,90]},{"rotate_rect":[268,96,25,453,90]}]');
assert.equal(result.text, "");
});
test("纯文本里的方括号不会被当成截断区间,且行与行保持独立", () => {
const raw = "参考范围:白细胞 [3.5-9.5] 10^9/L\n红细胞 [4.3-5.8] 10^12/L\n其余检查结果见报告。";
const result = parseVisionResult(raw);
assert.equal(result.data.text, raw);
assert.match(result.text, /【原文转写】\n\n参考范围:白细胞 \[3\.5-9\.5\] 10\^9\/L\n\n红细胞/);
assert.match(result.text, /红细胞 \[4\.3-5\.8\] 10\^12\/L\n\n其余检查结果见报告。/);
});
test("排版不把「姓名 性别 年龄」误判成表格", () => {
const text = [
"姓名:张三 性别:男 年龄:58岁",
"检查号:CT20261031001 检查日期:2026-10-31 检查部位:胸部",
"影像所见:双肺纹理清晰。",
].join("\n");
const formatted = formatTranscription(text);
assert.doesNotMatch(formatted, /\| --- \|/);
// 每行独立成段,避免 markdown 把多行合并
assert.equal(formatted.split("\n\n").length, 3);
});
test("Qwen 视觉会话提交整图并解析为可读文本", async () => {
let captured = null;
const fetchImpl = async (url, options) => {
captured = { url, options, body: JSON.parse(options.body) };
return {
ok: true,
json: async () => ({
choices: [{
message: {
content: '{"docType":"lab_report","title":"血常规","text":"白细胞 6.2","quality":{"readable":true,"issues":[]}}',
},
finish_reason: "stop",
}],
usage: { total_tokens: 120 },
}),
};
};
const session = createQwenVisionSession({
apiKey: "test-key",
model: "qwen-vl-ocr",
mime: "image/png",
width: 320,
height: 240,
fetchImpl,
logger: silentLogger,
});
session.push(Buffer.from([1, 2, 3]));
const text = await session.finish();
assert.match(text, /【图像识别】检验报告|血常规/);
assert.match(text, /【原文转写】\n\n白细胞 6\.2/);
assert.equal(captured.body.model, "qwen-vl-ocr");
assert.equal(captured.body.temperature, 0);
assert.match(captured.options.headers.Authorization, /Bearer test-key/);
});
test("Qwen 视觉会话把 HTTP 错误转成可判断的失败", async () => {
const fetchImpl = async () => ({
ok: false,
status: 401,
text: async () => "invalid api key",
});
const session = createQwenVisionSession({
apiKey: "bad-key",
fetchImpl,
logger: silentLogger,
});
session.push(Buffer.from("image"));
await assert.rejects(() => session.finish(), /ocr http 401/);
});
test("Qwen 视觉会话中止时取消进行中的请求", async () => {
const fetchImpl = (url, options) => new Promise((resolve, reject) => {
options.signal.addEventListener("abort", () => {
const err = new Error("aborted");
err.name = "AbortError";
reject(err);
});
});
const session = createQwenVisionSession({
apiKey: "test-key",
fetchImpl,
logger: silentLogger,
});
session.push(Buffer.from("image"));
const pending = session.finish();
session.abort();
await assert.rejects(() => pending, /ocr aborted/);
});