import { test } from "node:test"; import assert from "node:assert/strict"; import { createRecognizer } from "../agent/infra/media/asr.js"; import { createOcrSession } from "../agent/infra/media/ocr.js"; import { buildMessages, createQwenVisionSession, formatTranscription, parseVisionResult, } from "../agent/infra/media/ocr_qwen.js"; import { decodeBase64Media, normalizeAudioMeta, normalizeImageMeta, MediaInputError, } from "../agent/infra/media/index.js"; import { createStreamingAudioDecoder, createStreamingMp3Decoder, createStreamingWebmDecoder, downsamplePcm16, findEbmlHeader, mp3ToPcm, sniffAudioFormat, webmToPcm, } from "../agent/infra/media/audio.js"; /** 测试用静默 logger:不写日志文件,也不污染输出 */ const silentLogger = { info() {}, warn() {}, debug() {} }; test("音频元信息按白名单规范化", () => { assert.deepEqual( normalizeAudioMeta({ format: "mp3", sampleRate: 16000, channels: 1 }), { format: "mp3", codec: "mp3", mime: "audio/mpeg", rate: 16000, channels: 1 } ); // 大小写与 pcm 别名都归一到内部编码 assert.equal(normalizeAudioMeta({ format: "PCM" }).codec, "pcm16"); assert.equal(normalizeAudioMeta({}).codec, "mp3"); assert.throws(() => normalizeAudioMeta({ format: "ogg" }), /不支持的音频格式/); assert.throws(() => normalizeAudioMeta({ format: "mp3", sampleRate: 44100 }), /不支持的采样率/); assert.throws(() => normalizeAudioMeta({ format: "mp3", channels: 2 }), /单声道/); }); test("按文件头识别音频容器,开发者工具 WebM 不会被当成 mp3", () => { assert.equal(sniffAudioFormat(Buffer.from([0x1a, 0x45, 0xdf, 0xa3, 0x01])), "webm"); assert.equal(sniffAudioFormat(Buffer.from("RIFF\0\0\0\0WAVEfmt ", "binary")), "wav"); assert.equal(sniffAudioFormat(Buffer.from("ID3\x03\x00", "binary")), "mp3"); assert.equal(sniffAudioFormat(Buffer.from([0x49, 0x44, 0x33])), "mp3"); // 0xFFF1 是 ADTS AAC(layer 位为 00);0xFFFB 是 MPEG 音频 assert.equal(sniffAudioFormat(Buffer.from([0xff, 0xf1, 0x50, 0x80])), "aac"); assert.equal(sniffAudioFormat(Buffer.from([0xff, 0xfb, 0x90, 0x64])), "mp3"); assert.equal(sniffAudioFormat(Buffer.from([1, 2, 3])), "unknown"); assert.equal(sniffAudioFormat(Buffer.alloc(0)), "unknown"); }); test("首片带前导字节时能在头部窗口内定位 EBML 头", () => { const magic = Buffer.from([0x1a, 0x45, 0xdf, 0xa3]); const padded = Buffer.concat([Buffer.from([0, 0, 0]), magic, Buffer.alloc(8)]); assert.equal(findEbmlHeader(padded), 3); assert.equal(findEbmlHeader(Buffer.concat([magic, Buffer.alloc(8)]), 4096), 0); // 纯编码数据里不该出现魔数,扫描结果必须是 -1 assert.equal(findEbmlHeader(Buffer.from([0xff, 0xf3, 0x84, 0xc0, 0x00, 0x11]), 4096), -1); }); test("WebM 解码结果下混为单声道 16bit PCM", async () => { const decodeImpl = async () => ({ channelData: [new Float32Array([0.5, -0.5, 1.5, -1.5])], sampleRate: 48000, }); const { pcm, rate, samples } = await webmToPcm(Buffer.from("webm"), decodeImpl); assert.equal(rate, 48000); assert.equal(samples, 4); assert.equal(pcm.length, 8); assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767)); assert.equal(pcm.readInt16LE(2), Math.round(-0.5 * 32767)); // 浮点越界裁剪到满量程,避免回绕成爆音 assert.equal(pcm.readInt16LE(4), 32767); assert.equal(pcm.readInt16LE(6), -32767); // 立体声取平均下混 const stereo = await webmToPcm(Buffer.from("webm"), async () => ({ channelData: [new Float32Array([1, -1]), new Float32Array([0, 0])], sampleRate: 48000, })); assert.equal(stereo.pcm.readInt16LE(0), Math.round(0.5 * 32767)); assert.equal(stereo.pcm.readInt16LE(2), Math.round(-0.5 * 32767)); await assert.rejects( () => webmToPcm(Buffer.from("webm"), async () => ({ channelData: [] })), /decoded audio is empty/ ); }); test("WebM 流式解码器按分片连续产出 PCM", async () => { const received = []; const fakeDecoder = { decode(chunk) { received.push(chunk.length); // 每个分片解出一帧,实时链路按片回填 return { channelData: [new Float32Array([0.5, -0.5])], sampleRate: 48000 }; }, flush() { return { channelData: [new Float32Array([0.25])], sampleRate: 48000 }; }, free() {}, }; const decoder = await createStreamingWebmDecoder(async () => fakeDecoder); assert.equal(decoder.rate, 48000); const first = await decoder.push(Buffer.from([1, 2, 3])); const second = await decoder.push(Buffer.from([4, 5])); const tail = await decoder.flush(); // 分片原样交给解码器,解码结果按 16bit 单声道输出 assert.deepEqual(received, [3, 2]); assert.equal(first.length, 4); assert.equal(second.length, 4); assert.equal(tail.length, 2); decoder.free(); }); test("MP3 解码结果下混为 16bit 单声道 PCM", async () => { // mpg123 对单声道也输出左右两路相同的通道数据,下混后仍是原样本 const { pcm, rate, samples } = await mp3ToPcm(Buffer.from("mp3"), async () => ({ channelData: [ new Float32Array([0.5, -0.5]), new Float32Array([0.5, -0.5]), ], sampleRate: 16000, })); assert.equal(rate, 16000); assert.equal(samples, 2); assert.equal(pcm.length, 4); assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767)); assert.equal(pcm.readInt16LE(2), Math.round(-0.5 * 32767)); await assert.rejects( () => mp3ToPcm(Buffer.from("mp3"), async () => ({ channelData: [] })), /decoded audio is empty/ ); }); test("MP3 流式解码器按分片产出 PCM,并以解码器实际采样率为准", async () => { const fakeDecoder = { decode(chunk) { // 不足一帧时 mpg123 返回空通道数据和 0 采样率,不能把 rate 写成 0 if (chunk.length < 4) return { channelData: [], sampleRate: 0 }; return { channelData: [new Float32Array([0.5]), new Float32Array([0.5])], sampleRate: 16000, }; }, free() {}, }; const decoder = await createStreamingMp3Decoder(async () => fakeDecoder); assert.equal(decoder.rate, 16000); assert.equal((await decoder.push(Buffer.from([1, 2, 3]))).length, 0); assert.equal(decoder.rate, 16000); const pcm = await decoder.push(Buffer.from([4, 5, 6, 7])); assert.equal(pcm.length, 2); assert.equal(pcm.readInt16LE(0), Math.round(0.5 * 32767)); // mp3 没有尾部缓冲,flush 必须是空结果而不是抛错 assert.equal((await decoder.flush()).length, 0); decoder.free(); }); test("流式解码器按容器分发,未知容器直接报错", async () => { // 只验证分发入口的契约,避免在单测里加载 WASM 解码器 await assert.rejects( () => createStreamingAudioDecoder("aac"), /unsupported streaming audio format/ ); }); test("48k PCM 按整数倍抽取到 16k,非整数倍保持原样", () => { const pcm = Buffer.alloc(48 * 2); for (let i = 0; i < 48; i += 1) { pcm.writeInt16LE(100 * (i + 1), i * 2); } const out = downsamplePcm16(pcm, 48000); assert.equal(out.length, 32); // 每 3 个点取均值:(100+200+300)/3 = 200 assert.equal(out.readInt16LE(0), 200); assert.equal(out.readInt16LE(2), 500); // 44.1k 无法整数倍抽取,声明与数据必须保持一致 assert.equal(downsamplePcm16(pcm, 44100).length, pcm.length); }); test("图片元信息只接受白名单 MIME 且类型固定 auto", () => { assert.deepEqual( normalizeImageMeta({ mime: "IMAGE/PNG", width: 800, height: 600 }), { mime: "image/png", width: 800, height: 600, kind: "auto" } ); // 非法尺寸不阻断识别,仅丢弃 assert.equal(normalizeImageMeta({ mime: "image/jpeg", width: -1 }).width, 0); assert.throws(() => normalizeImageMeta({ mime: "image/gif" }), /不支持的图片格式/); assert.throws(() => normalizeImageMeta({ mime: "image/jpeg", kind: "report" }), /仅支持 auto/); }); test("base64 媒体解码兼容 data URL 并校验大小", () => { assert.equal(decodeBase64Media(Buffer.from("hello").toString("base64")).toString(), "hello"); assert.equal( decodeBase64Media(`data:audio/mpeg;base64,${Buffer.from("hi").toString("base64")}`).toString(), "hi" ); assert.throws(() => decodeBase64Media(""), MediaInputError); assert.throws(() => decodeBase64Media("***"), /不是有效的 base64/); assert.throws( () => decodeBase64Media(Buffer.alloc(16).toString("base64"), { maxBytes: 8, label: "图片" }), /图片过大/ ); }); test("批式 ASR 会话累积分片并返回去空白文本", async () => { let captured = null; const session = createRecognizer({ transcribe: async (audio, meta) => { captured = { bytes: audio.length, meta }; return " 你好,我是小橙 "; }, requestId: "test-asr", logger: silentLogger, }); session.push(Buffer.from([1, 2])); session.push(Buffer.from([3])); assert.equal(await session.finish(), "你好,我是小橙"); assert.deepEqual(captured, { bytes: 3, meta: { codec: "mp3", rate: 16000, channels: 1 }, }); }); test("ASR 会话中止后返回空串且不再识别", async () => { const session = createRecognizer({ transcribe: async () => "不应被调用", logger: silentLogger, }); session.push(Buffer.from("audio")); session.abort(); assert.equal(await session.finish(), ""); }); test("未配置 ASR 供应商时给出可判断的错误", async () => { const session = createRecognizer({ logger: silentLogger }); session.push(Buffer.from("audio")); await assert.rejects(() => session.finish(), /asr provider not configured/); }); test("DashScope 会话缺密钥时在本地失败且不建连", async () => { const session = createRecognizer({ provider: "dashscope", endpoint: "wss://example.invalid/asr", apiKey: "", apiKeyEnv: "__TEST_MISSING_ASR_KEY__", logger: silentLogger, }); session.push(Buffer.from("audio")); await assert.rejects(() => session.finish(), /api key missing/); }); test("DashScope 实时会话把中间结果经 onPartial 回传,末帧定稿", async () => { /** 假 WebSocket:只回放本用例关心的事件 */ class FakeWs { constructor(url, options) { this.url = url; this.options = options; this.readyState = 1; this.bufferedAmount = 0; this.sent = []; this.listeners = {}; FakeWs.instances.push(this); } on(event, callback) { this.listeners[event] = callback; } emit(event, data) { this.listeners[event]?.(data); } send(data) { this.sent.push(data); } close() { this.readyState = 3; } } FakeWs.instances = []; const partials = []; const session = createRecognizer({ provider: "dashscope", endpoint: "wss://example.invalid/asr", apiKey: "test-key", logger: silentLogger, // 关掉节流,断言每一个中间结果都回调 partialIntervalMs: 0, WebSocketImpl: FakeWs, onPartial: (text) => partials.push(text), }); session.push(Buffer.from([1, 2, 3, 4])); const ws = FakeWs.instances[0]; assert.ok(ws, "首帧应触发建连"); // 建连后先发 run-task,再在 task-started 时把缓存的音频冲出去 ws.emit("open"); const runTask = JSON.parse(ws.sent[0]); assert.equal(runTask.header.action, "run-task"); assert.equal(runTask.header.task_id.length > 0, true); ws.emit("message", JSON.stringify({ header: { event: "task-started" }, payload: {} })); assert.equal(Buffer.isBuffer(ws.sent[1]), true); // sentence_end=false 是中间结果:必须实时回显(不再等整句说完) ws.emit("message", JSON.stringify({ header: { event: "result-generated" }, payload: { output: { sentence: { text: "你好", sentence_end: false } } }, })); assert.deepEqual(partials, ["你好"]); ws.emit("message", JSON.stringify({ header: { event: "result-generated" }, payload: { output: { sentence: { text: "你好,我是小橙", sentence_end: false } } }, })); assert.deepEqual(partials, ["你好", "你好,我是小橙"]); // 定稿:句子结束 + finish-task 后的 task-finished 取全量 const finished = session.finish(); ws.emit("message", JSON.stringify({ header: { event: "task-finished" }, payload: { output: { text: "你好,我是小橙。" } }, })); assert.match(await finished, /你好,我是小橙。/); }); test("批式 OCR 会话累积分片并传递元信息", async () => { let captured = null; const session = createOcrSession({ recognize: async (image, meta) => { captured = { bytes: image.length, meta }; return " 报告文本 "; }, mime: "image/png", width: 640, height: 480, logger: silentLogger, }); session.push(Buffer.from([1, 2])); session.push(Buffer.from([3])); assert.equal(await session.finish(), "报告文本"); assert.deepEqual(captured, { bytes: 3, meta: { mime: "image/png", kind: "auto", width: 640, height: 480 }, }); }); test("未配置 OCR 供应商时给出可判断的错误", async () => { // 缺省走 qwen 视觉:没有密钥时在本地失败 const session = createOcrSession({ apiKey: "", apiKeyEnv: "__TEST_MISSING_OCR_KEY__", logger: silentLogger, }); session.push(Buffer.from("image")); await assert.rejects(() => session.finish(), /ocr api key missing/); // 显式指定批式供应商但没有 endpoint / recognize 时是「未配置」 const batch = createOcrSession({ provider: "http", logger: silentLogger }); batch.push(Buffer.from("image")); await assert.rejects(() => batch.finish(), /ocr provider not configured/); }); test("只转写模式只传图片,不附加会压缩原文的提示词", () => { const messages = buildMessages({ mime: "image/jpeg", base64: "AAAA", width: 800, height: 600, profile: "transcribe", }); // 专用 OCR 模型自带逐行模板,附加系统提示或 JSON 契约会把原文压成摘要 assert.equal(messages.length, 1); assert.equal(messages[0].role, "user"); assert.equal(messages[0].content.length, 1); assert.equal(messages[0].content[0].type, "image_url"); assert.equal(messages[0].content[0].image_url.url, "data:image/jpeg;base64,AAAA"); }); test("视觉结果解析兼容代码块并按内容纠正报告类型", () => { const fenced = parseVisionResult( '```json\n{"docType":"lab_report","title":"胸部CT","text":"双肺纹理清晰","quality":{"readable":true,"issues":[]}}\n```' ); // CT 被模型误判成 lab_report 时按标题纠正为影像 assert.match(fenced.text, /【图像识别】影像|胸部CT/); assert.match(fenced.text, /【原文转写】\n\n双肺纹理清晰/); assert.match(fenced.text, /【说明】/); // 非 JSON 输出按完整原文转写落盘,不再丢掉【原文转写】段落 const plain = parseVisionResult("这是一段无法结构化的识别文本"); assert.equal(plain.data.text, "这是一段无法结构化的识别文本"); assert.match(plain.text, /【图像识别】/); assert.match(plain.text, /【原文转写】\n\n这是一段无法结构化的识别文本/); }); test("原生 OCR 行结果还原为 markdown 表格并补齐空列", () => { // 原生输出用多个空格分列;markdown 会折叠空格,因此要转成表格 const raw = [ '```json', '[{"rotate_rect":[358,40,33,375,90],"text":"血常规检验报告单"},', '{"rotate_rect":[268,96,25,453,90],"text":"项目 结果 单位 参考范围 提示"},', '{"rotate_rect":[362,136,25,643,90],"text":"白细胞计数 WBC 6.2 10^9/L 3.5-9.5"},', '{"rotate_rect":[362,176,25,643,90],"text":"血红蛋白 HGB 128 g/L 130-175 偏低"}]', '```', ].join("\n"); const result = parseVisionResult(raw); assert.equal(result.data.text.split("\n").length, 4); assert.match(result.text, /【图像识别】检验报告|血常规检验报告单/); assert.match(result.text, /【原文转写】\n\n血常规检验报告单\n\n/); assert.match(result.text, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/); assert.match(result.text, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| 3\.5-9\.5 \| \|/); assert.match(result.text, /\| 血红蛋白 HGB \| 128 \| g\/L \| 130-175 \| 偏低 \|/); }); test("原生 OCR 的制表符与全角空格分列同样还原成表格", () => { const tabs = "项目\t结果\t单位\t参考范围\t提示\n白细胞计数 WBC\t6.2\t10^9/L\t3.5-9.5"; const tabTable = formatTranscription(tabs); assert.match(tabTable, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/); assert.match(tabTable, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| 3\.5-9\.5 \| \|/); // qwen-vl-ocr 原生输出常用全角空格对齐列 const fullWidth = "项目\u3000\u3000结果\u3000\u3000单位\u3000\u3000参考范围\u3000\u3000提示\n白细胞计数 WBC\u3000\u30006.2\u3000\u300010^9/L"; const fullWidthTable = formatTranscription(fullWidth); assert.match(fullWidthTable, /\| 项目 \| 结果 \| 单位 \| 参考范围 \| 提示 \|/); assert.match(fullWidthTable, /\| 白细胞计数 WBC \| 6\.2 \| 10\^9\/L \| \| \|/); }); test("OCR 表格超出列并入最后一列时不重复已有单元格", () => { const formatted = formatTranscription("项目 结果 单位\n白细胞 6.2 10^9/L 偏高"); assert.equal( formatted, "| 项目 | 结果 | 单位 |\n| --- | --- | --- |\n| 白细胞 | 6.2 | 10^9/L 偏高 |" ); }); test("识别行只有坐标没有文字时视为未识别", () => { const result = parseVisionResult('[{"rotate_rect":[358,40,33,375,90]},{"rotate_rect":[268,96,25,453,90]}]'); assert.equal(result.text, ""); }); test("纯文本里的方括号不会被当成截断区间,且行与行保持独立", () => { const raw = "参考范围:白细胞 [3.5-9.5] 10^9/L\n红细胞 [4.3-5.8] 10^12/L\n其余检查结果见报告。"; const result = parseVisionResult(raw); assert.equal(result.data.text, raw); assert.match(result.text, /【原文转写】\n\n参考范围:白细胞 \[3\.5-9\.5\] 10\^9\/L\n\n红细胞/); assert.match(result.text, /红细胞 \[4\.3-5\.8\] 10\^12\/L\n\n其余检查结果见报告。/); }); test("排版不把「姓名 性别 年龄」误判成表格", () => { const text = [ "姓名:张三 性别:男 年龄:58岁", "检查号:CT20261031001 检查日期:2026-10-31 检查部位:胸部", "影像所见:双肺纹理清晰。", ].join("\n"); const formatted = formatTranscription(text); assert.doesNotMatch(formatted, /\| --- \|/); // 每行独立成段,避免 markdown 把多行合并 assert.equal(formatted.split("\n\n").length, 3); }); test("Qwen 视觉会话提交整图并解析为可读文本", async () => { let captured = null; const fetchImpl = async (url, options) => { captured = { url, options, body: JSON.parse(options.body) }; return { ok: true, json: async () => ({ choices: [{ message: { content: '{"docType":"lab_report","title":"血常规","text":"白细胞 6.2","quality":{"readable":true,"issues":[]}}', }, finish_reason: "stop", }], usage: { total_tokens: 120 }, }), }; }; const session = createQwenVisionSession({ apiKey: "test-key", model: "qwen-vl-ocr", mime: "image/png", width: 320, height: 240, fetchImpl, logger: silentLogger, }); session.push(Buffer.from([1, 2, 3])); const text = await session.finish(); assert.match(text, /【图像识别】检验报告|血常规/); assert.match(text, /【原文转写】\n\n白细胞 6\.2/); assert.equal(captured.body.model, "qwen-vl-ocr"); assert.equal(captured.body.temperature, 0); assert.match(captured.options.headers.Authorization, /Bearer test-key/); }); test("Qwen 视觉会话把 HTTP 错误转成可判断的失败", async () => { const fetchImpl = async () => ({ ok: false, status: 401, text: async () => "invalid api key", }); const session = createQwenVisionSession({ apiKey: "bad-key", fetchImpl, logger: silentLogger, }); session.push(Buffer.from("image")); await assert.rejects(() => session.finish(), /ocr http 401/); }); test("Qwen 视觉会话中止时取消进行中的请求", async () => { const fetchImpl = (url, options) => new Promise((resolve, reject) => { options.signal.addEventListener("abort", () => { const err = new Error("aborted"); err.name = "AbortError"; reject(err); }); }); const session = createQwenVisionSession({ apiKey: "test-key", fetchImpl, logger: silentLogger, }); session.push(Buffer.from("image")); const pending = session.finish(); session.abort(); await assert.rejects(() => pending, /ocr aborted/); });