mirror of
https://github.com/heygen-com/hyperframes.git
synced 2026-09-14 18:01:20 +08:00
3569c4830c
* feat(producer): add Rio-style residual-RMS check to regression harness
The existing audio comparison in the regression harness measures the
Pearson correlation between RMS envelopes of the rendered and snapshot
streams. That catches shape-level drift but is insensitive to level
shifts, phase offsets, or codec-quantization noise — two streams can
correlate >0.9 while differing audibly.
Rio's approach (rio/tests/checksum.py:compare_audio_files_ffmpeg) is
sample-level: subtract the snapshot from the rendered stream, run
`astats`, read the residual Overall RMS in dBFS. Identical streams
cancel to silence (-inf, or sub -90 dBFS for AAC-vs-AAC); anything
>= -50 dBFS is considered drift.
This commit adds the same check as an optional secondary gate:
- utils/audioRegression.ts: new `computeAudioResidualRmsDb()` that
spawns ffmpeg with the same filter graph Rio uses (aresample +
pan + volume=-1 + amix + astats) and returns the parsed Overall
RMS plus a pass/fail flag.
- utils/audioRegression.test.ts: 3 new tests covering identical
streams (-inf result), drifted streams (440Hz vs 880Hz sine),
and missing-audio-stream input.
- regression-harness.ts: optional `maxAudioResidualRmsDb` field in
meta.json. Default is undefined (skip the check) so legacy
fixtures aren't retroactively gated; new fixtures opt in by
setting a threshold (e.g. -50). Harness emits `residualRmsDb` in
the audio_comparison_complete JSON event and the pretty log line.
The existing correlation check stays in place; the new residual check
is independent. They measure complementary properties (shape vs
sample-cancellation) and both should hold for a faithful render.
* fix(producer): harden residual-RMS check (parser, duration guard, error surfacing)
Addresses review feedback on PR #882:
- Stateful astats parse: modern ffmpeg emits `Overall` on its own line
followed by per-stat lines, so the single-line `Overall RMS level dB:`
regex never fires on 6.x/7.x/8.x. Find the `Overall` header, take the
next `RMS level dB:` line. Single-line fallback preserved for 4.x.
- Pre-probe both inputs' audio durations and fail up-front if they differ
by >5 ms — `amix=duration=shortest` was silently masking trailing
audio differences.
- Surface ffmpeg/ffprobe spawn errors, signal kills, and non-zero exits
with a stderr tail. Previously every failure mode collapsed into
"NaN, fail" with no diagnostic.
- Extend `TestResult.audio` with `residualRmsDb` + `residualError`,
propagate to `audio-failures.json`.
- Fix `residualSuffix` formatter: NaN (real failure) was being rendered
as "-inf dBFS" (perfect match). Split the branch on `Number.isNaN`
separately from `Number.isFinite` and add an explicit error label.
105 lines
3.5 KiB
TypeScript
105 lines
3.5 KiB
TypeScript
import { spawnSync } from "node:child_process";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
|
import {
|
|
buildRmsEnvelope,
|
|
compareAudioEnvelopes,
|
|
computeAudioResidualRmsDb,
|
|
} from "./audioRegression.js";
|
|
|
|
describe("compareAudioEnvelopes", () => {
|
|
it("treats silent-vs-silent audio as a perfect match", () => {
|
|
const silentSamples = new Int16Array(4096);
|
|
|
|
const rendered = buildRmsEnvelope(silentSamples);
|
|
const snapshot = buildRmsEnvelope(silentSamples);
|
|
|
|
expect(compareAudioEnvelopes(rendered, snapshot, 120)).toEqual({
|
|
correlation: 1,
|
|
lagWindows: 0,
|
|
});
|
|
});
|
|
});
|
|
|
|
// Skip the spawn-based tests entirely on hosts without ffmpeg. The
|
|
// regression harness only runs in environments where ffmpeg is present
|
|
// (`Dockerfile.test`, dev boxes with apt's ffmpeg), so an absent ffmpeg
|
|
// is a developer-laptop fact, not a producer regression.
|
|
const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"], { encoding: "utf-8" }).status === 0;
|
|
|
|
describe.skipIf(!HAS_FFMPEG)("computeAudioResidualRmsDb", () => {
|
|
let tmp: string;
|
|
|
|
beforeAll(() => {
|
|
tmp = mkdtempSync(join(tmpdir(), "hf-audio-residual-test-"));
|
|
// Two test wavs: identical 1-second 440 Hz sine, and a 880 Hz sine
|
|
// that's audibly different from the 440 reference.
|
|
for (const [name, freq] of [
|
|
["sine-440-a.wav", 440],
|
|
["sine-440-b.wav", 440],
|
|
["sine-880.wav", 880],
|
|
] as const) {
|
|
const result = spawnSync(
|
|
"ffmpeg",
|
|
[
|
|
"-nostdin",
|
|
"-v",
|
|
"error",
|
|
"-f",
|
|
"lavfi",
|
|
"-i",
|
|
`sine=frequency=${freq}:duration=1:sample_rate=48000`,
|
|
"-ac",
|
|
"2",
|
|
"-c:a",
|
|
"pcm_s16le",
|
|
join(tmp, name),
|
|
],
|
|
{ encoding: "utf-8" },
|
|
);
|
|
if (result.status !== 0) {
|
|
throw new Error(`ffmpeg setup failed for ${name}: ${result.stderr}`);
|
|
}
|
|
}
|
|
});
|
|
|
|
afterAll(() => {
|
|
rmSync(tmp, { recursive: true, force: true });
|
|
});
|
|
|
|
it("returns -inf (or very low dBFS) for two identical streams", () => {
|
|
const result = computeAudioResidualRmsDb(
|
|
join(tmp, "sine-440-a.wav"),
|
|
join(tmp, "sine-440-b.wav"),
|
|
);
|
|
expect(result.ok).toBe(true);
|
|
// 440-vs-440 PCM cancels to silence; ffmpeg reports -inf which we
|
|
// normalize to NEGATIVE_INFINITY, OR a value well below -90 if the
|
|
// resampler introduces sub-bit-quantization noise.
|
|
expect(result.overallDb).toBeLessThan(-80);
|
|
});
|
|
|
|
it("fails when streams are audibly different (440 Hz vs 880 Hz)", () => {
|
|
const result = computeAudioResidualRmsDb(
|
|
join(tmp, "sine-440-a.wav"),
|
|
join(tmp, "sine-880.wav"),
|
|
);
|
|
expect(result.ok).toBe(false);
|
|
// The residual of two uncorrelated unit-amplitude sines is roughly
|
|
// the sum of both signals at near-full level — typically around
|
|
// -3 dBFS in this resampled-stereo configuration.
|
|
expect(result.overallDb).toBeGreaterThan(-30);
|
|
});
|
|
|
|
it("reports ok=false when an input has no audio stream", () => {
|
|
// A bare empty file: ffmpeg can't probe it, so the function reports
|
|
// a parse failure (ok=false, NaN). Callers decide whether to treat
|
|
// that as a pass (no-audio fixture) or a fail (audio expected).
|
|
const result = computeAudioResidualRmsDb("/dev/null", join(tmp, "sine-440-a.wav"));
|
|
expect(result.ok).toBe(false);
|
|
expect(Number.isNaN(result.overallDb)).toBe(true);
|
|
});
|
|
});
|