From df8cf21310dd9f534f75afb88b428e7a3f708693 Mon Sep 17 00:00:00 2001 From: 0xDevNinja Date: Thu, 9 Jul 2026 18:15:05 +0530 Subject: [PATCH] fix(green_screen): scale chromakey background to frame size, not 1x1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _process_chromakey built the composite background from a 1x1 lavfi color source and tried to size it with `[0:v]scale=iw:ih`. That scale is a no-op — iw/ih are the 1x1 source's own dimensions, and there is no cross-reference to the frame. FFmpeg's overlay then takes the size of its first input (the 1x1 background), so every processed frame is clipped to a single pixel. The exception fallback never runs because the primary command exits 0 (a valid 1x1 PNG), and _reconstruct_video upscales those 1x1 frames — producing a solid-color video with the keyed subject entirely gone. Total data loss for method="chromakey" (and method="auto" when it selects chromakey). Use scale2ref to resize the background to the actual frame dimensions before overlaying, so the keyed subject is composited at full resolution. Verified with ffmpeg: a 320x240 green frame with a red subject now produces a 320x240 output with the subject preserved and green replaced by the background, instead of a 1x1 (then upscaled solid-color) frame. --- tests/tools/test_green_screen_chromakey.py | 110 +++++++++++++++++++++ tools/video/green_screen_processor.py | 11 ++- 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_green_screen_chromakey.py diff --git a/tests/tools/test_green_screen_chromakey.py b/tests/tools/test_green_screen_chromakey.py new file mode 100644 index 00000000..1a0cb05e --- /dev/null +++ b/tests/tools/test_green_screen_chromakey.py @@ -0,0 +1,110 @@ +"""Regression tests: chromakey compositing must not collapse frames to 1x1. + +`_process_chromakey` built the background from a 1x1 lavfi color source and +tried to size it with `[0:v]scale=iw:ih` — a no-op, since iw/ih are the 1x1 +source's own dimensions. FFmpeg's `overlay` takes the size of its first input +(the 1x1 background), so every processed frame was clipped to a single pixel. +`_reconstruct_video` then upscaled those 1x1 frames, yielding a solid-color +video with the keyed subject entirely gone. The fix uses `scale2ref` to resize +the background to the actual frame dimensions. +""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from tools.video.green_screen_processor import GreenScreenProcessor # noqa: E402 + + +def test_chromakey_filter_scales_background_to_frame(): + """Offline: the built filtergraph must not leave a 1x1 background.""" + captured = [] + + def fake_run(self, cmd, **kwargs): + captured.append(list(cmd)) + + class _R: + returncode = 0 + stdout = "" + stderr = "" + + return _R() + + tool = GreenScreenProcessor() + orig = GreenScreenProcessor.run_command + GreenScreenProcessor.run_command = fake_run + try: + frames_dir = Path(PROJECT_ROOT) / "tests" / "tools" # any dir; glob may be empty + # Drive the filter build directly with a temp frame present. + import tempfile + + with tempfile.TemporaryDirectory() as td: + fd = Path(td) / "frames" + pd = Path(td) / "processed" + fd.mkdir() + pd.mkdir() + (fd / "frame_0000.png").write_bytes(b"stub") + tool._process_chromakey(fd, pd, "#0E172A", 1) + finally: + GreenScreenProcessor.run_command = orig + + ffmpeg_cmds = [c for c in captured if c and c[0] == "ffmpeg"] + assert ffmpeg_cmds, "no ffmpeg command built" + fc_idx = ffmpeg_cmds[0].index("-filter_complex") + fc = ffmpeg_cmds[0][fc_idx + 1] + assert "scale2ref" in fc, f"background not scaled to frame: {fc}" + assert "[0:v]scale=iw:ih[bg]" not in fc, "still using the 1x1 no-op scale" + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, + reason="ffmpeg/ffprobe required") +def test_chromakey_preserves_frame_size_and_keys(tmp_path): + """End-to-end: output keeps the source size and keys green -> background.""" + frames_dir = tmp_path / "frames" + processed_dir = tmp_path / "processed" + frames_dir.mkdir() + processed_dir.mkdir() + + frame = frames_dir / "frame_0000.png" + subprocess.run( + ["ffmpeg", "-y", "-f", "lavfi", "-i", + "color=c=0x00FF00:size=320x240,drawbox=x=100:y=80:w=120:h=80:color=red:t=fill", + "-frames:v", "1", str(frame)], + capture_output=True, check=True, timeout=60, + ) + + ok = GreenScreenProcessor()._process_chromakey(frames_dir, processed_dir, "#0E172A", 1) + assert ok + + out = processed_dir / "frame_0000.png" + assert out.exists() + + def _size(p): + r = subprocess.run( + ["ffprobe", "-v", "quiet", "-show_entries", "stream=width,height", + "-of", "csv=p=0", str(p)], + capture_output=True, text=True, timeout=30, + ) + w, h = r.stdout.strip().split(",") + return int(w), int(h) + + assert _size(out) == (320, 240), "frame collapsed instead of keeping source size" + + def _pixel(p, x, y): + r = subprocess.run( + ["ffmpeg", "-v", "quiet", "-i", str(p), "-vf", f"crop=1:1:{x}:{y}", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + capture_output=True, timeout=30, + ) + return r.stdout[:3] + + center = _pixel(out, 160, 120) # red box -> stays red + corner = _pixel(out, 10, 10) # was green -> keyed to dark background + assert center[0] > 150 and center[1] < 80, f"subject lost, center={center!r}" + assert corner[0] < 60 and corner[1] < 60, f"green not keyed, corner={corner!r}" diff --git a/tools/video/green_screen_processor.py b/tools/video/green_screen_processor.py index de074589..ab5e3ce6 100644 --- a/tools/video/green_screen_processor.py +++ b/tools/video/green_screen_processor.py @@ -465,9 +465,16 @@ class GreenScreenProcessor(BaseTool): "-i", str(frame), "-filter_complex", ( - f"[0:v]scale=iw:ih[bg];" + # The background is a 1x1 color source; it MUST be scaled up + # to the frame's dimensions. `scale=iw:ih` on [0:v] alone is a + # no-op (iw/ih are the 1x1 source's own size), and overlay + # takes the size of its FIRST input — so that leaves a 1x1 + # canvas and the keyed frame gets clipped to a single pixel, + # producing a solid-color video with the subject gone. + # scale2ref resizes [0:v] to match the frame [1:v] first. f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];" - f"[bg][fg]overlay=0:0" + f"[0:v][fg]scale2ref[bg][fg2];" + f"[bg][fg2]overlay=0:0" ), "-frames:v", "1", str(out_path),