From acd563bfa0a5957e62338b21b5cdb1c95802fb3e Mon Sep 17 00:00:00 2001 From: bymyself Date: Thu, 6 Aug 2026 17:35:51 -0700 Subject: [PATCH] Transcode when the output container cannot store the source codec Saving a VP8 WebM with format=auto named the file .mp4 and then stream copied VP8 into it, which PyAV rejects. Check the destination muxer's supported codecs before copying and fall back to re-encoding when any source stream cannot be stored. --- comfy_api/latest/_input_impl/video_types.py | 75 +++++++++++++----- tests-unit/comfy_api_test/video_types_test.py | 79 +++++++++++++++++++ 2 files changed, 135 insertions(+), 19 deletions(-) diff --git a/comfy_api/latest/_input_impl/video_types.py b/comfy_api/latest/_input_impl/video_types.py index d486cd838..ff5ce7edd 100644 --- a/comfy_api/latest/_input_impl/video_types.py +++ b/comfy_api/latest/_input_impl/video_types.py @@ -134,6 +134,17 @@ def write_output_metadata(container: InputContainer, output, metadata: dict | No output.metadata[key] = value if isinstance(value, str) else json.dumps(value) +def unsupported_remux_codecs(streams, output_container) -> list[str]: + supported = output_container.supported_codecs + return [ + stream.codec_context.name + for stream in streams + if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)) + and stream.codec_context is not None + and stream.codec_context.name not in supported + ] + + def video_output_config(path: str | io.BytesIO, format: VideoContainer, codec: VideoCodec) -> tuple[dict, VideoContainer, VideoCodec]: if isinstance(format, str): format = VideoContainer(format) @@ -592,28 +603,54 @@ class VideoFromFile(VideoInput): bit_depth = source_bit_depth return self._save_transcoded(container, path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth, crf=crf, color_space=color_space) - streams = container.streams - open_kwargs = get_open_write_kwargs(path, container_format, format) - with av.open(path, **open_kwargs) as output_container: - # Add metadata before writing any streams - write_output_metadata(container, output_container, metadata) + if self._save_remuxed(container, path, open_kwargs, metadata): + return - # Add streams to the new container. Streams with no codec context cannot be used as an output template. - stream_map = {} - for stream in streams: - if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)): - if stream.codec_context is None: - logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index) - continue - out_stream = output_container.add_stream_from_template(template=stream, opaque=True) - stream_map[stream] = out_stream + if bit_depth is None: + bit_depth = source_bit_depth + if isinstance(path, io.BytesIO): + path.seek(0) + path.truncate() + return self._save_transcoded(container, path, format=format, codec=codec, metadata=metadata, bit_depth=bit_depth, crf=crf, color_space=color_space) - # Write packets to the new container - for packet in container.demux(): - if packet.stream in stream_map and packet.dts is not None: - packet.stream = stream_map[packet.stream] - output_container.mux(packet) + def _save_remuxed( + self, + container: InputContainer, + path: str | io.BytesIO, + open_kwargs: dict, + metadata: dict | None, + ) -> bool: + streams = container.streams + with av.open(path, **open_kwargs) as output_container: + unsupported = unsupported_remux_codecs(streams, output_container) + if unsupported: + logging.info( + "Cannot copy %s into a %s container; re-encoding instead.", + ", ".join(sorted(set(unsupported))), + output_container.format.name, + ) + return False + + # Add metadata before writing any streams + write_output_metadata(container, output_container, metadata) + + # Add streams to the new container. Streams with no codec context cannot be used as an output template. + stream_map = {} + for stream in streams: + if isinstance(stream, (av.VideoStream, av.AudioStream, SubtitleStream)): + if stream.codec_context is None: + logging.warning("Skipping %s stream %d with unsupported codec", stream.type, stream.index) + continue + out_stream = output_container.add_stream_from_template(template=stream, opaque=True) + stream_map[stream] = out_stream + + # Write packets to the new container + for packet in container.demux(): + if packet.stream in stream_map and packet.dts is not None: + packet.stream = stream_map[packet.stream] + output_container.mux(packet) + return True def _save_transcoded( self, diff --git a/tests-unit/comfy_api_test/video_types_test.py b/tests-unit/comfy_api_test/video_types_test.py index f90913e06..02d484310 100644 --- a/tests-unit/comfy_api_test/video_types_test.py +++ b/tests-unit/comfy_api_test/video_types_test.py @@ -727,6 +727,85 @@ def test_save_to_mp4_writes_metadata_before_media(video_components, tmp_path): assert data.index(b"moov") < data.index(b"mdat") +def create_matroska_source( + tmp_path, video_codec="libvpx", audio_codec=None, frames=6, fps=10, container_format="webm" +): + path = str(tmp_path / f"source_{video_codec}.{container_format}") + with av.open(path, mode="w", format=container_format) as container: + video_stream = container.add_stream(video_codec, rate=fps) + video_stream.width = 32 + video_stream.height = 32 + video_stream.pix_fmt = "yuv420p" + audio_stream = None + if audio_codec is not None: + audio_stream = container.add_stream(audio_codec, rate=48000) + audio_stream.sample_rate = 48000 + + for i in range(frames): + frame = av.VideoFrame.from_ndarray( + torch.full((32, 32, 3), (i * 40) % 256, dtype=torch.uint8).numpy(), + format="rgb24", + ) + container.mux(video_stream.encode(frame.reformat(format="yuv420p"))) + if audio_stream is not None: + for offset in range(0, 48000 * frames // fps, 960): + audio_frame = av.AudioFrame.from_ndarray( + torch.zeros(1, 960, dtype=torch.int16).numpy(), format="s16", layout="mono" + ) + audio_frame.sample_rate = 48000 + audio_frame.pts = offset + container.mux(audio_stream.encode(audio_frame)) + for stream in [video_stream, audio_stream]: + if stream is not None: + container.mux(stream.encode(None)) + return path + + +def test_save_to_auto_transcodes_codec_the_container_cannot_store(tmp_path): + source = create_matroska_source(tmp_path) + destination = str(tmp_path / "saved.mp4") + + VideoFromFile(source).save_to(destination) + + with av.open(destination) as container: + assert container.format.name == "mov,mp4,m4a,3gp,3g2,mj2" + assert container.streams.video[0].codec.name == "h264" + + +def test_save_to_auto_transcodes_when_only_audio_is_unsupported(tmp_path): + source = create_matroska_source( + tmp_path, video_codec="libx264", audio_codec="pcm_u8", container_format="matroska" + ) + destination = str(tmp_path / "saved.mp4") + + VideoFromFile(source).save_to(destination) + + with av.open(destination) as container: + assert container.streams.video[0].codec.name == "h264" + assert container.streams.audio[0].codec.name == "aac" + + +def test_save_to_auto_still_remuxes_a_compatible_codec(tmp_path): + source = create_matroska_source(tmp_path, video_codec="libvpx-vp9") + destination = str(tmp_path / "saved.mp4") + + VideoFromFile(source).save_to(destination) + + with av.open(destination) as container: + assert container.streams.video[0].codec.name == "vp9" + + +def test_save_to_buffer_transcodes_codec_the_container_cannot_store(tmp_path): + source = create_matroska_source(tmp_path) + buffer = io.BytesIO() + + VideoFromFile(source).save_to(buffer, format=VideoContainer.MP4) + + buffer.seek(0) + with av.open(buffer) as container: + assert container.streams.video[0].codec.name == "h264" + + def create_transcode_source( width=64, height=64, frames=30, fps=30, audio_streams=1, undecodable_audio=0, rotation=False, container_format="mov", audio_codec="pcm_s16le",