mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-08-26 02:42:36 +08:00
Merge master into qa/video-edit-combined
# Conflicts: # comfy_api/latest/_input_impl/video_types.py
This commit is contained in:
@@ -486,7 +486,10 @@ class ImageCompositor(io.ComfyNode):
|
||||
node_id="ImageCompositor",
|
||||
display_name="Create Layered Image",
|
||||
category="image",
|
||||
search_aliases=["compositor", "composite", "layer", "layers", "layer editor", "psd"],
|
||||
is_experimental=True,
|
||||
# both flags on purpose: terminal compositor graphs must execute (the
|
||||
# editor needs a run to open), and cache hits must replay the layer UI
|
||||
is_output_node=True,
|
||||
has_intermediate_output=True,
|
||||
inputs=[
|
||||
@@ -605,7 +608,7 @@ class AddLayer(io.ComfyNode):
|
||||
options=list(_LAYER_MODES),
|
||||
default="normal",
|
||||
optional=True,
|
||||
tooltip="Initial blend mode.",
|
||||
tooltip="Initial blend mode, applied against the layers below. On the bottom layer over the default transparent background, non-normal modes produce transparency.",
|
||||
),
|
||||
io.Float.Input(
|
||||
"rotation",
|
||||
|
||||
@@ -591,7 +591,7 @@ class SamplerER_SDE(io.ComfyNode):
|
||||
inputs=[
|
||||
io.Combo.Input("solver_type", options=["ER-SDE", "Reverse-time SDE", "ODE"]),
|
||||
io.Int.Input("max_stage", default=3, min=1, max=3, advanced=True),
|
||||
io.Float.Input("eta", default=1.0, min=0.0, max=100.0, step=0.01, round=False, tooltip="Stochastic strength of reverse-time SDE.\nWhen eta=0, it reduces to deterministic ODE. This setting doesn't apply to ER-SDE solver type.", advanced=True),
|
||||
io.Float.Input("eta", default=1.0, min=0.0, max=10.0, step=0.01, round=False, tooltip="Stochastic strength of SDEs.\nWhen eta=0, they reduce to deterministic ODE.\nLarge eta may cause invalid outputs. If this occurs, try decreasing this value.", advanced=True),
|
||||
io.Float.Input("s_noise", default=1.0, min=0.0, max=100.0, step=0.01, round=False, advanced=True),
|
||||
],
|
||||
outputs=[io.Sampler.Output()]
|
||||
@@ -599,21 +599,35 @@ class SamplerER_SDE(io.ComfyNode):
|
||||
|
||||
@classmethod
|
||||
def execute(cls, solver_type, max_stage, eta, s_noise) -> io.NodeOutput:
|
||||
if solver_type == "ODE" or (solver_type == "Reverse-time SDE" and eta == 0):
|
||||
eta = 0
|
||||
s_noise = 0
|
||||
# Extend existing noise scalers phi(x) with eta-controlled noise scalers:
|
||||
# psi(x) = x**(1-eta) * phi(x)**eta
|
||||
# where eta is constant and directly scales the h^2(t) contribution.
|
||||
|
||||
def reverse_time_sde_noise_scaler(x):
|
||||
def er_sde_noise_scaler(x: torch.Tensor) -> torch.Tensor:
|
||||
return x * ((x ** 0.3).exp() + 10.0) ** eta
|
||||
|
||||
def reverse_time_sde_noise_scaler(x: torch.Tensor) -> torch.Tensor:
|
||||
return x ** (eta + 1)
|
||||
|
||||
if solver_type == "ER-SDE":
|
||||
# Use the default one in sample_er_sde()
|
||||
noise_scaler = None
|
||||
else:
|
||||
noise_scaler = reverse_time_sde_noise_scaler
|
||||
def ode_noise_scaler(x: torch.Tensor) -> torch.Tensor:
|
||||
return x
|
||||
|
||||
solver_scalers = {
|
||||
"ER-SDE": er_sde_noise_scaler,
|
||||
"Reverse-time SDE": reverse_time_sde_noise_scaler,
|
||||
"ODE": ode_noise_scaler,
|
||||
}
|
||||
|
||||
if solver_type == "ODE" or eta == 0:
|
||||
s_noise = 0.0
|
||||
solver_type = "ODE"
|
||||
noise_scaler = solver_scalers[solver_type]
|
||||
|
||||
sampler_name = "er_sde"
|
||||
sampler = comfy.samplers.ksampler(sampler_name, {"s_noise": s_noise, "noise_scaler": noise_scaler, "max_stage": max_stage})
|
||||
sampler = comfy.samplers.ksampler(
|
||||
sampler_name,
|
||||
{"s_noise": s_noise, "noise_scaler": noise_scaler, "max_stage": max_stage},
|
||||
)
|
||||
return io.NodeOutput(sampler)
|
||||
|
||||
get_sampler = execute
|
||||
@@ -704,15 +718,7 @@ class Noise_EmptyNoise:
|
||||
self.seed = 0
|
||||
|
||||
def generate_noise(self, input_latent):
|
||||
latent_image = input_latent["samples"]
|
||||
if latent_image.is_nested:
|
||||
tensors = latent_image.unbind()
|
||||
zeros = []
|
||||
for t in tensors:
|
||||
zeros.append(torch.zeros(t.shape, dtype=t.dtype, layout=t.layout, device="cpu"))
|
||||
return comfy.nested_tensor.NestedTensor(zeros)
|
||||
else:
|
||||
return torch.zeros(latent_image.shape, dtype=latent_image.dtype, layout=latent_image.layout, device="cpu")
|
||||
return comfy.sample.prepare_empty_noise(input_latent["samples"])
|
||||
|
||||
|
||||
class Noise_RandomNoise:
|
||||
|
||||
@@ -692,6 +692,7 @@ class ImageProcessingNode(io.ComfyNode):
|
||||
category=cls.category,
|
||||
description=cls.description,
|
||||
is_experimental=True,
|
||||
is_deprecated=cls.is_deprecated,
|
||||
is_input_list=is_group, # True for group, False for individual
|
||||
inputs=inputs,
|
||||
outputs=[
|
||||
@@ -861,9 +862,12 @@ class TextProcessingNode(io.ComfyNode):
|
||||
|
||||
return io.Schema(
|
||||
node_id=cls.node_id,
|
||||
search_aliases=cls.search_aliases,
|
||||
display_name=cls.display_name or cls.node_id,
|
||||
category="text",
|
||||
description=cls.description,
|
||||
is_experimental=True,
|
||||
is_deprecated=cls.is_deprecated,
|
||||
is_input_list=is_group, # True for group, False for individual
|
||||
inputs=inputs,
|
||||
outputs=[
|
||||
|
||||
@@ -2,11 +2,14 @@ import nodes
|
||||
import node_helpers
|
||||
import torch
|
||||
import torchaudio
|
||||
import comfy.ldm.lightricks.duration_head
|
||||
import comfy.model_management
|
||||
import comfy.model_sampling
|
||||
import comfy.samplers
|
||||
import comfy.utils
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import numpy as np
|
||||
import av
|
||||
from io import BytesIO
|
||||
@@ -934,6 +937,243 @@ class LTXVReferenceAudio(io.ComfyNode):
|
||||
return io.NodeOutput(m, positive, negative)
|
||||
|
||||
|
||||
class LTXVSpatioTemporalGuidance(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="LTXVSpatioTemporalGuidance",
|
||||
display_name="LTXV Spatio-Temporal Guidance (STG)",
|
||||
category="advanced/guidance",
|
||||
description="Runs one extra pass per step with the self-attention of the selected blocks degraded to a value-passthrough, "
|
||||
"then guides away from it - improving spatial detail and motion coherence.",
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Float.Input("scale", default=1.0, min=0.0, max=100.0, step=0.01, round=0.01),
|
||||
io.String.Input("blocks", default="29", tooltip="Comma-separated transformer block indices to perturb."),
|
||||
io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, advanced=True),
|
||||
io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, advanced=True),
|
||||
],
|
||||
outputs=[io.Model.Output()],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, scale, blocks, start_percent, end_percent) -> io.NodeOutput:
|
||||
block_set = frozenset(int(b) for b in re.findall(r"\d+", blocks))
|
||||
|
||||
m = model.clone()
|
||||
model_sampling = m.get_model_object("model_sampling")
|
||||
sigma_start = model_sampling.percent_to_sigma(start_percent)
|
||||
sigma_end = model_sampling.percent_to_sigma(end_percent)
|
||||
|
||||
def post_cfg_function(args):
|
||||
if scale == 0 or not block_set:
|
||||
return args["denoised"]
|
||||
|
||||
sigma_ = args["sigma"][0].item()
|
||||
if sigma_ > sigma_start or sigma_ < sigma_end:
|
||||
return args["denoised"]
|
||||
|
||||
cond_pred = args["cond_denoised"]
|
||||
cond = args["cond"]
|
||||
cfg_result = args["denoised"]
|
||||
x = args["input"]
|
||||
|
||||
model_options = args["model_options"].copy()
|
||||
transformer_options = model_options.get("transformer_options", {}).copy()
|
||||
transformer_options["stg_self_attn_blocks"] = block_set
|
||||
model_options["transformer_options"] = transformer_options
|
||||
|
||||
(perturbed,) = comfy.samplers.calc_cond_batch(args["model"], [cond], x, args["sigma"], model_options)
|
||||
|
||||
return cfg_result + (cond_pred - perturbed) * scale
|
||||
|
||||
m.set_model_sampler_post_cfg_function(post_cfg_function)
|
||||
return io.NodeOutput(m)
|
||||
|
||||
|
||||
class LTXVModalityGuidance(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="LTXVModalityGuidance",
|
||||
display_name="LTXV Modality Guidance (A/V coupling)",
|
||||
category="advanced/guidance",
|
||||
description="Cross-modal (audio-video) guidance for LTXV-AV. Runs one extra forward "
|
||||
"pass per step with the a2v/v2a cross-attention severed, then pushes the "
|
||||
"result toward the coupled prediction - strengthening audio-visual sync "
|
||||
"(e.g. lip-sync). Reference default modality_scale is 3.0. Stacks with the "
|
||||
"dual-CFG guider and STG. Set to 1.0 to disable (no extra pass).",
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Float.Input("modality_scale", default=3.0, min=1.0, max=100.0, step=0.1, round=0.01),
|
||||
io.Float.Input("start_percent", default=0.0, min=0.0, max=1.0, step=0.001, advanced=True),
|
||||
io.Float.Input("end_percent", default=1.0, min=0.0, max=1.0, step=0.001, advanced=True),
|
||||
],
|
||||
outputs=[io.Model.Output()],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, modality_scale, start_percent, end_percent) -> io.NodeOutput:
|
||||
m = model.clone()
|
||||
model_sampling = m.get_model_object("model_sampling")
|
||||
sigma_start = model_sampling.percent_to_sigma(start_percent)
|
||||
sigma_end = model_sampling.percent_to_sigma(end_percent)
|
||||
|
||||
def post_cfg_function(args):
|
||||
if math.isclose(modality_scale, 1.0):
|
||||
return args["denoised"]
|
||||
|
||||
sigma_ = args["sigma"][0].item()
|
||||
if sigma_ > sigma_start or sigma_ < sigma_end:
|
||||
return args["denoised"]
|
||||
|
||||
cond_pred = args["cond_denoised"]
|
||||
cond = args["cond"]
|
||||
cfg_result = args["denoised"]
|
||||
x = args["input"]
|
||||
|
||||
# Extra pass with audio-video cross-attention severed (both directions)
|
||||
model_options = args["model_options"].copy()
|
||||
transformer_options = model_options.get("transformer_options", {}).copy()
|
||||
transformer_options["a2v_cross_attn"] = False
|
||||
transformer_options["v2a_cross_attn"] = False
|
||||
model_options["transformer_options"] = transformer_options
|
||||
|
||||
(mod_pred,) = comfy.samplers.calc_cond_batch(
|
||||
args["model"], [cond], x, args["sigma"], model_options
|
||||
)
|
||||
|
||||
# (modality_scale - 1) * (cond - uncond_modality), per the reference guider.
|
||||
return cfg_result + (cond_pred - mod_pred) * (modality_scale - 1.0)
|
||||
|
||||
m.set_model_sampler_post_cfg_function(post_cfg_function)
|
||||
return io.NodeOutput(m)
|
||||
|
||||
|
||||
class Guider_LTXAVDualCFG(comfy.samplers.CFGGuider):
|
||||
"""CFG guider that applies separate guidance scales to the video and audio
|
||||
modalities of a packed LTXV-AV latent.
|
||||
"""
|
||||
|
||||
def set_conds(self, positive, negative):
|
||||
self.inner_set_conds({"positive": positive, "negative": negative})
|
||||
|
||||
def set_cfg(self, video_cfg, audio_cfg):
|
||||
self.video_cfg = video_cfg
|
||||
self.audio_cfg = audio_cfg
|
||||
self.cfg = max(video_cfg, audio_cfg)
|
||||
|
||||
def sample(self, noise, latent_image, *args, **kwargs):
|
||||
# Capture the video/audio split from the nested latent before it is packed.
|
||||
self._v_numel = None
|
||||
if getattr(latent_image, "is_nested", False):
|
||||
parts = latent_image.unbind()
|
||||
if len(parts) >= 2:
|
||||
self._v_numel = math.prod(parts[0].shape[1:])
|
||||
return super().sample(noise, latent_image, *args, **kwargs)
|
||||
|
||||
def predict_noise(self, x, timestep, model_options={}, seed=None):
|
||||
v = getattr(self, "_v_numel", None)
|
||||
if v is None or math.isclose(self.video_cfg, self.audio_cfg):
|
||||
# Not an AV latent, or equal scales: fall back to standard single-CFG.
|
||||
self.cfg = self.video_cfg
|
||||
return super().predict_noise(x, timestep, model_options, seed)
|
||||
|
||||
video_cfg, audio_cfg = self.video_cfg, self.audio_cfg
|
||||
|
||||
def dual_cfg(args):
|
||||
# Noise-space: cond = x - cond_pred, uncond = x - uncond_pred; the
|
||||
# returned tensor is subtracted from x by cfg_function.
|
||||
cond, uncond = args["cond"], args["uncond"]
|
||||
out = uncond + (cond - uncond) * video_cfg
|
||||
out[..., v:] = uncond[..., v:] + (cond[..., v:] - uncond[..., v:]) * audio_cfg
|
||||
return out
|
||||
|
||||
# disable_cfg1_optimization so the uncond pass always runs even if one of the two scales is 1.0.
|
||||
model_options = {**model_options, "sampler_cfg_function": dual_cfg, "disable_cfg1_optimization": True}
|
||||
return super().predict_noise(x, timestep, model_options, seed)
|
||||
|
||||
|
||||
class LTXVDualCFGGuider(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="LTXVDualCFGGuider",
|
||||
display_name="LTXV Dual CFG Guider",
|
||||
category="model/sampling/guiders",
|
||||
description="Separate CFG scales for the video and audio modalities of a packed LTXV-AV latent.",
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Conditioning.Input("positive"),
|
||||
io.Conditioning.Input("negative"),
|
||||
io.Float.Input("video_cfg", default=3.0, min=0.0, max=100.0, step=0.1, round=0.01),
|
||||
io.Float.Input("audio_cfg", default=7.0, min=0.0, max=100.0, step=0.1, round=0.01),
|
||||
],
|
||||
outputs=[io.Guider.Output()],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, positive, negative, video_cfg, audio_cfg) -> io.NodeOutput:
|
||||
guider = Guider_LTXAVDualCFG(model)
|
||||
guider.set_conds(positive, negative)
|
||||
guider.set_cfg(video_cfg, audio_cfg)
|
||||
return io.NodeOutput(guider)
|
||||
|
||||
|
||||
class LTXVDurationPredictor(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="LTXVDurationPredictor",
|
||||
display_name="LTXV Duration Predictor",
|
||||
category="conditioning/video_models",
|
||||
description="Predicts the natural shot duration for a prompt using the LTX 2.4 duration "
|
||||
"head (loaded with ModelPatchLoader), and snaps it to the VAE's 8k+1 frame grid.",
|
||||
search_aliases=["auto duration", "duration head", "num_frames"],
|
||||
inputs=[
|
||||
io.Model.Input("model"),
|
||||
io.Conditioning.Input("positive"),
|
||||
io.Custom("MODEL_PATCH").Input("duration_head",
|
||||
tooltip="LTX 2.4 duration head loaded with ModelPatchLoader."),
|
||||
io.Float.Input("frame_rate", default=24.0, min=1.0, max=120.0, step=0.01),
|
||||
io.Float.Input("min_seconds", default=1.0, min=0.5, max=120.0, step=0.1),
|
||||
io.Float.Input("max_seconds", default=20.0, min=0.5, max=120.0, step=0.1),
|
||||
],
|
||||
outputs=[
|
||||
io.Int.Output(display_name="num_frames"),
|
||||
io.Float.Output(display_name="seconds", tooltip="Raw (unclamped) predicted duration."),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, model, positive, duration_head, frame_rate, min_seconds, max_seconds) -> io.NodeOutput:
|
||||
dm = model.model.diffusion_model
|
||||
head = duration_head.model
|
||||
if not isinstance(head, comfy.ldm.lightricks.duration_head.DurationHead):
|
||||
raise ValueError("The connected model_patch is not an LTX duration head.")
|
||||
|
||||
context = positive[0][0]
|
||||
meta = positive[0][1]
|
||||
if context.shape[0] != 1:
|
||||
context = context[:1]
|
||||
|
||||
# Run the caption connectors exactly the way sampling does.
|
||||
comfy.model_management.load_models_gpu([model, duration_head])
|
||||
device = model.load_device
|
||||
head = head.to(device)
|
||||
with torch.no_grad():
|
||||
context = context.to(device=device, dtype=model.model.get_dtype_inference())
|
||||
processed = dm.preprocess_text_embeds(context, unprocessed=meta.get("unprocessed_ltxav_embeds", False))
|
||||
video_tokens = processed[..., :dm.cross_attention_dim].float()
|
||||
audio_tokens = processed[..., dm.cross_attention_dim:].float()
|
||||
seconds = float(head(video_tokens, audio_tokens)[0])
|
||||
|
||||
num_frames = comfy.ldm.lightricks.duration_head.seconds_to_num_frames(
|
||||
seconds, frame_rate, min_seconds, max_seconds)
|
||||
logging.info("LTXV duration head predicted %.2fs -> %d frames @ %.2f fps", seconds, num_frames, frame_rate)
|
||||
return io.NodeOutput(num_frames, seconds)
|
||||
|
||||
|
||||
class LtxvExtension(ComfyExtension):
|
||||
@override
|
||||
async def get_node_list(self) -> list[type[io.ComfyNode]]:
|
||||
@@ -951,6 +1191,10 @@ class LtxvExtension(ComfyExtension):
|
||||
LTXVConcatAVLatent,
|
||||
LTXVSeparateAVLatent,
|
||||
LTXVReferenceAudio,
|
||||
LTXVDualCFGGuider,
|
||||
LTXVModalityGuidance,
|
||||
LTXVSpatioTemporalGuidance,
|
||||
LTXVDurationPredictor,
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -173,7 +173,7 @@ class LTXAVTextEncoderLoader(io.ComfyNode):
|
||||
node_id="LTXAVTextEncoderLoader",
|
||||
display_name="Load LTXV Audio Text Encoder",
|
||||
category="model/loaders",
|
||||
description="Recipes:\nltxav: gemma 3 12B",
|
||||
description="Recipes:\nltxav: gemma 3 12B or matching gemma 4 model",
|
||||
inputs=[
|
||||
io.Combo.Input(
|
||||
"text_encoder",
|
||||
|
||||
@@ -20,6 +20,7 @@ import comfy.model_sampling
|
||||
import comfy.nested_tensor
|
||||
import comfy.utils
|
||||
import node_helpers
|
||||
from comfy.ldm.minimax.model import FRAME_PER_TOKEN, FRAME_RESCALE
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
|
||||
CANVAS_MULTIPLE = 32
|
||||
@@ -67,6 +68,16 @@ def _resize(image, width, height, crop):
|
||||
return samples.movedim(1, -1)
|
||||
|
||||
|
||||
def _encode_ref_audio(audio_vae, audio):
|
||||
waveform = audio["waveform"] # [B, C, L]
|
||||
sr = audio["sample_rate"]
|
||||
vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
|
||||
if sr != vae_sr:
|
||||
waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
|
||||
z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]
|
||||
return z, z.shape[-1]
|
||||
|
||||
|
||||
def _empty_av_latent(width, height, length, batch_size=1):
|
||||
frame_count, latent_t, audio_t = temporal_shape(length)
|
||||
video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],
|
||||
@@ -144,13 +155,87 @@ class MiniMaxH3ImageToVideo(io.ComfyNode):
|
||||
if keyframes:
|
||||
for kf in keyframes:
|
||||
kf["latent"] = vae.encode(kf.pop("image"))
|
||||
cond = node_helpers.conditioning_set_values(cond, {
|
||||
"minimax_keyframes": keyframes,
|
||||
"minimax_frame_count": frame_count,
|
||||
})
|
||||
cond = node_helpers.conditioning_set_values(cond, {"minimax_keyframes": keyframes})
|
||||
return io.NodeOutput(cond, latent)
|
||||
|
||||
|
||||
class MiniMaxH3AddGuide(io.ComfyNode):
|
||||
"""Anchor image and/or audio guides at an arbitrary pixel frame of the target video."""
|
||||
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="MiniMaxH3AddGuide",
|
||||
display_name="Add Guide for MiniMax H3",
|
||||
category="model/conditioning/minimax",
|
||||
description="Anchor an image, a short clip, audio, or a clip with its soundtrack at any frame of a MiniMax H3 video. Chain several nodes to anchor several frames.",
|
||||
inputs=[
|
||||
io.Conditioning.Input("positive"),
|
||||
io.Vae.Input("vae", optional=True, tooltip="Video VAE, needed when an image is connected."),
|
||||
io.Vae.Input("audio_vae", optional=True, tooltip="Audio VAE, needed when an audio is connected."),
|
||||
io.Latent.Input("latent"),
|
||||
io.Image.Input("image", optional=True, tooltip="Image or video frames to anchor. Multi-frame batches are anchored as a clip and cropped down to the model's valid clip lengths: 5, 22, 39... (17k + 5) frames. Batches shorter than 5 frames use only the first image."),
|
||||
io.Audio.Input("audio", optional=True,
|
||||
tooltip="Soundtrack to anchor starting at the same frame index, cropped to the video's remaining duration."),
|
||||
io.Int.Input("frame_idx", default=0, min=-9999, max=9999,
|
||||
tooltip="Frame index to anchor the image or the clip's first frame at. Negative values are counted from the end of the video."),
|
||||
],
|
||||
outputs=[io.Conditioning.Output(display_name="positive")],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, positive, latent, frame_idx, vae=None, audio_vae=None, image=None, audio=None) -> io.NodeOutput:
|
||||
samples = latent["samples"]
|
||||
if not samples.is_nested or len(samples.tensors) != 2 or samples.tensors[0].ndim != 5 or samples.tensors[0].shape[1] != 24:
|
||||
raise ValueError("MiniMaxH3AddGuide expects a MiniMax H3 AV latent")
|
||||
if image is None and audio is None:
|
||||
raise ValueError("MiniMaxH3AddGuide needs an image or an audio to anchor")
|
||||
video = samples.tensors[0]
|
||||
height = video.shape[3] * 16
|
||||
width = video.shape[4] * 16
|
||||
frame_count = sum(FRAME_PER_TOKEN[k % 5] for k in range(video.shape[2]))
|
||||
|
||||
guide_frames = 1
|
||||
if image is not None:
|
||||
if vae is None:
|
||||
raise ValueError("anchoring guide frames needs the vae input")
|
||||
guide_frames = image.shape[0]
|
||||
if guide_frames < 5:
|
||||
guide_frames = 1
|
||||
else:
|
||||
while guide_frames % 17 != 5:
|
||||
guide_frames -= 1
|
||||
|
||||
resolved_frame_index = frame_idx if frame_idx >= 0 else frame_count + frame_idx
|
||||
if resolved_frame_index < 0 or resolved_frame_index + guide_frames > frame_count:
|
||||
if guide_frames == 1:
|
||||
raise ValueError("frame_idx {} is outside the video's {} frames".format(frame_idx, frame_count))
|
||||
raise ValueError("a {} frame guide clip at frame_idx {} does not fit in the video's {} frames".format(
|
||||
guide_frames, frame_idx, frame_count))
|
||||
|
||||
keyframe = {"resolved_frame_index": resolved_frame_index}
|
||||
if image is not None:
|
||||
frames = _resize(image[:guide_frames], width, height, "center")
|
||||
keyframe["latent"] = vae.encode(frames)
|
||||
|
||||
if audio is not None:
|
||||
if audio_vae is None:
|
||||
raise ValueError("anchoring guide audio needs the audio_vae input")
|
||||
audio_latent, audio_rt = _encode_ref_audio(audio_vae, audio)
|
||||
# the streams share one time axis: FRAME_RESCALE per pixel frame, 1.0 per audio latent frame
|
||||
max_rt = math.floor(samples.tensors[1].shape[-1] - FRAME_RESCALE * resolved_frame_index)
|
||||
if max_rt < 1:
|
||||
raise ValueError("frame_idx {} is past the end of the video's audio track".format(frame_idx))
|
||||
if audio_rt > max_rt:
|
||||
audio_latent = audio_latent[..., :max_rt].clone()
|
||||
keyframe["audio_latent"] = audio_latent
|
||||
|
||||
keyframes = list(positive[0][1].get("minimax_keyframes", []))
|
||||
keyframes.append(keyframe)
|
||||
positive = node_helpers.conditioning_set_values(positive, {"minimax_keyframes": keyframes})
|
||||
return io.NodeOutput(positive)
|
||||
|
||||
|
||||
class MiniMaxH3ReferenceToVideo(io.ComfyNode):
|
||||
"""ref2va: prompt + reference images / videos / audio -> conditioning + AV latent.
|
||||
|
||||
@@ -197,16 +282,6 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
|
||||
outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _encode_ref_audio(audio_vae, audio):
|
||||
waveform = audio["waveform"] # [B, C, L]
|
||||
sr = audio["sample_rate"]
|
||||
vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
|
||||
if sr != vae_sr:
|
||||
waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
|
||||
z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]
|
||||
return z, z.shape[-1]
|
||||
|
||||
@classmethod
|
||||
def execute(cls, clip, vae, audio_vae, prompt, width, height, length, ref_image_size="match",
|
||||
ref_images=None, ref_videos=None, ref_video_audios=None, ref_audios=None) -> io.NodeOutput:
|
||||
@@ -254,7 +329,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
|
||||
z = vae.encode(frames)
|
||||
audio_latent, ref_audio_t = (None, 0)
|
||||
if soundtrack is not None:
|
||||
audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, soundtrack)
|
||||
audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, soundtrack)
|
||||
# the soundtrack gets its own <Audio j> label, emitted before <Video k>
|
||||
ref_items.append({"type": "audio"})
|
||||
# Qwen sees the video at 2 fps with timestamps
|
||||
@@ -269,7 +344,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
|
||||
for audio in (ref_audios or {}).values():
|
||||
if audio is None:
|
||||
continue
|
||||
audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, audio)
|
||||
audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, audio)
|
||||
ref_items.append({"type": "audio"})
|
||||
ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})
|
||||
|
||||
@@ -329,6 +404,7 @@ class MiniMaxH3Extension(ComfyExtension):
|
||||
return [
|
||||
EmptyMiniMaxH3LatentAV,
|
||||
MiniMaxH3ImageToVideo,
|
||||
MiniMaxH3AddGuide,
|
||||
MiniMaxH3ReferenceToVideo,
|
||||
MiniMaxH3SigmaShift,
|
||||
]
|
||||
|
||||
77
comfy_extras/nodes_minimax_music.py
Normal file
77
comfy_extras/nodes_minimax_music.py
Normal file
@@ -0,0 +1,77 @@
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
import comfy.model_management
|
||||
from comfy.ldm.minimax_music.ar import AUDIO_FRAMES_PER_SECOND, CFG_SCALE, CFG_TOP_K, C0_VOCAB_SIZE, MAX_AUDIO_FRAMES
|
||||
from comfy.ldm.minimax_music.dit import latent_length
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
|
||||
|
||||
class MiniMaxMusic3TextEncode(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="MiniMaxMusic3TextEncode",
|
||||
display_name="MiniMax Music3 Text Encode",
|
||||
category="model/conditioning/minimax music",
|
||||
description="Uses a MiniMax Music3 CLIP model to generate the acoustic conditioning sequence.",
|
||||
inputs=[
|
||||
io.Clip.Input("clip"),
|
||||
io.String.Input("caption", multiline=True, dynamic_prompts=True),
|
||||
io.String.Input("lyrics", multiline=True, dynamic_prompts=True),
|
||||
io.Int.Input("seed", default=0, min=0, max=0xffffffffffffffff, control_after_generate=True),
|
||||
io.Float.Input("max_duration", default=120.0, min=0.04, max=MAX_AUDIO_FRAMES / AUDIO_FRAMES_PER_SECOND, step=0.04, tooltip="Maximum duration in seconds; the model can end the song earlier."),
|
||||
io.Float.Input("cfg_scale", default=CFG_SCALE, min=0.0, max=100.0, step=0.1, round=0.01, advanced=True),
|
||||
io.Int.Input("top_k", default=CFG_TOP_K, min=1, max=C0_VOCAB_SIZE, advanced=True),
|
||||
],
|
||||
outputs=[
|
||||
io.Conditioning.Output(),
|
||||
io.Float.Output(display_name="seconds"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, clip, caption, lyrics, seed, max_duration, cfg_scale, top_k):
|
||||
max_audio_frames = min(MAX_AUDIO_FRAMES, max(1, round(max_duration * AUDIO_FRAMES_PER_SECOND)))
|
||||
tokens = clip.tokenize(caption, lyrics=lyrics, seed=seed, max_audio_frames=max_audio_frames, cfg_scale=cfg_scale, top_k=top_k)
|
||||
conditioning = clip.encode_from_tokens_scheduled(tokens)
|
||||
for cond in conditioning:
|
||||
hidden = cond[0]
|
||||
cond[1]["conditioning_scale"] = torch.ones((hidden.shape[0], 1, 1), device=hidden.device, dtype=hidden.dtype)
|
||||
return io.NodeOutput(conditioning, conditioning[0][0].shape[1] / AUDIO_FRAMES_PER_SECOND)
|
||||
|
||||
|
||||
class EmptyMiniMaxMusic3LatentAudio(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
return io.Schema(
|
||||
node_id="EmptyMiniMaxMusic3LatentAudio",
|
||||
display_name="Empty MiniMax Music3 Latent Audio",
|
||||
category="model/latent/minimax music",
|
||||
description="Creates an empty MiniMax Music3 audio latent for the requested duration.",
|
||||
inputs=[
|
||||
io.Float.Input("seconds", default=120.0, min=0.04, max=MAX_AUDIO_FRAMES / AUDIO_FRAMES_PER_SECOND, step=0.04),
|
||||
io.Int.Input("batch_size", default=1, min=1, max=4096),
|
||||
],
|
||||
outputs=[io.Latent.Output()],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, seconds, batch_size):
|
||||
audio_frames = min(MAX_AUDIO_FRAMES, max(1, round(seconds * AUDIO_FRAMES_PER_SECOND)))
|
||||
latent = torch.zeros(
|
||||
(batch_size, 128, latent_length(audio_frames)),
|
||||
device=comfy.model_management.intermediate_device(),
|
||||
dtype=comfy.model_management.intermediate_dtype(),
|
||||
)
|
||||
return io.NodeOutput({"samples": latent, "type": "audio", "downscale_ratio_temporal": 512})
|
||||
|
||||
|
||||
class MiniMaxMusic3Extension(ComfyExtension):
|
||||
@override
|
||||
async def get_node_list(self):
|
||||
return [MiniMaxMusic3TextEncode, EmptyMiniMaxMusic3LatentAudio]
|
||||
|
||||
|
||||
async def comfy_entrypoint():
|
||||
return MiniMaxMusic3Extension()
|
||||
@@ -1,6 +1,9 @@
|
||||
import logging
|
||||
|
||||
import comfy.sd
|
||||
import comfy.model_sampling
|
||||
import comfy.latent_formats
|
||||
import comfy.ldm.modules.attention
|
||||
import nodes
|
||||
import torch
|
||||
import node_helpers
|
||||
@@ -346,6 +349,39 @@ class ModelComputeDtype:
|
||||
return (m, )
|
||||
|
||||
|
||||
class ModelAttentionBackend:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
backends = ["pytorch attention"]
|
||||
if comfy.ldm.modules.attention.COMFY_KITCHEN_INT8_ATTENTION_IS_AVAILABLE:
|
||||
backends.append("comfy kitchen attention")
|
||||
return {"required": {"model": ("MODEL",),
|
||||
"attention": (backends,),
|
||||
}}
|
||||
|
||||
@classmethod
|
||||
def VALIDATE_INPUTS(s, attention):
|
||||
return True
|
||||
|
||||
RETURN_TYPES = ("MODEL",)
|
||||
FUNCTION = "patch"
|
||||
|
||||
CATEGORY = "model/patch"
|
||||
|
||||
def patch(self, model, attention):
|
||||
attention_name = {
|
||||
"comfy kitchen attention": "comfy_kitchen_int8",
|
||||
"pytorch attention": "pytorch",
|
||||
}.get(attention)
|
||||
attention_function = comfy.ldm.modules.attention.get_attention_function(attention_name, None)
|
||||
if attention_function is None:
|
||||
logging.warning("Attention backend '%s' is unavailable; using PyTorch attention.", attention)
|
||||
attention_function = comfy.ldm.modules.attention.get_attention_function("pytorch")
|
||||
m = model.clone()
|
||||
m.set_model_optimized_attention(attention_function)
|
||||
return (m, )
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"ModelSamplingDiscrete": ModelSamplingDiscrete,
|
||||
"ModelSamplingContinuousEDM": ModelSamplingContinuousEDM,
|
||||
@@ -357,4 +393,5 @@ NODE_CLASS_MAPPINGS = {
|
||||
"ModelNoiseScale": ModelNoiseScale,
|
||||
"RescaleCFG": RescaleCFG,
|
||||
"ModelComputeDtype": ModelComputeDtype,
|
||||
"ModelAttentionBackend": ModelAttentionBackend,
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import comfy.ldm.lumina.controlnet
|
||||
import comfy.ldm.supir.supir_modules
|
||||
import comfy.ldm.anima.lllite
|
||||
import comfy.ldm.wan.uni3c
|
||||
import comfy.ldm.lightricks.duration_head
|
||||
from comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel
|
||||
from comfy_api.latest import io
|
||||
from comfy.ldm.supir.supir_patch import SUPIRPatch
|
||||
@@ -296,6 +297,10 @@ class ModelPatchLoader:
|
||||
device=comfy.model_management.unet_offload_device(),
|
||||
dtype=dtype,
|
||||
operations=comfy.ops.manual_cast)
|
||||
elif any(k.endswith("duration_head.attention_pooler.query_tokens") for k in sd) or "attention_pooler.query_tokens" in sd:
|
||||
sd = comfy.ldm.lightricks.duration_head.normalize_state_dict(sd)
|
||||
sd = {k: v.float() for k, v in sd.items()} # tiny head, keep fp32
|
||||
model = comfy.ldm.lightricks.duration_head.DurationHead()
|
||||
elif "audio_proj.proj1.weight" in sd:
|
||||
model = MultiTalkModelPatch(
|
||||
audio_window=5, context_tokens=32, vae_scale=4,
|
||||
|
||||
@@ -29,7 +29,7 @@ class PreviewAny():
|
||||
value = str(source)
|
||||
elif source is not None:
|
||||
try:
|
||||
value = json.dumps(source, indent=4)
|
||||
value = json.dumps(source, indent=4, ensure_ascii=False)
|
||||
except Exception:
|
||||
try:
|
||||
value = str(source)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import re
|
||||
from comfy_api.latest import ComfyExtension, io
|
||||
from typing_extensions import override
|
||||
|
||||
@@ -152,6 +153,64 @@ You are a Creative Assistant writing concise, action-focused image-to-video prom
|
||||
Style: realistic - cinematic - The woman glances at her watch and smiles warmly. She speaks in a cheerful, friendly voice, "I think we're right on time!" In the background, a café barista prepares drinks at the counter. The barista calls out in a clear, upbeat tone, "Two cappuccinos ready!" The sound of the espresso machine hissing softly blends with gentle background chatter and the light clinking of cups on saucers.
|
||||
"""
|
||||
|
||||
LTX24_T2V_SYSTEM_PROMPT = """You are given a user's short text-to-video request. Write a single, highly detailed audio-visual caption describing the video that best fulfills that request, in the EXACT style of the training captions used for this video model. The generated video is scored against the user's ORIGINAL request, so preserve every element the user stated; expand faithfully into the full caption style without contradicting or dropping anything they asked for.
|
||||
|
||||
Match this captioning style precisely:
|
||||
|
||||
1. Begin immediately with the action or visual detail. Do NOT use "The scene opens…", "We see…", "There is…".
|
||||
|
||||
2. Objective, observable description only. Do not infer emotions or intentions — describe what is visible and audible (e.g. not "he looks sad" but "his eyebrows angle downward and his lips are pressed together").
|
||||
|
||||
3. Full visual detail: environment (materials, textures, lighting, colors), character appearance (clothing, posture, facial details), and the spatial positioning of all elements. When a human appears, identify them specifically (gendered terms when clearly implied; differentiate multiple people consistently) and describe visible physical attributes — apparent gender presentation, skin tone, estimated age group, hair color/length/style, build, clothing and accessories. Do not infer ethnicity, nationality, religion, or culture.
|
||||
|
||||
4. Precise motion and cinematic description. For every shot you MUST include, woven naturally into the prose (never as tags or labels):
|
||||
- Shot type (exactly one: extreme wide shot / wide shot / medium shot / medium close-up / close-up / extreme close-up)
|
||||
- Camera motion (always stated; if none, explicitly say the camera remains static). Camera movement is expected and good — match the user if they specified it, otherwise choose the treatment that best presents the requested scene.
|
||||
- Camera viewpoint relative to subject (front-facing / back-facing / side view / over-the-shoulder / top-down / low-angle / high-angle).
|
||||
Express these as flowing prose: "a medium shot frames…, captured from a front-facing angle as the camera slowly pans…". Never as "medium shot, static camera —".
|
||||
|
||||
5. Complete soundscape, integrated naturally: any dialogue (quote it exactly, in the original language), tone of voice, background music (type, mood, volume changes), and environmental sounds (footsteps, wind, traffic, animals). If the request implies sound, describe it plausibly.
|
||||
|
||||
6. Strict chronological, real-time flow using transitions like "Initially…", "A moment later…", "Simultaneously…". Keep every stated action in motion.
|
||||
|
||||
7. One single continuous paragraph. No bullet points, no section headers, no labels like "Audio:" or "Visual:". Exhaustive and lossless — include background elements, subtle movements, lighting, secondary sounds — detailed enough to reconstruct the scene. Aim for a rich, complete paragraph (roughly 150–220 words).
|
||||
|
||||
If the user wrote in another language, produce the English caption of the same content. Output ONLY the caption text — no JSON, no preamble.
|
||||
|
||||
AESTHETIC QUALITY (in addition to the above, without breaking the objective caption style): render the described scene with strong visual production value — cinematic, film-grade color and contrast, beautiful natural lighting, crisp fine detail and texture, pleasing composition and depth. Weave these quality descriptors naturally into the same observable prose (e.g. "warm cinematic lighting", "richly saturated film-grade color", "crisp high-resolution detail") — describe how the exact requested scene LOOKS at its most visually striking, never adding new objects or actions. Keep everything else (framing triple, soundscape, chronological single paragraph, faithfulness) exactly as specified.
|
||||
"""
|
||||
|
||||
|
||||
LTX24_I2V_SYSTEM_PROMPT = """You are given a REFERENCE IMAGE (the exact first frame of the video) and a user's short image-to-video request. Write a single, highly detailed audio-visual caption describing the video that BEGINS from this exact reference image and best fulfills that request, in the EXACT style of the training captions used for this video model. The generated video is scored against the user's ORIGINAL request, so preserve every element the user stated; expand faithfully into the full caption style without contradicting or dropping anything they asked for.
|
||||
|
||||
FIRST-FRAME / IMAGE GROUNDING (do this first): the opening of your caption must match the reference image exactly — same subject(s), identity, appearance, clothing, setting, lighting, and composition as shown. The video starts on this frame; describe it faithfully, then narrate chronologically as the user's requested action unfolds from it. Never contradict, replace, or invent things not consistent with the image. Single continuous take — no hard cuts.
|
||||
|
||||
Match this captioning style precisely:
|
||||
|
||||
1. Begin immediately with the action or visual detail. Do NOT use "The scene opens…", "We see…", "There is…".
|
||||
|
||||
2. Objective, observable description only. Do not infer emotions or intentions — describe what is visible and audible (e.g. not "he looks sad" but "his eyebrows angle downward and his lips are pressed together").
|
||||
|
||||
3. Full visual detail: environment (materials, textures, lighting, colors), character appearance (clothing, posture, facial details), and the spatial positioning of all elements — grounded in and consistent with the reference image. When a human appears, identify them specifically (gendered terms when clearly implied; differentiate multiple people consistently) and describe visible physical attributes — apparent gender presentation, skin tone, estimated age group, hair color/length/style, build, clothing and accessories. Do not infer ethnicity, nationality, religion, or culture.
|
||||
|
||||
4. Precise motion and cinematic description. For every shot you MUST include, woven naturally into the prose (never as tags or labels):
|
||||
- Shot type (exactly one: extreme wide shot / wide shot / medium shot / medium close-up / close-up / extreme close-up) — consistent with how the reference image is framed at the start.
|
||||
- Camera motion (always stated; if none, explicitly say the camera remains static). Camera movement is expected and good — match the user if they specified it, otherwise choose the treatment that best presents the requested scene starting from this frame.
|
||||
- Camera viewpoint relative to subject (front-facing / back-facing / side view / over-the-shoulder / top-down / low-angle / high-angle) — matching the reference image's viewpoint at the opening.
|
||||
Express these as flowing prose: "a medium shot frames…, captured from a front-facing angle as the camera slowly pans…". Never as "medium shot, static camera —".
|
||||
|
||||
5. Complete soundscape, integrated naturally: any dialogue (quote it exactly, in the original language), tone of voice, background music (type, mood, volume changes), and environmental sounds (footsteps, wind, traffic, animals). If the request implies sound, describe it plausibly.
|
||||
|
||||
6. Strict chronological, real-time flow using transitions like "Initially…", "A moment later…", "Simultaneously…". Keep the user's requested motion/action central and in motion throughout.
|
||||
|
||||
7. One single continuous paragraph. No bullet points, no section headers, no labels like "Audio:" or "Visual:". Exhaustive and lossless — include background elements, subtle movements, lighting, secondary sounds — detailed enough to reconstruct the scene. Aim for a rich, complete paragraph (roughly 150–220 words).
|
||||
|
||||
If the user wrote in another language, produce the English caption of the same content. Output ONLY the caption text — no JSON, no preamble.
|
||||
|
||||
AESTHETIC QUALITY (in addition to the above, without breaking the objective caption style or contradicting the reference image): render the described scene with strong visual production value — cinematic, film-grade color and contrast, beautiful natural lighting, crisp fine detail and texture, pleasing composition and depth. Weave these quality descriptors naturally into the same observable prose (e.g. "warm cinematic lighting", "richly saturated film-grade color", "crisp high-resolution detail") — describe how the exact requested scene, starting from this frame, LOOKS at its most visually striking, never adding new objects or actions and never contradicting the first frame. Keep everything else (first-frame grounding, framing triple, soundscape, chronological single paragraph, faithfulness) exactly as specified.
|
||||
"""
|
||||
|
||||
|
||||
class TextGenerateLTX2Prompt(TextGenerate):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
@@ -167,11 +226,40 @@ class TextGenerateLTX2Prompt(TextGenerate):
|
||||
|
||||
@classmethod
|
||||
def execute(cls, clip, prompt, max_length, sampling_mode, image=None, thinking=False, use_default_template=True, video=None, audio=None) -> io.NodeOutput:
|
||||
if image is None:
|
||||
formatted_prompt = f"<start_of_turn>system\n{LTX2_T2V_SYSTEM_PROMPT.strip()}<end_of_turn>\n<start_of_turn>user\nUser Raw Input Prompt: {prompt}.<end_of_turn>\n<start_of_turn>model\n"
|
||||
# Gemma 3 and Gemma 4 use different chat-turn markers and image tokens.
|
||||
# The Gemma 4 text encoder is the LTX 2.4 path; Gemma 3 is LTX 2.0.
|
||||
is_gemma4 = "gemma4" in getattr(clip.tokenizer, "clip_name", "")
|
||||
|
||||
if is_gemma4:
|
||||
if image is not None:
|
||||
system = LTX24_I2V_SYSTEM_PROMPT.strip()
|
||||
user_text = f"User Raw Input Prompt: {prompt}."
|
||||
else:
|
||||
system = LTX24_T2V_SYSTEM_PROMPT.strip()
|
||||
user_text = f"user prompt: {prompt}"
|
||||
think_prefix = "<|think|>\n" if thinking else ""
|
||||
model_open = "" if thinking else "<|channel>final\n"
|
||||
media = "<|image><|image|><image|>\n\n" if image is not None else ""
|
||||
formatted_prompt = (
|
||||
f"<|turn>system\n{think_prefix}{system}<turn|>\n"
|
||||
f"<|turn>user\n{media}{user_text}<turn|>\n"
|
||||
f"<|turn>model\n{model_open}"
|
||||
)
|
||||
else:
|
||||
formatted_prompt = f"<start_of_turn>system\n{LTX2_I2V_SYSTEM_PROMPT.strip()}<end_of_turn>\n<start_of_turn>user\n\n<image_soft_token>\n\nUser Raw Input Prompt: {prompt}.<end_of_turn>\n<start_of_turn>model\n"
|
||||
return super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio)
|
||||
system = (LTX2_I2V_SYSTEM_PROMPT if image is not None else LTX2_T2V_SYSTEM_PROMPT).strip()
|
||||
media = "\n<image_soft_token>\n" if image is not None else ""
|
||||
formatted_prompt = (
|
||||
f"<start_of_turn>system\n{system}<end_of_turn>\n"
|
||||
f"<start_of_turn>user\n{media}\nUser Raw Input Prompt: {prompt}.<end_of_turn>\n"
|
||||
f"<start_of_turn>model\n"
|
||||
)
|
||||
|
||||
out = super().execute(clip, formatted_prompt, max_length, sampling_mode, image=image, thinking=thinking, use_default_template=use_default_template, video=video, audio=audio)
|
||||
|
||||
# Drop reasoning, including a block left unclosed by max_length. Both system prompts ask
|
||||
# for the original prompt back when there is nothing to give; empty conditions on nothing.
|
||||
text = re.sub(r"<think>.*?(?:</think>|$)", "", out.args[0], flags=re.DOTALL).strip()
|
||||
return io.NodeOutput(text or prompt)
|
||||
|
||||
|
||||
class TextgenExtension(ComfyExtension):
|
||||
|
||||
@@ -72,7 +72,7 @@ class ImageUpscaleWithModel(io.ComfyNode):
|
||||
|
||||
memory_required = (512 * 512 * 3) * image.element_size() * max(upscale_model.scale, 1.0) * 384.0 #The 384.0 is an estimate of how much some of these models take, TODO: make it more accurate
|
||||
memory_required += image.nelement() * image.element_size()
|
||||
model_management.load_models_gpu([upscale_model.patcher], memory_required=memory_required)
|
||||
model_management.load_models_gpu([upscale_model.patcher], memory_required=memory_required, force_full_load=True)
|
||||
|
||||
in_img = image.movedim(-1,-3).to(device)
|
||||
|
||||
|
||||
@@ -72,6 +72,70 @@ class SaveWEBM(io.ComfyNode):
|
||||
|
||||
return io.NodeOutput(images, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)]))
|
||||
|
||||
def _save_video_codec_input(supported_codecs: list[str], *, optional=False, hidden=False):
|
||||
codec_options = []
|
||||
if "auto" in supported_codecs:
|
||||
codec_options.append(io.DynamicCombo.Option("auto", []))
|
||||
if "h264" in supported_codecs:
|
||||
codec_options.append(
|
||||
io.DynamicCombo.Option(
|
||||
"h264",
|
||||
[
|
||||
io.DynamicCombo.Input(
|
||||
"encoding",
|
||||
display_name="encoding mode",
|
||||
options=[
|
||||
io.DynamicCombo.Option("auto", []),
|
||||
io.DynamicCombo.Option(
|
||||
"re-encode",
|
||||
[io.Float.Input("crf", default=23.0, min=0.0, max=51.0, step=1.0, tooltip="Lower values produce higher quality and larger files.")],
|
||||
),
|
||||
],
|
||||
optional=True,
|
||||
tooltip="Automatic preserves compatible H.264 streams. Re-encode applies a custom CRF.",
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
if "av1" in supported_codecs:
|
||||
codec_options.append(
|
||||
io.DynamicCombo.Option(
|
||||
"av1",
|
||||
[
|
||||
io.DynamicCombo.Input(
|
||||
"encoding",
|
||||
display_name="encoding mode",
|
||||
options=[
|
||||
io.DynamicCombo.Option("auto", []),
|
||||
io.DynamicCombo.Option(
|
||||
"re-encode",
|
||||
[
|
||||
io.Float.Input("crf", default=30.0, min=0.0, max=63.0, step=1.0, tooltip="Lower values produce higher quality and larger files."),
|
||||
io.Combo.Input(
|
||||
"color_space",
|
||||
options=["auto", "sRGB", "HDR", "HDR PQ"],
|
||||
default="auto",
|
||||
display_name="color space",
|
||||
tooltip="Auto uses sRGB for videos created from images and preserves recognized colors on loaded videos. sRGB writes SDR BT.709/sRGB. HDR writes 10-bit BT.2020/HLG; HDR PQ writes BT.2020/PQ. Other input pixels must already use the selected color space.",
|
||||
),
|
||||
],
|
||||
),
|
||||
],
|
||||
optional=True,
|
||||
tooltip="Automatic preserves compatible AV1 streams. Re-encode applies custom encoding options.",
|
||||
),
|
||||
],
|
||||
)
|
||||
)
|
||||
return io.DynamicCombo.Input(
|
||||
"codec",
|
||||
options=codec_options,
|
||||
optional=optional,
|
||||
tooltip="The output video codec. Auto preserves a compatible source stream. H.264 re-encoding supports SDR; AV1 re-encoding supports SDR, HDR (HLG), and HDR PQ.",
|
||||
extra_dict={"hidden": True} if hidden else None,
|
||||
)
|
||||
|
||||
|
||||
class SaveVideo(io.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
@@ -85,42 +149,37 @@ class SaveVideo(io.ComfyNode):
|
||||
inputs=[
|
||||
io.Video.Input("video", tooltip="The video to save."),
|
||||
io.String.Input("filename_prefix", default="video/ComfyUI", tooltip="The prefix for the file to save. This may include formatting information such as %date:yyyy-MM-dd% or %Empty Latent Image.width% to include values from nodes."),
|
||||
io.Combo.Input("format", options=Types.VideoContainer.as_input(), default="auto", tooltip="The format to save the video as."),
|
||||
io.DynamicCombo.Input(
|
||||
"codec",
|
||||
"format",
|
||||
options=[
|
||||
io.DynamicCombo.Option("auto", []),
|
||||
io.DynamicCombo.Option(
|
||||
"h264",
|
||||
[
|
||||
io.DynamicCombo.Input(
|
||||
"encoding",
|
||||
display_name="encoding mode",
|
||||
options=[
|
||||
io.DynamicCombo.Option("auto", []),
|
||||
io.DynamicCombo.Option(
|
||||
"re-encode",
|
||||
[io.Float.Input("crf", default=23.0, min=0.0, max=51.0, step=1.0, tooltip="Lower values produce higher quality and larger files.")],
|
||||
),
|
||||
],
|
||||
optional=True,
|
||||
tooltip="Automatic preserves compatible H.264 streams. Re-encode applies a custom CRF.",
|
||||
),
|
||||
],
|
||||
),
|
||||
io.DynamicCombo.Option("auto", [_save_video_codec_input(["auto", "h264", "av1"])]),
|
||||
io.DynamicCombo.Option("mp4", [_save_video_codec_input(["auto", "h264", "av1"])]),
|
||||
io.DynamicCombo.Option("mkv", [_save_video_codec_input(["auto", "h264", "av1"])]),
|
||||
io.DynamicCombo.Option("webm", [_save_video_codec_input(["auto", "av1"])]),
|
||||
],
|
||||
tooltip="The codec to use for the video.",
|
||||
tooltip="The output container. Auto preserves the source container when possible; MP4, MKV, and WebM select a specific container.",
|
||||
),
|
||||
_save_video_codec_input(["auto", "h264", "av1"], optional=True, hidden=True),
|
||||
],
|
||||
hidden=[io.Hidden.prompt, io.Hidden.extra_pnginfo],
|
||||
is_output_node=True,
|
||||
outputs=[io.Video.Output("video")],
|
||||
outputs=[io.Video.Output("video", tooltip="The input video, unchanged.")],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, video: Input.Video, filename_prefix, format: str, codec: io.DynamicCombo.Type) -> io.NodeOutput:
|
||||
def execute(cls, video: Input.Video, filename_prefix, format: io.DynamicCombo.Type | str, codec: io.DynamicCombo.Type | None = None) -> io.NodeOutput:
|
||||
if isinstance(format, dict):
|
||||
format_name = format["format"]
|
||||
codec = format.get("codec") or codec
|
||||
else:
|
||||
format_name = format
|
||||
if codec is None:
|
||||
codec = {"codec": "auto"}
|
||||
codec_name = codec["codec"]
|
||||
encoding = codec.get("encoding") or {}
|
||||
color_space = encoding.get("color_space")
|
||||
if color_space == "auto":
|
||||
color_space = None
|
||||
width, height = video.get_dimensions()
|
||||
full_output_folder, filename, counter, subfolder, filename_prefix = folder_paths.get_save_image_path(
|
||||
filename_prefix,
|
||||
@@ -137,13 +196,14 @@ class SaveVideo(io.ComfyNode):
|
||||
metadata["prompt"] = cls.hidden.prompt
|
||||
if len(metadata) > 0:
|
||||
saved_metadata = metadata
|
||||
file = f"{filename}_{counter:05}_.{Types.VideoContainer.get_extension(format)}"
|
||||
file = f"{filename}_{counter:05}_.{Types.VideoContainer.get_extension(format_name)}"
|
||||
video.save_to(
|
||||
os.path.join(full_output_folder, file),
|
||||
format=Types.VideoContainer(format),
|
||||
codec=codec_name,
|
||||
format=Types.VideoContainer(format_name),
|
||||
codec=Types.VideoCodec(codec_name),
|
||||
metadata=saved_metadata,
|
||||
crf=encoding.get("crf"),
|
||||
color_space=color_space,
|
||||
)
|
||||
|
||||
return io.NodeOutput(video, ui=ui.PreviewVideo([ui.SavedResult(file, subfolder, io.FolderType.output)]))
|
||||
@@ -199,7 +259,7 @@ class GetVideoComponents(io.ComfyNode):
|
||||
search_aliases=["extract frames", "split video", "video to images", "demux"],
|
||||
display_name="Get Video Components",
|
||||
category="video",
|
||||
description="Extracts all components from a video: frames, audio, framerate, and bit depth.",
|
||||
description="Extracts video frames, audio, frame rate, bit depth, and color space.",
|
||||
inputs=[
|
||||
io.Video.Input("video", tooltip="The video to extract components from."),
|
||||
],
|
||||
@@ -208,13 +268,20 @@ class GetVideoComponents(io.ComfyNode):
|
||||
io.Audio.Output(display_name="audio"),
|
||||
io.Float.Output(display_name="fps"),
|
||||
io.Int.Output(display_name="bit_depth"),
|
||||
io.Combo.Output(display_name="color_space"),
|
||||
],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def execute(cls, video: Input.Video) -> io.NodeOutput:
|
||||
components = video.get_components()
|
||||
return io.NodeOutput(components.images, components.audio, float(components.frame_rate), video.get_bit_depth())
|
||||
return io.NodeOutput(
|
||||
components.images,
|
||||
components.audio,
|
||||
float(components.frame_rate),
|
||||
video.get_bit_depth(),
|
||||
video.get_color_space(),
|
||||
)
|
||||
|
||||
|
||||
class LoadVideo(io.ComfyNode):
|
||||
|
||||
Reference in New Issue
Block a user