zlei6/ComfyUI
1
0
Fork 0
mirror of https://github.com/Comfy-Org/ComfyUI.git synced 2026-08-25 02:20:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support anima lllite control models. (#14954)

Browse Source 
Put them in the models/model_patches folder. Use the new AnimaLLLiteApply node.
This commit is contained in:
comfyanonymous
2026-07-17 07:36:21 -07:00
committed by GitHub
parent 71b73e3b2b
commit 0f42ba5146
3 changed files with 373 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
comfy_extras/nodes_model_patch.py
View File

@@ -8,6 +8,7 @@ import comfy.ldm.common_dit
import comfy.latent_formats
import comfy.ldm.lumina.controlnet
import comfy.ldm.supir.supir_modules
import comfy.ldm.anima.lllite
from comfy.ldm.wan.model_multitalk import WanMultiTalkAttentionBlock, MultiTalkAudioProjModel
from comfy_api.latest import io
from comfy.ldm.supir.supir_patch import SUPIRPatch
@@ -236,10 +237,12 @@ class ModelPatchLoader:
def load_model_patch(self, name):
model_patch_path = folder_paths.get_full_path_or_raise("model_patches", name)
sd = comfy.utils.load_torch_file(model_patch_path, safe_load=True)
sd, metadata = comfy.utils.load_torch_file(model_patch_path, safe_load=True, return_metadata=True)
dtype = comfy.utils.weight_dtype(sd)
if 'controlnet_blocks.0.y_rms.weight' in sd:
if 'lllite_conditioning1.conv1.weight' in sd:
model = comfy.ldm.anima.lllite.AnimaLLLite(sd, metadata, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
elif 'controlnet_blocks.0.y_rms.weight' in sd:
additional_in_dim = sd["img_in.weight"].shape[1] - 64
model = QwenImageBlockWiseControlNet(additional_in_dim=additional_in_dim, device=comfy.model_management.unet_offload_device(), dtype=dtype, operations=comfy.ops.manual_cast)
elif 'feature_embedder.mid_layer_norm.bias' in sd:
@@ -296,6 +299,50 @@ class ModelPatchLoader:
return (model_patcher,)
class AnimaLLLiteApply:
@classmethod
def INPUT_TYPES(s):
return {"required": {"model": ("MODEL",),
"model_patch": ("MODEL_PATCH",),
"image": ("IMAGE",),
"strength": ("FLOAT", {"default": 1.0, "min": -10.0, "max": 10.0, "step": 0.01}),
"start_percent": ("FLOAT", {"default": 0.0, "min": 0.0, "max": 1.0, "step": 0.001}),
"end_percent": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.001}),
},
"optional": {"mask": ("MASK",),
}}
RETURN_TYPES = ("MODEL",)
FUNCTION = "apply_patch"
EXPERIMENTAL = True
CATEGORY = "model_patches/anima"
def apply_patch(self, model, model_patch, image, strength, start_percent, end_percent, mask=None):
image = image[..., :3]
if model_patch.model.cond_in_channels == 4 and mask is None:
mask = torch.zeros_like(image[..., 0])
elif model_patch.model.cond_in_channels != 4:
mask = None
model_sampling = model.get_model_object("model_sampling")
sigma_start = float(model_sampling.percent_to_sigma(start_percent))
sigma_end = float(model_sampling.percent_to_sigma(end_percent))
patch = comfy.ldm.anima.lllite.AnimaLLLitePatch(model_patch, image, mask, strength, sigma_start, sigma_end)
model_patched = model.clone()
model_patched.set_model_post_input_patch(patch)
model_patched.set_model_attn1_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(
patch,
{"q": "self_attn_q_proj", "k": "self_attn_k_proj", "v": "self_attn_v_proj"},
))
model_patched.set_model_attn2_patch(comfy.ldm.anima.lllite.AnimaLLLiteAttentionPatch(
patch,
{"q": "cross_attn_q_proj"},
))
model_patched.set_model_patch(comfy.ldm.anima.lllite.AnimaLLLiteMLPPatch(patch), "mlp_patch")
return (model_patched,)
class DiffSynthCnetPatch:
def __init__(self, model_patch, vae, image, strength, mask=None):
self.model_patch = model_patch
@@ -674,6 +721,7 @@ NODE_CLASS_MAPPINGS = {
"ZImageFunControlnet": ZImageFunControlnet,
"USOStyleReference": USOStyleReference,
"SUPIRApply": SUPIRApply,
"AnimaLLLiteApply": AnimaLLLiteApply,
}
NODE_DISPLAY_NAME_MAPPINGS = {
@@ -682,4 +730,5 @@ NODE_DISPLAY_NAME_MAPPINGS = {
"ZImageFunControlnet": "Apply Z-Image Fun ControlNet",
"USOStyleReference": "Apply USO Style Reference",
"SUPIRApply": "Apply SUPIR Patch",
"AnimaLLLiteApply": "Apply Anima LLLite",
}