mirror of
https://github.com/Comfy-Org/ComfyUI.git
synced 2026-09-08 18:16:30 +08:00
[Partner Nodes] feat(MiniMax): add H3 Max option to H3 text-to-video and first-last-frame nodes (#16025)
* [Partner Nodes] feat(MiniMax): add H3 Max option to H3 text-to-video and first-last-frame nodes Signed-off-by: Alexander Piskun <bigcat88@icloud.com> * [Partner Nodes] fix(downscale): never make a downscaled image more elongated than its source Signed-off-by: Alexander Piskun <bigcat88@icloud.com> * [Partner Nodes] fix(M3-Max): add check fpr max prompt length Signed-off-by: Alexander Piskun <bigcat88@icloud.com> * [Partner Nodes] chore(Minimax): correct route name Signed-off-by: bigcat88 <bigcat88@icloud.com> --------- Signed-off-by: Alexander Piskun <bigcat88@icloud.com> Signed-off-by: bigcat88 <bigcat88@icloud.com> Co-authored-by: Alexander Piskun <13381981+bigcat88@users.noreply.github.com> Co-authored-by: Alexander Piskun <bigcat88@icloud.com>
This commit is contained in:
Purz
@@ -215,3 +215,35 @@ class Hailuo03Task(BaseModel):
|
||||
|
||||
class Hailuo03TaskQueryResponse(BaseModel):
|
||||
task: Hailuo03Task = Field(...)
|
||||
|
||||
|
||||
class Hailuo03MaxTaskCreationResponse(BaseModel):
|
||||
request_id: str = Field(...)
|
||||
status: str | None = Field(None)
|
||||
|
||||
|
||||
class Hailuo03MaxTaskStatusResponse(BaseModel):
|
||||
status: str | None = Field(None)
|
||||
|
||||
|
||||
class Hailuo03MaxVideoFile(BaseModel):
|
||||
url: str = Field(...)
|
||||
content_type: str | None = Field(None)
|
||||
file_name: str | None = Field(None)
|
||||
file_size: int | None = Field(None)
|
||||
|
||||
|
||||
class Hailuo03MaxVideoRequest(BaseModel):
|
||||
prompt: str = Field(...)
|
||||
duration: int = Field(..., ge=5, le=15)
|
||||
resolution: str = Field(...)
|
||||
prompt_expansion_mode: str = Field(...)
|
||||
seed: int = Field(...)
|
||||
aspect_ratio: str | None = Field(None)
|
||||
image_url: str | None = Field(None)
|
||||
end_image_url: str | None = Field(None)
|
||||
|
||||
|
||||
class Hailuo03MaxVideoResult(BaseModel):
|
||||
video: Hailuo03MaxVideoFile = Field(...)
|
||||
expanded_prompt: str | None = Field(None)
|
||||
|
||||
@@ -10,6 +10,10 @@ from comfy_api_nodes.apis.minimax import (
|
||||
Hailuo03ContextIRRequest,
|
||||
Hailuo03ImageContent,
|
||||
Hailuo03ImageContentUrl,
|
||||
Hailuo03MaxTaskCreationResponse,
|
||||
Hailuo03MaxTaskStatusResponse,
|
||||
Hailuo03MaxVideoRequest,
|
||||
Hailuo03MaxVideoResult,
|
||||
Hailuo03RegenerationRequest,
|
||||
Hailuo03TaskCreationRequest,
|
||||
Hailuo03TaskCreationResponse,
|
||||
@@ -461,6 +465,10 @@ HAILUO_03_FAILED_STATUSES = ["failed", "cancelled", "expired"]
|
||||
HAILUO_03_CONTEXT_IR_ENDPOINT = "/proxy/minimax/v2/h3_context_ir"
|
||||
HAILUO_03_REGENERATION_ENDPOINT = "/proxy/minimax/v2/video_regeneration"
|
||||
|
||||
HAILUO_03_MAX_MODEL = "MiniMax H3 Max"
|
||||
HAILUO_03_MAX_ENDPOINT = "/proxy/fal/minimax/h3-max"
|
||||
HAILUO_03_MAX_PROMPT_MAX_LENGTH = 50000
|
||||
|
||||
|
||||
def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True):
|
||||
inputs = [
|
||||
@@ -541,6 +549,81 @@ async def _hailuo03_run_task(
|
||||
return IO.NodeOutput(await download_url_to_video_output(video_url))
|
||||
|
||||
|
||||
def _hailuo03_max_model_inputs(include_ratio: bool = True):
|
||||
inputs = [
|
||||
IO.String.Input(
|
||||
"prompt",
|
||||
multiline=True,
|
||||
default="",
|
||||
tooltip="Text prompt for video generation.",
|
||||
),
|
||||
IO.Combo.Input(
|
||||
"resolution",
|
||||
options=["480P", "768P"],
|
||||
default="768P",
|
||||
tooltip="Resolution of the output video.",
|
||||
),
|
||||
]
|
||||
if include_ratio:
|
||||
inputs.append(
|
||||
IO.Combo.Input(
|
||||
"ratio",
|
||||
options=["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
|
||||
default="16:9",
|
||||
tooltip="Aspect ratio of the output video.",
|
||||
)
|
||||
)
|
||||
inputs.extend(
|
||||
[
|
||||
IO.Int.Input(
|
||||
"duration",
|
||||
default=5,
|
||||
min=5,
|
||||
max=15,
|
||||
step=1,
|
||||
tooltip="Duration of the output video in seconds (5-15).",
|
||||
display_mode=IO.NumberDisplay.slider,
|
||||
),
|
||||
IO.Combo.Input(
|
||||
"prompt_expansion_mode",
|
||||
options=["balanced", "quality"],
|
||||
default="balanced",
|
||||
tooltip="How much effort is spent rewriting the prompt before generation.",
|
||||
),
|
||||
]
|
||||
)
|
||||
return inputs
|
||||
|
||||
|
||||
async def _hailuo03_max_run_task(
|
||||
cls: type[IO.ComfyNode],
|
||||
*,
|
||||
endpoint: str,
|
||||
request: Hailuo03MaxVideoRequest,
|
||||
) -> IO.NodeOutput:
|
||||
submit = await sync_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{HAILUO_03_MAX_ENDPOINT}/{endpoint}", method="POST"),
|
||||
response_model=Hailuo03MaxTaskCreationResponse,
|
||||
data=request,
|
||||
)
|
||||
await poll_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{HAILUO_03_MAX_ENDPOINT}/requests/{submit.request_id}/status"),
|
||||
response_model=Hailuo03MaxTaskStatusResponse,
|
||||
status_extractor=lambda r: r.status,
|
||||
completed_statuses=["COMPLETED"],
|
||||
queued_statuses=["IN_QUEUE"],
|
||||
poll_interval=5,
|
||||
)
|
||||
result = await sync_op(
|
||||
cls,
|
||||
ApiEndpoint(path=f"{HAILUO_03_MAX_ENDPOINT}/requests/{submit.request_id}"),
|
||||
response_model=Hailuo03MaxVideoResult,
|
||||
)
|
||||
return IO.NodeOutput(await download_url_to_video_output(result.video.url))
|
||||
|
||||
|
||||
class MinimaxHailuo03TextToVideoNode(IO.ComfyNode):
|
||||
@classmethod
|
||||
def define_schema(cls):
|
||||
@@ -548,11 +631,14 @@ class MinimaxHailuo03TextToVideoNode(IO.ComfyNode):
|
||||
node_id="MinimaxHailuo03TextToVideoNode",
|
||||
display_name="MiniMax H3 Text to Video",
|
||||
category="partner/video/MiniMax",
|
||||
description="Generate video from a text prompt using the MiniMax H3 model.",
|
||||
description="Generate video from a text prompt using the MiniMax H3 models.",
|
||||
inputs=[
|
||||
IO.DynamicCombo.Input(
|
||||
"model",
|
||||
options=[IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(allow_adaptive=False))],
|
||||
options=[
|
||||
IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(allow_adaptive=False)),
|
||||
IO.DynamicCombo.Option(HAILUO_03_MAX_MODEL, _hailuo03_max_model_inputs()),
|
||||
],
|
||||
tooltip="Model to use for video generation.",
|
||||
),
|
||||
IO.Int.Input(
|
||||
@@ -583,11 +669,14 @@ class MinimaxHailuo03TextToVideoNode(IO.ComfyNode):
|
||||
],
|
||||
is_api_node=True,
|
||||
price_badge=IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model.resolution", "model.duration"]),
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),
|
||||
expr="""
|
||||
(
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$rate := $lookup(widgets, "model") = "minimax h3 max"
|
||||
? ($res = "480p" ? 0.0715 : 0.1144)
|
||||
: ($res = "768p" ? 0.1287 : 0.1859);
|
||||
{"type": "usd", "usd": $dur * $rate}
|
||||
)
|
||||
""",
|
||||
@@ -602,6 +691,22 @@ class MinimaxHailuo03TextToVideoNode(IO.ComfyNode):
|
||||
watermark: bool,
|
||||
) -> IO.NodeOutput:
|
||||
validate_string(model["prompt"], strip_whitespace=True, min_length=1)
|
||||
if model["model"] == HAILUO_03_MAX_MODEL:
|
||||
if watermark:
|
||||
raise ValueError("Watermark is only supported by MiniMax H3.")
|
||||
validate_string(model["prompt"], strip_whitespace=False, max_length=HAILUO_03_MAX_PROMPT_MAX_LENGTH)
|
||||
return await _hailuo03_max_run_task(
|
||||
cls,
|
||||
endpoint="text-to-video",
|
||||
request=Hailuo03MaxVideoRequest(
|
||||
prompt=model["prompt"],
|
||||
duration=model["duration"],
|
||||
resolution=model["resolution"],
|
||||
prompt_expansion_mode=model["prompt_expansion_mode"],
|
||||
seed=seed,
|
||||
aspect_ratio=model["ratio"],
|
||||
),
|
||||
)
|
||||
return await _hailuo03_run_task(
|
||||
cls,
|
||||
model_id=HAILUO_03_MODELS[model["model"]],
|
||||
@@ -622,11 +727,14 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):
|
||||
display_name="MiniMax H3 First-Last-Frame to Video",
|
||||
category="partner/video/MiniMax",
|
||||
description="Generate video from a first frame image and an optional last frame image "
|
||||
"using the MiniMax H3 model. The aspect ratio of the video follows the supplied images.",
|
||||
"using the MiniMax H3 models. The aspect ratio of the video follows the supplied images.",
|
||||
inputs=[
|
||||
IO.DynamicCombo.Input(
|
||||
"model",
|
||||
options=[IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(include_ratio=False))],
|
||||
options=[
|
||||
IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(include_ratio=False)),
|
||||
IO.DynamicCombo.Option(HAILUO_03_MAX_MODEL, _hailuo03_max_model_inputs(include_ratio=False)),
|
||||
],
|
||||
tooltip="Model to use for video generation.",
|
||||
),
|
||||
IO.Image.Input(
|
||||
@@ -666,11 +774,14 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):
|
||||
],
|
||||
is_api_node=True,
|
||||
price_badge=IO.PriceBadge(
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model.resolution", "model.duration"]),
|
||||
depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),
|
||||
expr="""
|
||||
(
|
||||
$dur := $lookup(widgets, "model.duration");
|
||||
$rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859;
|
||||
$res := $lookup(widgets, "model.resolution");
|
||||
$rate := $lookup(widgets, "model") = "minimax h3 max"
|
||||
? ($res = "480p" ? 0.0715 : 0.1144)
|
||||
: ($res = "768p" ? 0.1287 : 0.1859);
|
||||
{"type": "usd", "usd": $dur * $rate}
|
||||
)
|
||||
""",
|
||||
@@ -691,6 +802,31 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):
|
||||
if frame is not None:
|
||||
validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5
|
||||
validate_image_dimensions(frame, min_width=256, min_height=256)
|
||||
if model["model"] == HAILUO_03_MAX_MODEL:
|
||||
if watermark:
|
||||
raise ValueError("Watermark is only supported by MiniMax H3.")
|
||||
validate_string(model["prompt"], strip_whitespace=False, max_length=HAILUO_03_MAX_PROMPT_MAX_LENGTH)
|
||||
image_url = (
|
||||
await upload_images_to_comfyapi(cls, first_frame, max_images=1, wait_label="Uploading first frame")
|
||||
)[0]
|
||||
end_image_url = None
|
||||
if last_frame is not None:
|
||||
end_image_url = (
|
||||
await upload_images_to_comfyapi(cls, last_frame, max_images=1, wait_label="Uploading last frame")
|
||||
)[0]
|
||||
return await _hailuo03_max_run_task(
|
||||
cls,
|
||||
endpoint="image-to-video",
|
||||
request=Hailuo03MaxVideoRequest(
|
||||
prompt=model["prompt"],
|
||||
duration=model["duration"],
|
||||
resolution=model["resolution"],
|
||||
prompt_expansion_mode=model["prompt_expansion_mode"],
|
||||
seed=seed,
|
||||
image_url=image_url,
|
||||
end_image_url=end_image_url,
|
||||
),
|
||||
)
|
||||
|
||||
content: list = [
|
||||
Hailuo03TextContent(text=model["prompt"]),
|
||||
|
||||
@@ -147,25 +147,25 @@ def pil_to_bytesio(img: Image.Image, mime_type: str = "image/png") -> BytesIO:
|
||||
def _compute_downscale_dims(src_w: int, src_h: int, total_pixels: int) -> tuple[int, int] | None:
|
||||
"""Return downscaled (w, h) with even dims fitting ``total_pixels``, or None if already fits.
|
||||
|
||||
Source aspect ratio is preserved; output may drift by a fraction of a percent because both dimensions
|
||||
are rounded down to even values (many codecs require divisible-by-2).
|
||||
Both dimensions are rounded to even values (many codecs require divisible-by-2).
|
||||
"""
|
||||
pixels = src_w * src_h
|
||||
if pixels <= total_pixels:
|
||||
return None
|
||||
scale = math.sqrt(total_pixels / pixels)
|
||||
new_w = max(2, int(src_w * scale))
|
||||
new_h = max(2, int(src_h * scale))
|
||||
new_w -= new_w % 2
|
||||
new_h -= new_h % 2
|
||||
return new_w, new_h
|
||||
long_src, short_src = max(src_w, src_h), min(src_w, src_h)
|
||||
long_new = max(2, int(long_src * scale) // 2 * 2)
|
||||
short_new = max(2, math.ceil(long_new * short_src / long_src / 2) * 2)
|
||||
if long_new * short_new > total_pixels:
|
||||
long_new = max(2, total_pixels // short_new // 2 * 2)
|
||||
short_new = max(2, math.ceil(long_new * short_src / long_src / 2) * 2)
|
||||
return (long_new, short_new) if src_w >= src_h else (short_new, long_new)
|
||||
|
||||
|
||||
def downscale_image_tensor(image: torch.Tensor, total_pixels: int = 1536 * 1024) -> torch.Tensor:
|
||||
"""Downscale input image tensor to roughly the specified total pixels.
|
||||
|
||||
Output dimensions are rounded down to even values so that the result is guaranteed to fit within ``total_pixels``
|
||||
and is compatible with codecs that require even dimensions (e.g. yuv420p).
|
||||
Output dimensions are even and guaranteed to fit within ``total_pixels``
|
||||
"""
|
||||
samples = image.movedim(-1, 1)
|
||||
dims = _compute_downscale_dims(samples.shape[3], samples.shape[2], int(total_pixels))
|
||||
|
||||
@@ -9,7 +9,11 @@ from comfy.cli_args import args
|
||||
if not torch.cuda.is_available():
|
||||
args.cpu = True
|
||||
|
||||
from comfy_api_nodes.util.conversions import bytesio_to_image_tensor, pad_images_to_common_channels # noqa: E402
|
||||
from comfy_api_nodes.util.conversions import ( # noqa: E402
|
||||
bytesio_to_image_tensor,
|
||||
downscale_image_tensor,
|
||||
pad_images_to_common_channels,
|
||||
)
|
||||
|
||||
|
||||
def encode(image: Image.Image, image_format: str = "PNG") -> BytesIO:
|
||||
@@ -78,3 +82,47 @@ def test_pad_leaves_homogeneous_channels_unchanged():
|
||||
images = [torch.rand(1, 4, 4, 3), torch.rand(2, 4, 4, 3)]
|
||||
padded = pad_images_to_common_channels(images)
|
||||
assert all(p is i for p, i in zip(padded, images))
|
||||
|
||||
|
||||
DOWNSCALE_CASES = [
|
||||
(5000, 2000, 2048 * 2048),
|
||||
(2000, 5000, 2048 * 2048),
|
||||
(4096, 1638, 2048 * 2048),
|
||||
(1000, 400, 128 * 128),
|
||||
(400, 1000, 128 * 128),
|
||||
(999, 333, 100 * 100),
|
||||
(333, 999, 100 * 100),
|
||||
(3000, 3000, 256 * 256),
|
||||
]
|
||||
|
||||
|
||||
def downscaled_size(width, height, total_pixels):
|
||||
out = downscale_image_tensor(torch.zeros(1, height, width, 3), total_pixels=total_pixels)
|
||||
return out.shape[2], out.shape[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width, height, total_pixels", DOWNSCALE_CASES)
|
||||
def test_downscale_dims_are_even(width, height, total_pixels):
|
||||
new_w, new_h = downscaled_size(width, height, total_pixels)
|
||||
assert new_w % 2 == 0 and new_h % 2 == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width, height, total_pixels", DOWNSCALE_CASES)
|
||||
def test_downscale_fits_total_pixels(width, height, total_pixels):
|
||||
new_w, new_h = downscaled_size(width, height, total_pixels)
|
||||
assert new_w * new_h <= total_pixels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width, height, total_pixels", DOWNSCALE_CASES)
|
||||
def test_downscale_never_makes_aspect_more_elongated(width, height, total_pixels):
|
||||
new_w, new_h = downscaled_size(width, height, total_pixels)
|
||||
src_ratio, new_ratio = width / height, new_w / new_h
|
||||
if src_ratio >= 1:
|
||||
assert 1 <= new_ratio <= src_ratio
|
||||
else:
|
||||
assert src_ratio <= new_ratio <= 1
|
||||
|
||||
|
||||
def test_downscale_leaves_fitting_images_untouched():
|
||||
image = torch.zeros(1, 300, 700, 3)
|
||||
assert downscale_image_tensor(image, total_pixels=700 * 300) is image
|
||||
|
||||
Reference in New Issue
Block a user