mirror of
https://github.com/calesthio/OpenMontage.git
synced 2026-08-24 17:10:27 +08:00
Add official Kling API providers
This commit is contained in:
6
tools/_kling/__init__.py
Normal file
6
tools/_kling/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
"""Shared helpers for Kling official API providers."""
|
||||
|
||||
from .client import KlingClient
|
||||
from .errors import KlingAPIError, is_retryable_kling_error
|
||||
|
||||
__all__ = ["KlingAPIError", "KlingClient", "is_retryable_kling_error"]
|
||||
121
tools/_kling/account.py
Normal file
121
tools/_kling/account.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""Account usage diagnostics for Kling official API.
|
||||
|
||||
This module is a low-frequency helper, not an OpenMontage registry tool.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
|
||||
from .client import KlingClient
|
||||
from .errors import KlingAPIError
|
||||
|
||||
_CACHE: dict[tuple[tuple[str, str], ...], dict[str, Any]] = {}
|
||||
_LAST_QUERY_AT = 0.0
|
||||
|
||||
|
||||
def reset_account_usage_cache() -> None:
|
||||
"""Clear in-process account usage cache. Intended for tests."""
|
||||
|
||||
global _LAST_QUERY_AT
|
||||
_CACHE.clear()
|
||||
_LAST_QUERY_AT = 0.0
|
||||
|
||||
|
||||
def get_account_costs(
|
||||
*,
|
||||
start_time: str | None = None,
|
||||
end_time: str | None = None,
|
||||
resource_pack_name: str | None = None,
|
||||
client: KlingClient | None = None,
|
||||
ttl_seconds: float = 300.0,
|
||||
min_interval_seconds: float = 10.0,
|
||||
now: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Read `/account/costs` with in-process cache and throttle protection."""
|
||||
|
||||
global _LAST_QUERY_AT
|
||||
|
||||
timestamp = time.time() if now is None else now
|
||||
params = {
|
||||
key: value
|
||||
for key, value in {
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"resource_pack_name": resource_pack_name,
|
||||
}.items()
|
||||
if value
|
||||
}
|
||||
api = client or KlingClient()
|
||||
key = _cache_key(api, params)
|
||||
cached = _CACHE.get(key)
|
||||
if cached and timestamp - float(cached["fetched_at"]) <= ttl_seconds:
|
||||
return {**cached["payload"], "cached": True, "throttle_status": "cache_hit"}
|
||||
|
||||
if _LAST_QUERY_AT and timestamp - _LAST_QUERY_AT < min_interval_seconds:
|
||||
if cached:
|
||||
return {**cached["payload"], "cached": True, "throttle_status": "throttled_cache"}
|
||||
return {
|
||||
"provider": "kling_official",
|
||||
"queried_range": {
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"resource_pack_name": resource_pack_name,
|
||||
},
|
||||
"cached": False,
|
||||
"throttle_status": "throttled_no_cache",
|
||||
"message": "Account Usage is rate-limited; retry after the local throttle window.",
|
||||
}
|
||||
|
||||
raw = api.get("/account/costs", params=params)
|
||||
data = raw.get("data") if isinstance(raw, dict) else {}
|
||||
if not isinstance(data, dict):
|
||||
data = {}
|
||||
payload = {
|
||||
"provider": "kling_official",
|
||||
"queried_range": {
|
||||
"start_time": start_time,
|
||||
"end_time": end_time,
|
||||
"resource_pack_name": resource_pack_name,
|
||||
},
|
||||
"resource_pack_subscribe_infos": data.get("resource_pack_subscribe_infos", []),
|
||||
"raw": raw,
|
||||
"cached": False,
|
||||
"throttle_status": "fresh",
|
||||
}
|
||||
_CACHE[key] = {"fetched_at": timestamp, "payload": payload}
|
||||
_LAST_QUERY_AT = timestamp
|
||||
return payload
|
||||
|
||||
|
||||
def _cache_key(client: Any, params: dict[str, Any]) -> tuple[tuple[str, str], ...]:
|
||||
"""Scope Account Usage cache by request params and account endpoint identity."""
|
||||
|
||||
api_key = getattr(client, "api_key", None) or ""
|
||||
api_key_hash = sha256(str(api_key).encode("utf-8")).hexdigest() if api_key else ""
|
||||
scope = {
|
||||
"base_url": getattr(client, "base_url", ""),
|
||||
"api_key_sha256": api_key_hash,
|
||||
**{name: str(value) for name, value in params.items()},
|
||||
}
|
||||
return tuple(sorted((name, str(value)) for name, value in scope.items()))
|
||||
|
||||
|
||||
def account_usage_hint_for_error(error: KlingAPIError) -> dict[str, Any]:
|
||||
"""Return a diagnostic hint for balance/resource-pack related errors."""
|
||||
|
||||
code = str(error.code) if error.code is not None else ""
|
||||
if code not in {"1101", "1102"}:
|
||||
return {}
|
||||
return {
|
||||
"provider": "kling_official",
|
||||
"reason": "account_balance_or_resource_pack",
|
||||
"message": (
|
||||
"Kling returned an account/resource-pack error. Use tools._kling.account.get_account_costs() "
|
||||
"for a low-frequency account usage diagnostic, or check the Kling Open Platform console."
|
||||
),
|
||||
"error_code": error.code,
|
||||
"request_id": error.request_id,
|
||||
}
|
||||
17
tools/_kling/callbacks.py
Normal file
17
tools/_kling/callbacks.py
Normal file
@@ -0,0 +1,17 @@
|
||||
"""Callback validation helpers for Kling official providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def validate_callback_url(callback_url: str | None) -> str | None:
|
||||
"""Return a normalized callback URL or raise for obviously invalid input."""
|
||||
|
||||
if not callback_url:
|
||||
return None
|
||||
value = str(callback_url).strip()
|
||||
parsed = urlparse(value)
|
||||
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
||||
raise ValueError("callback_url must be an absolute http(s) URL")
|
||||
return value
|
||||
216
tools/_kling/client.py
Normal file
216
tools/_kling/client.py
Normal file
@@ -0,0 +1,216 @@
|
||||
"""HTTP client and task parsers for Kling official API providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
import requests
|
||||
|
||||
from .errors import KlingAPIError, is_retryable_kling_error
|
||||
from .schemas import (
|
||||
CLASSIC_FAILURE_STATUS,
|
||||
CLASSIC_PENDING_STATUSES,
|
||||
CLASSIC_SUCCESS_STATUS,
|
||||
DEFAULT_API_BASE_URL,
|
||||
TURBO_FAILURE_STATUS,
|
||||
TURBO_PENDING_STATUSES,
|
||||
TURBO_SUCCESS_STATUS,
|
||||
)
|
||||
|
||||
|
||||
class KlingClient:
|
||||
"""Small synchronous client for the official Kling API."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str | None = None,
|
||||
base_url: str | None = None,
|
||||
session: Any | None = None,
|
||||
max_retries: int = 2,
|
||||
) -> None:
|
||||
self.api_key = api_key if api_key is not None else os.environ.get("KLING_API_KEY")
|
||||
self.base_url = (base_url or os.environ.get("KLING_API_BASE_URL") or DEFAULT_API_BASE_URL).rstrip("/")
|
||||
self.session = session or requests.Session()
|
||||
self.max_retries = max_retries
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
if not self.api_key:
|
||||
raise KlingAPIError(
|
||||
"KLING_API_KEY is not set. Configure KLING_API_KEY for official Kling API access.",
|
||||
http_status=401,
|
||||
)
|
||||
return {
|
||||
"Authorization": f"Bearer {self.api_key}",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
|
||||
return self._request("post", path, json=payload)
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
|
||||
return self._request("get", path, params=params)
|
||||
|
||||
def download(self, url: str, output_path: Path, timeout: int = 180) -> Path:
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
response = self.session.get(url, timeout=timeout)
|
||||
self._raise_for_http_error(response)
|
||||
content = getattr(response, "content", None)
|
||||
if content is None and hasattr(response, "iter_content"):
|
||||
content = b"".join(chunk for chunk in response.iter_content(chunk_size=1024 * 128) if chunk)
|
||||
output_path.write_bytes(content or b"")
|
||||
return output_path
|
||||
|
||||
def create_classic_task(self, path: str, payload: dict[str, Any]) -> str:
|
||||
data = self.post(path, payload)
|
||||
task_id = ((data.get("data") or {}).get("task_id"))
|
||||
if not task_id:
|
||||
raise KlingAPIError(f"Kling Classic create response missing data.task_id: {data}")
|
||||
return str(task_id)
|
||||
|
||||
def poll_classic(
|
||||
self,
|
||||
path: str,
|
||||
task_id: str,
|
||||
result_key: str,
|
||||
timeout_seconds: int = 900,
|
||||
poll_interval: float = 5.0,
|
||||
) -> list[dict[str, Any]]:
|
||||
deadline = time.time() + timeout_seconds
|
||||
while time.time() < deadline:
|
||||
data = self.get(f"{path.rstrip('/')}/{task_id}")
|
||||
payload = data.get("data") or {}
|
||||
status = payload.get("task_status") or payload.get("status")
|
||||
if status == CLASSIC_SUCCESS_STATUS:
|
||||
task_result = payload.get("task_result") or {}
|
||||
outputs = task_result.get(result_key) or []
|
||||
if not isinstance(outputs, list):
|
||||
raise KlingAPIError(f"Kling Classic result path data.task_result.{result_key} is not a list")
|
||||
return outputs
|
||||
if status == CLASSIC_FAILURE_STATUS:
|
||||
message = payload.get("task_status_msg") or payload.get("message") or "Kling Classic task failed"
|
||||
raise KlingAPIError(str(message), code=payload.get("task_status"), response=data)
|
||||
if status not in CLASSIC_PENDING_STATUSES:
|
||||
raise KlingAPIError(f"Unexpected Kling Classic task status {status!r}", response=data)
|
||||
time.sleep(min(poll_interval, max(0.0, deadline - time.time())))
|
||||
raise TimeoutError(f"Kling Classic task {task_id} timed out after {timeout_seconds}s")
|
||||
|
||||
def create_turbo(self, path: str, payload: dict[str, Any]) -> str:
|
||||
data = self.post(path, payload)
|
||||
task_id = ((data.get("data") or {}).get("id"))
|
||||
if not task_id:
|
||||
raise KlingAPIError(f"Kling Turbo create response missing data.id: {data}")
|
||||
return str(task_id)
|
||||
|
||||
def poll_turbo(
|
||||
self,
|
||||
task_id: str,
|
||||
timeout_seconds: int = 900,
|
||||
poll_interval: float = 5.0,
|
||||
) -> list[dict[str, Any]]:
|
||||
deadline = time.time() + timeout_seconds
|
||||
while time.time() < deadline:
|
||||
data = self.get("/tasks", params={"task_ids": task_id})
|
||||
records = data.get("data") or []
|
||||
if not records:
|
||||
raise KlingAPIError(f"Kling Turbo poll response missing data[0]: {data}")
|
||||
record = records[0]
|
||||
status = record.get("status") or record.get("task_status")
|
||||
if status == TURBO_SUCCESS_STATUS:
|
||||
outputs = record.get("outputs") or []
|
||||
if not isinstance(outputs, list):
|
||||
raise KlingAPIError("Kling Turbo result path data[0].outputs is not a list")
|
||||
return outputs
|
||||
if status == TURBO_FAILURE_STATUS:
|
||||
message = record.get("message") or record.get("error") or "Kling Turbo task failed"
|
||||
raise KlingAPIError(str(message), code=record.get("code"), request_id=record.get("request_id"), response=data)
|
||||
if status not in TURBO_PENDING_STATUSES:
|
||||
raise KlingAPIError(f"Unexpected Kling Turbo task status {status!r}", response=data)
|
||||
time.sleep(min(poll_interval, max(0.0, deadline - time.time())))
|
||||
raise TimeoutError(f"Kling Turbo task {task_id} timed out after {timeout_seconds}s")
|
||||
|
||||
def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
|
||||
url = self._url(path)
|
||||
last_error: KlingAPIError | None = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
try:
|
||||
response = getattr(self.session, method)(url, headers=self.headers, timeout=30, **kwargs)
|
||||
self._raise_for_http_error(response)
|
||||
data = response.json()
|
||||
self._raise_for_business_error(data)
|
||||
return data
|
||||
except KlingAPIError as error:
|
||||
last_error = error
|
||||
if attempt >= self.max_retries or not is_retryable_kling_error(error):
|
||||
raise
|
||||
time.sleep(min(2.0 * (attempt + 1), 8.0))
|
||||
except requests.RequestException as exc:
|
||||
last_error = KlingAPIError(str(exc))
|
||||
if attempt >= self.max_retries:
|
||||
raise last_error from exc
|
||||
time.sleep(min(2.0 * (attempt + 1), 8.0))
|
||||
raise last_error or KlingAPIError("Kling API request failed")
|
||||
|
||||
def _url(self, path: str) -> str:
|
||||
if path.startswith("http://") or path.startswith("https://"):
|
||||
return path
|
||||
return urljoin(f"{self.base_url}/", path.lstrip("/"))
|
||||
|
||||
def _raise_for_http_error(self, response: Any) -> None:
|
||||
status = getattr(response, "status_code", None)
|
||||
if status is not None and 200 <= int(status) < 300:
|
||||
return
|
||||
code = None
|
||||
message = None
|
||||
request_id = None
|
||||
body: dict[str, Any] | None = None
|
||||
try:
|
||||
body = response.json()
|
||||
code = body.get("code")
|
||||
message = body.get("message") or body.get("msg")
|
||||
request_id = body.get("request_id") or body.get("requestId")
|
||||
except Exception:
|
||||
text = getattr(response, "text", "")
|
||||
message = text[:500] if text else f"HTTP {status}"
|
||||
raise self._format_error(
|
||||
code=code,
|
||||
message=message or f"HTTP {status}",
|
||||
request_id=request_id,
|
||||
http_status=int(status) if status is not None else None,
|
||||
response=body,
|
||||
)
|
||||
|
||||
def _raise_for_business_error(self, data: dict[str, Any]) -> None:
|
||||
code = data.get("code")
|
||||
if code in (None, 0, "0"):
|
||||
return
|
||||
raise self._format_error(
|
||||
code=code,
|
||||
message=str(data.get("message") or data.get("msg") or "Kling API returned an error"),
|
||||
request_id=data.get("request_id") or data.get("requestId"),
|
||||
response=data,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _format_error(
|
||||
*,
|
||||
code: str | int | None,
|
||||
message: str,
|
||||
request_id: str | None = None,
|
||||
http_status: int | None = None,
|
||||
response: dict[str, Any] | None = None,
|
||||
) -> KlingAPIError:
|
||||
if str(code) == "1303" and "并发/资源包限制" not in message:
|
||||
message = f"{message} (并发/资源包限制: parallel task over resource pack limit)"
|
||||
return KlingAPIError(
|
||||
message=message,
|
||||
code=code,
|
||||
request_id=request_id,
|
||||
http_status=http_status,
|
||||
response=response,
|
||||
)
|
||||
86
tools/_kling/elements.py
Normal file
86
tools/_kling/elements.py
Normal file
@@ -0,0 +1,86 @@
|
||||
"""Element reference helpers for Kling official Omni providers.
|
||||
|
||||
These helpers intentionally do not inherit from BaseTool. Elements are an
|
||||
internal provider reference mechanism in Phase 2, not a registry capability.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .client import KlingClient
|
||||
|
||||
|
||||
def normalize_element_list(element_list: Any | None) -> list[dict[str, int]]:
|
||||
"""Normalize official Kling element references to element_list objects."""
|
||||
|
||||
if not element_list:
|
||||
return []
|
||||
if not isinstance(element_list, list):
|
||||
raise ValueError("element_list must be a list of element ids or objects")
|
||||
|
||||
normalized: list[dict[str, int]] = []
|
||||
for item in element_list:
|
||||
raw_id: Any
|
||||
if isinstance(item, dict):
|
||||
raw_id = item.get("element_id", item.get("id"))
|
||||
else:
|
||||
raw_id = item
|
||||
if raw_id is None:
|
||||
raise ValueError("each element_list item must include element_id")
|
||||
try:
|
||||
element_id = int(raw_id)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"element_id must be an integer-compatible value: {raw_id!r}") from exc
|
||||
if element_id <= 0:
|
||||
raise ValueError("element_id must be positive")
|
||||
normalized.append({"element_id": element_id})
|
||||
return normalized
|
||||
|
||||
|
||||
def element_ids(element_list: Any | None) -> list[int]:
|
||||
"""Return normalized element ids from an element reference list."""
|
||||
|
||||
return [item["element_id"] for item in normalize_element_list(element_list)]
|
||||
|
||||
|
||||
def get_custom_element(element_id: int, client: KlingClient | None = None) -> dict[str, Any]:
|
||||
"""Fetch one custom element for validation or diagnostics."""
|
||||
|
||||
api = client or KlingClient()
|
||||
return api.get(f"/v1/general/advanced-custom-elements/{int(element_id)}")
|
||||
|
||||
|
||||
def list_custom_elements(
|
||||
client: KlingClient | None = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""List custom elements without creating or deleting assets."""
|
||||
|
||||
api = client or KlingClient()
|
||||
return api.get("/v1/general/advanced-custom-elements", params=params)
|
||||
|
||||
|
||||
def list_preset_elements(
|
||||
client: KlingClient | None = None,
|
||||
params: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""List official preset elements without entering the tool registry."""
|
||||
|
||||
api = client or KlingClient()
|
||||
return api.get("/v1/general/advanced-presets-elements", params=params)
|
||||
|
||||
|
||||
def write_elements_artifact(
|
||||
artifact_path: str | Path,
|
||||
elements: list[dict[str, Any]],
|
||||
) -> Path:
|
||||
"""Write element metadata in the Phase 2 reproducibility artifact shape."""
|
||||
|
||||
path = Path(artifact_path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {"provider": "kling_official", "elements": elements}
|
||||
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
return path
|
||||
46
tools/_kling/errors.py
Normal file
46
tools/_kling/errors.py
Normal file
@@ -0,0 +1,46 @@
|
||||
"""Error types and retry policy for Kling official API calls."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class KlingAPIError(Exception):
|
||||
"""Structured error returned by the Kling official API."""
|
||||
|
||||
message: str
|
||||
code: str | int | None = None
|
||||
request_id: str | None = None
|
||||
http_status: int | None = None
|
||||
response: dict[str, Any] | None = None
|
||||
|
||||
def __str__(self) -> str:
|
||||
parts = [self.message]
|
||||
if self.code is not None:
|
||||
parts.append(f"code={self.code}")
|
||||
if self.request_id:
|
||||
parts.append(f"request_id={self.request_id}")
|
||||
if self.http_status is not None:
|
||||
parts.append(f"http_status={self.http_status}")
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
_RETRYABLE_CODES = {"1302", "1303", "5000", "5001", "5002"}
|
||||
_RETRYABLE_HTTP = {500, 503, 504}
|
||||
|
||||
|
||||
def _code_str(code: str | int | None) -> str | None:
|
||||
if code is None:
|
||||
return None
|
||||
return str(code)
|
||||
|
||||
|
||||
def is_retryable_kling_error(error: KlingAPIError) -> bool:
|
||||
"""Return whether an official Kling error is safe for limited retry."""
|
||||
|
||||
code = _code_str(error.code)
|
||||
if code in _RETRYABLE_CODES:
|
||||
return True
|
||||
return error.http_status in _RETRYABLE_HTTP
|
||||
111
tools/_kling/media.py
Normal file
111
tools/_kling/media.py
Normal file
@@ -0,0 +1,111 @@
|
||||
"""Media normalization and download helpers for Kling official providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import mimetypes
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
def strip_data_uri_prefix(value: str | None) -> str | None:
|
||||
"""Return raw base64/content by removing a data URI prefix if present."""
|
||||
|
||||
if value is None:
|
||||
return None
|
||||
marker = ";base64,"
|
||||
if value.startswith("data:") and marker in value:
|
||||
return value.split(marker, 1)[1]
|
||||
return value
|
||||
|
||||
|
||||
def image_file_to_raw_base64(path: str | Path) -> str:
|
||||
"""Read a local image file and return raw base64 without data URI prefix."""
|
||||
|
||||
image_path = Path(path)
|
||||
if not image_path.is_file():
|
||||
raise FileNotFoundError(f"Image not found: {image_path}")
|
||||
return base64.b64encode(image_path.read_bytes()).decode("ascii")
|
||||
|
||||
|
||||
def file_to_raw_base64(path: str | Path, *, label: str = "File") -> str:
|
||||
"""Read a local media file and return raw base64 without a data URI prefix."""
|
||||
|
||||
media_path = Path(path)
|
||||
if not media_path.is_file():
|
||||
raise FileNotFoundError(f"{label} not found: {media_path}")
|
||||
return base64.b64encode(media_path.read_bytes()).decode("ascii")
|
||||
|
||||
|
||||
def normalize_image_input(url: str | None = None, path: str | Path | None = None) -> str | None:
|
||||
"""Normalize a Kling image input to either URL or raw base64."""
|
||||
|
||||
if url:
|
||||
return strip_data_uri_prefix(url)
|
||||
if path:
|
||||
return image_file_to_raw_base64(path)
|
||||
return None
|
||||
|
||||
|
||||
def normalize_media_input(
|
||||
url: str | None = None,
|
||||
path: str | Path | None = None,
|
||||
value: str | None = None,
|
||||
*,
|
||||
label: str = "Media file",
|
||||
) -> str | None:
|
||||
"""Normalize a generic Kling media input to URL, raw base64, or raw provided value."""
|
||||
|
||||
if value:
|
||||
return strip_data_uri_prefix(value)
|
||||
if url:
|
||||
return strip_data_uri_prefix(url)
|
||||
if path:
|
||||
return file_to_raw_base64(path, label=label)
|
||||
return None
|
||||
|
||||
|
||||
def extension_from_url(url: str | None, default: str = ".png") -> str:
|
||||
"""Infer a file extension from a URL path."""
|
||||
|
||||
if not url:
|
||||
return default
|
||||
suffix = Path(urlparse(url).path).suffix.lower()
|
||||
if suffix in {
|
||||
".png",
|
||||
".jpg",
|
||||
".jpeg",
|
||||
".webp",
|
||||
".gif",
|
||||
".mp4",
|
||||
".mov",
|
||||
".m4v",
|
||||
".mp3",
|
||||
".wav",
|
||||
".m4a",
|
||||
".aac",
|
||||
".ogg",
|
||||
".opus",
|
||||
}:
|
||||
return suffix
|
||||
return default
|
||||
|
||||
|
||||
def extension_from_content_type(content_type: str | None, default: str = ".png") -> str:
|
||||
if not content_type:
|
||||
return default
|
||||
ext = mimetypes.guess_extension(content_type.split(";", 1)[0].strip())
|
||||
return ext or default
|
||||
|
||||
|
||||
def output_path_with_suffix(path: str | Path, suffix: str) -> Path:
|
||||
output_path = Path(path)
|
||||
if output_path.suffix:
|
||||
return output_path
|
||||
return output_path.with_suffix(suffix)
|
||||
|
||||
|
||||
def numbered_output_path(first_path: Path, index: int, suffix: str) -> Path:
|
||||
if index == 0:
|
||||
return output_path_with_suffix(first_path, suffix)
|
||||
return first_path.with_name(f"{first_path.stem}_{index + 1}{suffix}")
|
||||
40
tools/_kling/omni.py
Normal file
40
tools/_kling/omni.py
Normal file
@@ -0,0 +1,40 @@
|
||||
"""Omni reference helpers for Kling official providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
PLACEHOLDER_RE = re.compile(r"<<<image_(\d+)>>>")
|
||||
|
||||
|
||||
def build_image_prompt_references(
|
||||
prompt: str,
|
||||
image_list: list[dict[str, Any]],
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
"""Bind image_list entries to stable Image Omni placeholders."""
|
||||
|
||||
references = [
|
||||
{
|
||||
"index": index,
|
||||
"placeholder": f"<<<image_{index}>>>",
|
||||
"source": item.get("source") or item.get("image") or item.get("image_url"),
|
||||
"source_type": item.get("source_type", "unknown"),
|
||||
}
|
||||
for index, item in enumerate(image_list, start=1)
|
||||
]
|
||||
if not references:
|
||||
return prompt, []
|
||||
|
||||
existing_numbers = [int(value) for value in PLACEHOLDER_RE.findall(prompt)]
|
||||
if existing_numbers:
|
||||
if max(existing_numbers) > len(references):
|
||||
raise ValueError(
|
||||
f"prompt references <<<image_{max(existing_numbers)}>>> but only {len(references)} image(s) were provided"
|
||||
)
|
||||
if min(existing_numbers) < 1:
|
||||
raise ValueError("Image Omni prompt placeholders must start at <<<image_1>>>")
|
||||
return prompt, references
|
||||
|
||||
placeholders = " ".join(item["placeholder"] for item in references)
|
||||
return f"{prompt}\nReferences: {placeholders}", references
|
||||
121
tools/_kling/schemas.py
Normal file
121
tools/_kling/schemas.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""Lightweight schema constants for Kling official providers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
|
||||
DEFAULT_API_BASE_URL = "https://api-singapore.klingai.com"
|
||||
|
||||
|
||||
class KlingProtocol(str, Enum):
|
||||
CLASSIC = "classic"
|
||||
TURBO = "turbo"
|
||||
|
||||
|
||||
CLASSIC_PENDING_STATUSES = {"submitted", "processing"}
|
||||
CLASSIC_SUCCESS_STATUS = "succeed"
|
||||
CLASSIC_FAILURE_STATUS = "failed"
|
||||
CLASSIC_STATUSES = [
|
||||
"submitted",
|
||||
"processing",
|
||||
"succeed",
|
||||
"failed",
|
||||
]
|
||||
|
||||
TURBO_PENDING_STATUSES = {"submitted", "processing"}
|
||||
TURBO_SUCCESS_STATUS = "succeeded"
|
||||
TURBO_FAILURE_STATUS = "failed"
|
||||
TURBO_STATUSES = [
|
||||
"submitted",
|
||||
"processing",
|
||||
"succeeded",
|
||||
"failed",
|
||||
]
|
||||
|
||||
VIDEO_MODELS = [
|
||||
"kling-v1",
|
||||
"kling-v1-5",
|
||||
"kling-v1-6",
|
||||
"kling-v2-master",
|
||||
"kling-v2-1",
|
||||
"kling-v2-1-master",
|
||||
"kling-v2-5-turbo",
|
||||
"kling-v2-6",
|
||||
"kling-v3",
|
||||
"kling-video-o1",
|
||||
"kling-v3-omni",
|
||||
]
|
||||
CLASSIC_VIDEO_MODELS = [
|
||||
"kling-v1",
|
||||
"kling-v1-5",
|
||||
"kling-v1-6",
|
||||
"kling-v2-master",
|
||||
"kling-v2-1",
|
||||
"kling-v2-1-master",
|
||||
"kling-v2-5-turbo",
|
||||
"kling-v2-6",
|
||||
"kling-v3",
|
||||
]
|
||||
OMNI_VIDEO_MODELS = ["kling-video-o1", "kling-v3-omni"]
|
||||
|
||||
IMAGE_MODELS = [
|
||||
"kling-v1",
|
||||
"kling-v1-5",
|
||||
"kling-v2",
|
||||
"kling-v2-new",
|
||||
"kling-v2-1",
|
||||
"kling-v3",
|
||||
"kling-image-o1",
|
||||
"kling-v3-omni",
|
||||
]
|
||||
IMAGE_GENERATION_MODELS = [
|
||||
"kling-v1",
|
||||
"kling-v1-5",
|
||||
"kling-v2",
|
||||
"kling-v2-new",
|
||||
"kling-v2-1",
|
||||
"kling-v3",
|
||||
]
|
||||
OMNI_IMAGE_MODELS = ["kling-image-o1", "kling-v3-omni"]
|
||||
|
||||
VIDEO_DURATIONS = [str(value) for value in range(3, 16)]
|
||||
VIDEO_ASPECT_RATIOS = ["16:9", "9:16", "1:1"]
|
||||
VIDEO_RESOLUTIONS = ["720p", "1080p"]
|
||||
VIDEO_MODES = ["std", "pro", "4k"]
|
||||
SOUND_VALUES = ["on", "off"]
|
||||
|
||||
IMAGE_RESOLUTIONS = ["1k", "2k", "4k"]
|
||||
IMAGE_ASPECT_RATIOS = ["16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3", "21:9", "auto"]
|
||||
IMAGE_REFERENCE_TYPES = ["subject", "face"]
|
||||
IMAGE_RESULT_TYPES = ["single", "series"]
|
||||
|
||||
RESULT_PATHS = {
|
||||
"classic_video": "data.task_result.videos[]",
|
||||
"classic_image": "data.task_result.images[]",
|
||||
"classic_audio": "data.task_result.audios[]",
|
||||
"turbo": "data[0].outputs[]",
|
||||
}
|
||||
|
||||
TTS_LANGUAGES = ["zh", "en"]
|
||||
TTS_SPEED_MIN = 0.5
|
||||
TTS_SPEED_MAX = 2.0
|
||||
|
||||
AVATAR_MODES = ["std", "pro"]
|
||||
LIP_SYNC_OPERATIONS = ["identify_face", "advanced_lip_sync", "full_lip_sync"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClassicTaskResult:
|
||||
task_id: str
|
||||
status: str
|
||||
outputs: list[dict[str, Any]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TurboTaskResult:
|
||||
task_id: str
|
||||
status: str
|
||||
outputs: list[dict[str, Any]]
|
||||
340
tools/audio/kling_tts.py
Normal file
340
tools/audio/kling_tts.py
Normal file
@@ -0,0 +1,340 @@
|
||||
"""Kling official API text-to-speech provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools._kling.account import account_usage_hint_for_error, get_account_costs
|
||||
from tools._kling.callbacks import validate_callback_url
|
||||
from tools._kling.client import KlingClient
|
||||
from tools._kling.errors import KlingAPIError
|
||||
from tools._kling.media import extension_from_url, numbered_output_path, output_path_with_suffix
|
||||
from tools._kling.schemas import TTS_LANGUAGES, TTS_SPEED_MAX, TTS_SPEED_MIN
|
||||
from tools.analysis.audio_probe import probe_duration
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
DependencyError,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class KlingTTS(BaseTool):
|
||||
name = "kling_tts"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.VOICE
|
||||
capability = "tts"
|
||||
provider = "kling_official"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:KLING_API_KEY"]
|
||||
install_instructions = (
|
||||
"Set KLING_API_KEY in .env for the official Kling API. "
|
||||
"Pass voice_id explicitly; OpenMontage does not guess Kling voice IDs."
|
||||
)
|
||||
agent_skills = ["kling-official", "text-to-speech"]
|
||||
|
||||
capabilities = ["text_to_speech", "voice_selection", "multilingual"]
|
||||
supports = {
|
||||
"multilingual": True,
|
||||
"voice_selection": True,
|
||||
"offline": False,
|
||||
"native_audio": True,
|
||||
}
|
||||
best_for = [
|
||||
"official Kling text-to-speech",
|
||||
"Chinese or English narration when a Kling voice_id is known",
|
||||
"keeping narration provider provenance inside the Kling official account",
|
||||
]
|
||||
not_good_for = [
|
||||
"fully offline narration",
|
||||
"voice cloning without a configured official voice_id",
|
||||
"auto-discovering voices",
|
||||
]
|
||||
fallback_tools = ["doubao_tts", "elevenlabs_tts", "openai_tts", "google_tts", "piper_tts"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["text", "voice_id"],
|
||||
"properties": {
|
||||
"text": {"type": "string"},
|
||||
"voice_id": {
|
||||
"type": "string",
|
||||
"description": "Official Kling voice ID. Required; do not rely on an unknown default.",
|
||||
},
|
||||
"voice_language": {"type": "string", "enum": TTS_LANGUAGES, "default": "en"},
|
||||
"voice_speed": {
|
||||
"type": "number",
|
||||
"minimum": TTS_SPEED_MIN,
|
||||
"maximum": TTS_SPEED_MAX,
|
||||
"default": 1.0,
|
||||
},
|
||||
"callback_url": {"type": "string"},
|
||||
"external_task_id": {"type": "string"},
|
||||
"include_account_usage": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Optional low-frequency account usage diagnostic; not used by default.",
|
||||
},
|
||||
"timeout_seconds": {"type": "integer", "default": 300},
|
||||
"poll_interval": {"type": "number", "default": 3.0},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
output_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"output": {"type": "string"},
|
||||
"output_path": {"type": "string"},
|
||||
"audio_paths": {"type": "array"},
|
||||
"task_id": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=100, network_required=True
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=2,
|
||||
backoff_seconds=2.0,
|
||||
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
|
||||
)
|
||||
idempotency_key_fields = ["text", "voice_id", "voice_language", "voice_speed"]
|
||||
side_effects = ["paid remote generation via official Kling API", "writes audio file to output_path"]
|
||||
user_visible_verification = ["Listen to generated audio for voice, language, and pacing"]
|
||||
quality_score = 0.78
|
||||
latency_p50_seconds = 20.0
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
text_length = len(str(inputs.get("text") or ""))
|
||||
return round(max(text_length, 1) * 0.000018, 4)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 30.0
|
||||
|
||||
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
result = super().dry_run(inputs)
|
||||
result.update(
|
||||
{
|
||||
"paid_api": True,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative character-based OpenMontage estimate pending official account-usage reconciliation.",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
try:
|
||||
self.check_dependencies()
|
||||
except DependencyError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
request = self._build_request(inputs)
|
||||
client = KlingClient()
|
||||
task_id, outputs = self._create_and_collect_audios(client, request, inputs)
|
||||
paths = self._download_audios(client, outputs, inputs)
|
||||
audio_duration = probe_duration(paths[0])
|
||||
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
|
||||
data: dict[str, Any] = {"provider": self.provider}
|
||||
if isinstance(exc, KlingAPIError):
|
||||
data.update(
|
||||
{
|
||||
"error_code": exc.code,
|
||||
"request_id": exc.request_id,
|
||||
"http_status": exc.http_status,
|
||||
"account_usage_diagnostic": account_usage_hint_for_error(exc),
|
||||
}
|
||||
)
|
||||
return ToolResult(success=False, data=data, error=f"Kling official TTS failed: {exc}")
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official TTS failed: {exc}")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": "kling-official-tts",
|
||||
"task_id": task_id,
|
||||
"operation": "text_to_speech",
|
||||
"text_length": len(request["payload"]["text"]),
|
||||
"voice_id": request["payload"]["voice_id"],
|
||||
"voice_language": request["payload"].get("voice_language"),
|
||||
"voice_speed": request["payload"].get("voice_speed"),
|
||||
"remote_outputs": outputs,
|
||||
"output": str(paths[0]),
|
||||
"output_path": str(paths[0]),
|
||||
"audio_paths": [str(path) for path in paths],
|
||||
"format": paths[0].suffix.lstrip(".") or "mp3",
|
||||
"audio_duration_seconds": round(audio_duration, 2) if audio_duration else None,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
|
||||
**self._account_usage_result(inputs, client),
|
||||
**self._callback_result_data(inputs, task_id),
|
||||
},
|
||||
artifacts=[str(path) for path in paths],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model="kling-official-tts",
|
||||
)
|
||||
|
||||
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
text = str(inputs.get("text") or "").strip()
|
||||
if not text:
|
||||
raise ValueError("text is required")
|
||||
if len(text) > 5000:
|
||||
raise ValueError("text exceeds Kling TTS safety limit of 5000 characters")
|
||||
|
||||
voice_id = str(inputs.get("voice_id") or "").strip()
|
||||
if not voice_id:
|
||||
raise ValueError("voice_id is required for Kling official TTS")
|
||||
|
||||
voice_language = str(inputs.get("voice_language") or "en")
|
||||
if voice_language not in TTS_LANGUAGES:
|
||||
raise ValueError(f"voice_language must be one of: {', '.join(TTS_LANGUAGES)}")
|
||||
|
||||
voice_speed = float(inputs.get("voice_speed", 1.0))
|
||||
if voice_speed < TTS_SPEED_MIN or voice_speed > TTS_SPEED_MAX:
|
||||
raise ValueError(f"voice_speed must be between {TTS_SPEED_MIN} and {TTS_SPEED_MAX}")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"text": text,
|
||||
"voice_id": voice_id,
|
||||
"voice_language": voice_language,
|
||||
"voice_speed": voice_speed,
|
||||
}
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/audio/tts",
|
||||
"payload": payload,
|
||||
"operation": "text_to_speech",
|
||||
"model": "kling-official-tts",
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _create_and_collect_audios(
|
||||
client: KlingClient,
|
||||
request: dict[str, Any],
|
||||
inputs: dict[str, Any],
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
"""Create a TTS task and return audio outputs.
|
||||
|
||||
Official TTS may return a completed task and task_result.audios[]
|
||||
directly from POST /v1/audio/tts. Older/async behavior still requires
|
||||
polling GET /v1/audio/tts/{task_id}, so support both shapes.
|
||||
"""
|
||||
if hasattr(client, "post"):
|
||||
data = client.post(request["path"], request["payload"])
|
||||
payload = data.get("data") or {}
|
||||
task_id = payload.get("task_id")
|
||||
if not task_id:
|
||||
raise KlingAPIError(f"Kling TTS create response missing data.task_id: {data}")
|
||||
|
||||
task_result = payload.get("task_result") or {}
|
||||
outputs = task_result.get("audios")
|
||||
if outputs is not None:
|
||||
if not isinstance(outputs, list):
|
||||
raise KlingAPIError("Kling TTS result path data.task_result.audios is not a list")
|
||||
return str(task_id), outputs
|
||||
|
||||
status = payload.get("task_status") or payload.get("status")
|
||||
if status == "failed":
|
||||
message = payload.get("task_status_msg") or payload.get("message") or "Kling TTS task failed"
|
||||
raise KlingAPIError(str(message), code=payload.get("task_status"), response=data)
|
||||
|
||||
return str(task_id), client.poll_classic(
|
||||
request["path"],
|
||||
str(task_id),
|
||||
"audios",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 300)),
|
||||
poll_interval=float(inputs.get("poll_interval", 3.0)),
|
||||
)
|
||||
|
||||
task_id = client.create_classic_task(request["path"], request["payload"])
|
||||
return task_id, client.poll_classic(
|
||||
request["path"],
|
||||
task_id,
|
||||
"audios",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 300)),
|
||||
poll_interval=float(inputs.get("poll_interval", 3.0)),
|
||||
)
|
||||
|
||||
def _download_audios(
|
||||
self,
|
||||
client: KlingClient,
|
||||
outputs: list[dict[str, Any]],
|
||||
inputs: dict[str, Any],
|
||||
) -> list[Path]:
|
||||
if not outputs:
|
||||
raise ValueError("Kling TTS response contained no audios")
|
||||
base_path = Path(inputs.get("output_path", "kling_tts.mp3"))
|
||||
paths: list[Path] = []
|
||||
for index, item in enumerate(outputs):
|
||||
url = self._output_url(item)
|
||||
suffix = extension_from_url(url, ".mp3")
|
||||
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
|
||||
client.download(url, output_path)
|
||||
paths.append(output_path)
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _output_url(item: dict[str, Any]) -> str:
|
||||
url = item.get("url") or item.get("audio_url") or item.get("resource_url")
|
||||
if url:
|
||||
return str(url)
|
||||
resource = item.get("resource") or {}
|
||||
if isinstance(resource, dict) and resource.get("url"):
|
||||
return str(resource["url"])
|
||||
raise ValueError(f"Kling TTS response item contained no downloadable URL: {item}")
|
||||
|
||||
@staticmethod
|
||||
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
payload["external_task_id"] = inputs["external_task_id"]
|
||||
|
||||
@staticmethod
|
||||
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
callback_url = inputs.get("callback_url")
|
||||
if not callback_url:
|
||||
return {}
|
||||
return {
|
||||
"callback_url": str(callback_url),
|
||||
"callback_requested": True,
|
||||
"polling_used": True,
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
|
||||
if not inputs.get("include_account_usage"):
|
||||
return {}
|
||||
try:
|
||||
usage = get_account_costs(client=client)
|
||||
return {
|
||||
"account_usage": usage,
|
||||
"cost_source": "estimate_with_account_usage_context",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"account_usage_error": str(exc),
|
||||
"cost_source": "estimate",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
@@ -45,6 +45,17 @@ class TTSSelector(BaseTool):
|
||||
"type": "string",
|
||||
"description": "Provider-specific voice ID. Passed through to the selected TTS provider.",
|
||||
},
|
||||
"voice_language": {
|
||||
"type": "string",
|
||||
"enum": ["zh", "en"],
|
||||
"description": "Kling official voice language. Passed through when selected provider supports it.",
|
||||
},
|
||||
"voice_speed": {
|
||||
"type": "number",
|
||||
"minimum": 0.5,
|
||||
"maximum": 2.0,
|
||||
"description": "Kling official voice speed. Use speed for OpenAI/ElevenLabs-style controls.",
|
||||
},
|
||||
"model_id": {
|
||||
"type": "string",
|
||||
"description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.",
|
||||
|
||||
332
tools/avatar/kling_avatar.py
Normal file
332
tools/avatar/kling_avatar.py
Normal file
@@ -0,0 +1,332 @@
|
||||
"""Kling official API avatar image-to-video provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools._kling.account import account_usage_hint_for_error, get_account_costs
|
||||
from tools._kling.callbacks import validate_callback_url
|
||||
from tools._kling.client import KlingClient
|
||||
from tools._kling.errors import KlingAPIError
|
||||
from tools._kling.media import (
|
||||
extension_from_url,
|
||||
normalize_image_input,
|
||||
normalize_media_input,
|
||||
numbered_output_path,
|
||||
output_path_with_suffix,
|
||||
)
|
||||
from tools._kling.schemas import AVATAR_MODES
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
DependencyError,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
from tools.video._shared import probe_output
|
||||
|
||||
|
||||
class KlingAvatar(BaseTool):
|
||||
name = "kling_avatar"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.GENERATE
|
||||
capability = "avatar"
|
||||
provider = "kling_official"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:KLING_API_KEY"]
|
||||
install_instructions = (
|
||||
"Set KLING_API_KEY in .env for the official Kling API. "
|
||||
"Provide an avatar image plus either audio_id or sound_file/audio_path."
|
||||
)
|
||||
agent_skills = ["kling-official", "avatar-video"]
|
||||
|
||||
capabilities = ["photo_to_video", "avatar_video", "audio_driven_avatar"]
|
||||
supports = {
|
||||
"photo_to_video": True,
|
||||
"audio_driven_animation": True,
|
||||
"offline": False,
|
||||
"cloud_render": True,
|
||||
}
|
||||
best_for = [
|
||||
"official Kling cloud avatar presenter clips",
|
||||
"high-quality image-to-video avatar generation from a supplied portrait",
|
||||
"projects already using a Kling official account and resource pack",
|
||||
]
|
||||
not_good_for = [
|
||||
"fully offline avatar generation",
|
||||
"free local drafts",
|
||||
"silently replacing the local talking_head provider",
|
||||
]
|
||||
fallback_tools = ["talking_head", "lip_sync"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"anyOf": [
|
||||
{"required": ["image_url"]},
|
||||
{"required": ["image_path"]},
|
||||
],
|
||||
"allOf": [
|
||||
{
|
||||
"anyOf": [
|
||||
{"required": ["audio_id"]},
|
||||
{"required": ["sound_file"]},
|
||||
{"required": ["sound_file_url"]},
|
||||
{"required": ["sound_file_path"]},
|
||||
{"required": ["audio_path"]},
|
||||
]
|
||||
}
|
||||
],
|
||||
"properties": {
|
||||
"image_url": {"type": "string"},
|
||||
"image_path": {"type": "string"},
|
||||
"audio_id": {"type": "string"},
|
||||
"sound_file": {
|
||||
"type": "string",
|
||||
"description": "Official Kling sound_file value or raw base64 audio.",
|
||||
},
|
||||
"sound_file_url": {"type": "string"},
|
||||
"sound_file_path": {"type": "string"},
|
||||
"audio_path": {
|
||||
"type": "string",
|
||||
"description": "Alias for sound_file_path for compatibility with local avatar tools.",
|
||||
},
|
||||
"prompt": {"type": "string"},
|
||||
"mode": {"type": "string", "enum": AVATAR_MODES, "default": "std"},
|
||||
"callback_url": {"type": "string"},
|
||||
"external_task_id": {"type": "string"},
|
||||
"include_account_usage": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Optional low-frequency account usage diagnostic; not used by default.",
|
||||
},
|
||||
"timeout_seconds": {"type": "integer", "default": 900},
|
||||
"poll_interval": {"type": "number", "default": 5.0},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=2,
|
||||
backoff_seconds=2.0,
|
||||
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
|
||||
)
|
||||
idempotency_key_fields = ["image_url", "image_path", "audio_id", "sound_file", "sound_file_path", "mode"]
|
||||
side_effects = ["paid remote generation via official Kling API", "writes avatar video to output_path"]
|
||||
user_visible_verification = ["Watch generated avatar video for identity preservation and mouth motion"]
|
||||
quality_score = 0.82
|
||||
latency_p50_seconds = 240.0
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
base = 0.35
|
||||
if inputs.get("mode") == "pro":
|
||||
base *= 1.7
|
||||
if inputs.get("sound_file_path") or inputs.get("audio_path"):
|
||||
base += 0.04
|
||||
return round(base, 4)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 240.0
|
||||
|
||||
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
result = super().dry_run(inputs)
|
||||
result.update(
|
||||
{
|
||||
"paid_api": True,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative OpenMontage avatar estimate pending official account-usage reconciliation.",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
try:
|
||||
self.check_dependencies()
|
||||
except DependencyError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
request = self._build_request(inputs)
|
||||
client = KlingClient()
|
||||
task_id = client.create_classic_task(request["path"], request["payload"])
|
||||
outputs = client.poll_classic(
|
||||
request["path"],
|
||||
task_id,
|
||||
"videos",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
|
||||
poll_interval=float(inputs.get("poll_interval", 5.0)),
|
||||
)
|
||||
paths = self._download_videos(client, outputs, inputs)
|
||||
probed = probe_output(paths[0])
|
||||
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
|
||||
data: dict[str, Any] = {"provider": self.provider}
|
||||
if isinstance(exc, KlingAPIError):
|
||||
data.update(
|
||||
{
|
||||
"error_code": exc.code,
|
||||
"request_id": exc.request_id,
|
||||
"http_status": exc.http_status,
|
||||
"account_usage_diagnostic": account_usage_hint_for_error(exc),
|
||||
}
|
||||
)
|
||||
return ToolResult(success=False, data=data, error=f"Kling official avatar generation failed: {exc}")
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official avatar generation failed: {exc}")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": "kling-official-avatar",
|
||||
"task_id": task_id,
|
||||
"operation": "image_to_avatar_video",
|
||||
"mode": request["payload"].get("mode"),
|
||||
"prompt": request["payload"].get("prompt"),
|
||||
"avatar_source": request["avatar_source"],
|
||||
"audio_source": request["audio_source"],
|
||||
"remote_outputs": outputs,
|
||||
"output": str(paths[0]),
|
||||
"output_path": str(paths[0]),
|
||||
"video_paths": [str(path) for path in paths],
|
||||
"format": "mp4",
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
|
||||
**self._account_usage_result(inputs, client),
|
||||
**self._callback_result_data(inputs, task_id),
|
||||
**probed,
|
||||
},
|
||||
artifacts=[str(path) for path in paths],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model="kling-official-avatar",
|
||||
)
|
||||
|
||||
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
image = normalize_image_input(inputs.get("image_url"), inputs.get("image_path"))
|
||||
if not image:
|
||||
raise ValueError("Kling avatar requires image_url or image_path")
|
||||
|
||||
mode = str(inputs.get("mode") or "std")
|
||||
if mode not in AVATAR_MODES:
|
||||
raise ValueError(f"mode must be one of: {', '.join(AVATAR_MODES)}")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"image": image,
|
||||
"mode": mode,
|
||||
}
|
||||
if inputs.get("prompt"):
|
||||
payload["prompt"] = str(inputs["prompt"])
|
||||
|
||||
audio_source = self._copy_audio_input(inputs, payload)
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/videos/avatar/image2video",
|
||||
"payload": payload,
|
||||
"operation": "image_to_avatar_video",
|
||||
"model": "kling-official-avatar",
|
||||
"avatar_source": inputs.get("image_url") or inputs.get("image_path"),
|
||||
"audio_source": audio_source,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
||||
audio_id = str(inputs.get("audio_id") or "").strip()
|
||||
if audio_id:
|
||||
payload["audio_id"] = audio_id
|
||||
return {"type": "audio_id", "value": audio_id}
|
||||
|
||||
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
|
||||
sound_file = normalize_media_input(
|
||||
url=inputs.get("sound_file_url"),
|
||||
path=sound_path,
|
||||
value=inputs.get("sound_file"),
|
||||
label="Avatar audio file",
|
||||
)
|
||||
if not sound_file:
|
||||
raise ValueError("Kling avatar requires audio_id, sound_file, sound_file_url, sound_file_path, or audio_path")
|
||||
payload["sound_file"] = sound_file
|
||||
return {
|
||||
"type": "sound_file",
|
||||
"source": inputs.get("sound_file_url") or sound_path or "inline",
|
||||
}
|
||||
|
||||
def _download_videos(
|
||||
self,
|
||||
client: KlingClient,
|
||||
outputs: list[dict[str, Any]],
|
||||
inputs: dict[str, Any],
|
||||
) -> list[Path]:
|
||||
if not outputs:
|
||||
raise ValueError("Kling avatar response contained no videos")
|
||||
base_path = Path(inputs.get("output_path", "kling_avatar.mp4"))
|
||||
paths: list[Path] = []
|
||||
for index, item in enumerate(outputs):
|
||||
url = self._output_url(item)
|
||||
suffix = extension_from_url(url, ".mp4")
|
||||
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
|
||||
client.download(url, output_path)
|
||||
paths.append(output_path)
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _output_url(item: dict[str, Any]) -> str:
|
||||
url = item.get("url") or item.get("video_url") or item.get("resource_url")
|
||||
if url:
|
||||
return str(url)
|
||||
resource = item.get("resource") or {}
|
||||
if isinstance(resource, dict) and resource.get("url"):
|
||||
return str(resource["url"])
|
||||
raise ValueError(f"Kling avatar response item contained no downloadable URL: {item}")
|
||||
|
||||
@staticmethod
|
||||
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
payload["external_task_id"] = inputs["external_task_id"]
|
||||
|
||||
@staticmethod
|
||||
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
callback_url = inputs.get("callback_url")
|
||||
if not callback_url:
|
||||
return {}
|
||||
return {
|
||||
"callback_url": str(callback_url),
|
||||
"callback_requested": True,
|
||||
"polling_used": True,
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
|
||||
if not inputs.get("include_account_usage"):
|
||||
return {}
|
||||
try:
|
||||
usage = get_account_costs(client=client)
|
||||
return {
|
||||
"account_usage": usage,
|
||||
"cost_source": "estimate_with_account_usage_context",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"account_usage_error": str(exc),
|
||||
"cost_source": "estimate",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
551
tools/avatar/kling_lip_sync.py
Normal file
551
tools/avatar/kling_lip_sync.py
Normal file
@@ -0,0 +1,551 @@
|
||||
"""Kling official API lip-sync provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools._kling.account import account_usage_hint_for_error, get_account_costs
|
||||
from tools._kling.callbacks import validate_callback_url
|
||||
from tools._kling.client import KlingClient
|
||||
from tools._kling.errors import KlingAPIError
|
||||
from tools._kling.media import (
|
||||
extension_from_url,
|
||||
normalize_media_input,
|
||||
numbered_output_path,
|
||||
output_path_with_suffix,
|
||||
)
|
||||
from tools._kling.schemas import LIP_SYNC_OPERATIONS
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
DependencyError,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
from tools.video._shared import probe_output
|
||||
|
||||
|
||||
class KlingLipSync(BaseTool):
|
||||
name = "kling_lip_sync"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.GENERATE
|
||||
capability = "avatar"
|
||||
provider = "kling_official"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:KLING_API_KEY"]
|
||||
install_instructions = (
|
||||
"Set KLING_API_KEY in .env for the official Kling API. "
|
||||
"Use identify_face first for multi-person clips, then pass face_choose or face_id."
|
||||
)
|
||||
agent_skills = ["kling-official", "avatar-video"]
|
||||
|
||||
capabilities = ["lip_sync", "identify_face", "audio_video_alignment"]
|
||||
supports = {
|
||||
"lip_sync": True,
|
||||
"face_selection": True,
|
||||
"offline": False,
|
||||
"cloud_render": True,
|
||||
}
|
||||
best_for = [
|
||||
"official Kling cloud lip-sync for existing presenter video",
|
||||
"dubbing workflows that can use Kling face identification",
|
||||
"manual or explicit automatic face selection before paid lip-sync generation",
|
||||
]
|
||||
not_good_for = [
|
||||
"fully offline lip-sync",
|
||||
"silent first-face selection in multi-person footage",
|
||||
"replacing local lip_sync behavior implicitly",
|
||||
]
|
||||
fallback_tools = ["lip_sync"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {"type": "string", "enum": LIP_SYNC_OPERATIONS, "default": "advanced_lip_sync"},
|
||||
"video_id": {"type": "string"},
|
||||
"video_url": {"type": "string"},
|
||||
"video_path": {
|
||||
"type": "string",
|
||||
"description": "Not silently uploaded. Provide video_url unless an official upload path is added.",
|
||||
},
|
||||
"session_id": {"type": "string"},
|
||||
"face_id": {"type": "string"},
|
||||
"face_choose": {"type": "array"},
|
||||
"auto_select_face": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Explicitly allow largest-face automatic selection after identify_face.",
|
||||
},
|
||||
"audio_id": {"type": "string"},
|
||||
"sound_file": {
|
||||
"type": "string",
|
||||
"description": "Official Kling sound_file value or raw base64 audio.",
|
||||
},
|
||||
"sound_file_url": {"type": "string"},
|
||||
"sound_file_path": {"type": "string"},
|
||||
"audio_path": {
|
||||
"type": "string",
|
||||
"description": "Alias for sound_file_path for compatibility with local lip_sync.",
|
||||
},
|
||||
"faces_artifact_path": {"type": "string"},
|
||||
"callback_url": {"type": "string"},
|
||||
"external_task_id": {"type": "string"},
|
||||
"include_account_usage": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Optional low-frequency account usage diagnostic; not used by default.",
|
||||
},
|
||||
"timeout_seconds": {"type": "integer", "default": 900},
|
||||
"poll_interval": {"type": "number", "default": 5.0},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=2,
|
||||
backoff_seconds=2.0,
|
||||
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
|
||||
)
|
||||
idempotency_key_fields = ["video_id", "video_url", "session_id", "face_id", "audio_id", "sound_file_path"]
|
||||
side_effects = [
|
||||
"paid remote generation via official Kling API",
|
||||
"writes face selection artifact",
|
||||
"writes lip-synced video to output_path",
|
||||
]
|
||||
user_visible_verification = ["Watch output video to verify the selected face matches the new audio"]
|
||||
quality_score = 0.80
|
||||
latency_p50_seconds = 240.0
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
operation = str(inputs.get("operation", "advanced_lip_sync"))
|
||||
if operation == "identify_face":
|
||||
return 0.02
|
||||
if operation == "full_lip_sync":
|
||||
return 0.34
|
||||
return 0.32
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
if inputs.get("operation") == "identify_face":
|
||||
return 15.0
|
||||
return 240.0
|
||||
|
||||
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
result = super().dry_run(inputs)
|
||||
result.update(
|
||||
{
|
||||
"paid_api": True,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative OpenMontage estimate for identify-face plus advanced lip-sync.",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
try:
|
||||
self.check_dependencies()
|
||||
except DependencyError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
operation = str(inputs.get("operation") or "advanced_lip_sync")
|
||||
start = time.time()
|
||||
client = KlingClient()
|
||||
try:
|
||||
if operation == "identify_face":
|
||||
identify = self._identify_faces(client, inputs)
|
||||
return self._identify_result(inputs, identify, start)
|
||||
if operation == "full_lip_sync":
|
||||
identify = self._identify_faces(client, inputs)
|
||||
artifact_path = self._write_faces_artifact(inputs, identify)
|
||||
face_choose, selection = self._face_selection(identify["faces"], inputs)
|
||||
artifact_path = self._write_faces_artifact(inputs, identify, selection=selection)
|
||||
if selection["selection_method"] == "requires_user_selection":
|
||||
return ToolResult(
|
||||
success=False,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"operation": operation,
|
||||
"session_id": identify["session_id"],
|
||||
"faces": identify["faces"],
|
||||
"requires_face_selection": True,
|
||||
"selection_reason": selection["selection_reason"],
|
||||
"faces_artifact_path": str(artifact_path),
|
||||
},
|
||||
artifacts=[str(artifact_path)],
|
||||
error="Multiple faces detected. Pass face_id/face_choose or set auto_select_face=True.",
|
||||
cost_usd=self.estimate_cost({"operation": "identify_face"}),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model="kling-official-lip-sync",
|
||||
)
|
||||
merged = {**inputs, "session_id": identify["session_id"], "face_choose": face_choose}
|
||||
request = self._build_advanced_request(merged)
|
||||
result = self._run_advanced_lip_sync(client, merged, request, start)
|
||||
result.data["faces_artifact_path"] = str(artifact_path)
|
||||
result.data["face_selection"] = selection
|
||||
result.artifacts.append(str(artifact_path))
|
||||
return result
|
||||
if operation == "advanced_lip_sync":
|
||||
request = self._build_advanced_request(inputs)
|
||||
return self._run_advanced_lip_sync(client, inputs, request, start)
|
||||
raise ValueError(f"Unsupported Kling lip-sync operation: {operation}")
|
||||
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
|
||||
data: dict[str, Any] = {"provider": self.provider}
|
||||
if isinstance(exc, KlingAPIError):
|
||||
data.update(
|
||||
{
|
||||
"error_code": exc.code,
|
||||
"request_id": exc.request_id,
|
||||
"http_status": exc.http_status,
|
||||
"account_usage_diagnostic": account_usage_hint_for_error(exc),
|
||||
}
|
||||
)
|
||||
return ToolResult(success=False, data=data, error=f"Kling official lip-sync failed: {exc}")
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official lip-sync failed: {exc}")
|
||||
|
||||
def _identify_faces(self, client: KlingClient, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
request = self._build_identify_request(inputs)
|
||||
data = client.post(request["path"], request["payload"])
|
||||
payload = data.get("data") or {}
|
||||
session_id = payload.get("session_id")
|
||||
if not session_id:
|
||||
raise ValueError(f"Kling identify-face response missing data.session_id: {data}")
|
||||
faces = (
|
||||
payload.get("faces")
|
||||
or payload.get("face_list")
|
||||
or payload.get("face_infos")
|
||||
or payload.get("faces_info")
|
||||
or []
|
||||
)
|
||||
if not isinstance(faces, list):
|
||||
raise ValueError("Kling identify-face response face list is not a list")
|
||||
if not faces:
|
||||
raise ValueError("Kling identify-face response contained no faces")
|
||||
return {
|
||||
"session_id": str(session_id),
|
||||
"faces": faces,
|
||||
"raw_response": data,
|
||||
"request": request,
|
||||
}
|
||||
|
||||
def _identify_result(self, inputs: dict[str, Any], identify: dict[str, Any], start: float) -> ToolResult:
|
||||
artifact_path = self._write_faces_artifact(inputs, identify)
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": "kling-official-lip-sync",
|
||||
"operation": "identify_face",
|
||||
"session_id": identify["session_id"],
|
||||
"faces": identify["faces"],
|
||||
"face_count": len(identify["faces"]),
|
||||
"faces_artifact_path": str(artifact_path),
|
||||
},
|
||||
artifacts=[str(artifact_path)],
|
||||
cost_usd=self.estimate_cost({"operation": "identify_face"}),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model="kling-official-lip-sync",
|
||||
)
|
||||
|
||||
def _run_advanced_lip_sync(
|
||||
self,
|
||||
client: KlingClient,
|
||||
inputs: dict[str, Any],
|
||||
request: dict[str, Any],
|
||||
start: float,
|
||||
) -> ToolResult:
|
||||
task_id = client.create_classic_task(request["path"], request["payload"])
|
||||
outputs = client.poll_classic(
|
||||
request["path"],
|
||||
task_id,
|
||||
"videos",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
|
||||
poll_interval=float(inputs.get("poll_interval", 5.0)),
|
||||
)
|
||||
paths = self._download_videos(client, outputs, inputs)
|
||||
probed = probe_output(paths[0])
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": "kling-official-lip-sync",
|
||||
"task_id": task_id,
|
||||
"operation": request["operation"],
|
||||
"session_id": request["payload"]["session_id"],
|
||||
"face_choose": request["payload"]["face_choose"],
|
||||
"audio_source": request["audio_source"],
|
||||
"remote_outputs": outputs,
|
||||
"output": str(paths[0]),
|
||||
"output_path": str(paths[0]),
|
||||
"video_paths": [str(path) for path in paths],
|
||||
"format": "mp4",
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
|
||||
**self._account_usage_result(inputs, client),
|
||||
**self._callback_result_data(inputs, task_id),
|
||||
**probed,
|
||||
},
|
||||
artifacts=[str(path) for path in paths],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model="kling-official-lip-sync",
|
||||
)
|
||||
|
||||
def _build_identify_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
if inputs.get("video_path") and not (inputs.get("video_url") or inputs.get("video_id")):
|
||||
raise ValueError("Kling identify_face requires video_url or video_id; local video paths cannot be silently uploaded.")
|
||||
payload: dict[str, Any] = {}
|
||||
if inputs.get("video_id"):
|
||||
payload["video_id"] = str(inputs["video_id"])
|
||||
if inputs.get("video_url"):
|
||||
payload["video_url"] = str(inputs["video_url"])
|
||||
if not payload:
|
||||
raise ValueError("Kling identify_face requires video_id or video_url")
|
||||
return {
|
||||
"path": "/v1/videos/identify-face",
|
||||
"payload": payload,
|
||||
"operation": "identify_face",
|
||||
}
|
||||
|
||||
def _build_advanced_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
session_id = str(inputs.get("session_id") or "").strip()
|
||||
if not session_id:
|
||||
raise ValueError("advanced_lip_sync requires session_id")
|
||||
face_choose = self._normalize_face_choose(inputs)
|
||||
if not face_choose:
|
||||
raise ValueError("advanced_lip_sync requires face_choose or face_id")
|
||||
payload: dict[str, Any] = {
|
||||
"session_id": session_id,
|
||||
"face_choose": face_choose,
|
||||
}
|
||||
audio_source = self._copy_audio_input(inputs, payload)
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/videos/advanced-lip-sync",
|
||||
"payload": payload,
|
||||
"operation": "advanced_lip_sync",
|
||||
"model": "kling-official-lip-sync",
|
||||
"audio_source": audio_source,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_face_choose(inputs: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if inputs.get("face_choose"):
|
||||
raw = inputs["face_choose"]
|
||||
if isinstance(raw, dict):
|
||||
raw = [raw]
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("face_choose must be a list of face choice objects")
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in raw:
|
||||
if isinstance(item, str):
|
||||
normalized.append({"face_id": item})
|
||||
elif isinstance(item, dict):
|
||||
if not (item.get("face_id") or item.get("id")):
|
||||
raise ValueError("face_choose items must include face_id")
|
||||
record = dict(item)
|
||||
if "face_id" not in record and record.get("id"):
|
||||
record["face_id"] = record.pop("id")
|
||||
normalized.append(record)
|
||||
else:
|
||||
raise ValueError("face_choose items must be strings or objects")
|
||||
return normalized
|
||||
if inputs.get("face_id"):
|
||||
return [{"face_id": str(inputs["face_id"])}]
|
||||
return []
|
||||
|
||||
def _face_selection(
|
||||
self,
|
||||
faces: list[dict[str, Any]],
|
||||
inputs: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
|
||||
explicit = self._normalize_face_choose(inputs)
|
||||
if explicit:
|
||||
return explicit, {
|
||||
"selection_method": "user_selected",
|
||||
"selection_reason": "face_choose or face_id was provided",
|
||||
"selected_face": explicit,
|
||||
}
|
||||
if len(faces) == 1:
|
||||
choice = [self._face_to_choice(faces[0])]
|
||||
return choice, {
|
||||
"selection_method": "single_face",
|
||||
"selection_reason": "Only one face was returned by identify_face",
|
||||
"selected_face": choice,
|
||||
}
|
||||
if not inputs.get("auto_select_face"):
|
||||
return [], {
|
||||
"selection_method": "requires_user_selection",
|
||||
"selection_reason": "Multiple faces detected and auto_select_face was not enabled",
|
||||
"face_count": len(faces),
|
||||
}
|
||||
selected = max(faces, key=self._face_area)
|
||||
choice = [self._face_to_choice(selected)]
|
||||
return choice, {
|
||||
"selection_method": "auto_selected",
|
||||
"selection_reason": "auto_select_face=True selected the largest detected face area",
|
||||
"selected_face": choice,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _face_to_choice(face: dict[str, Any]) -> dict[str, Any]:
|
||||
face_id = face.get("face_id") or face.get("id")
|
||||
if not face_id:
|
||||
raise ValueError(f"Cannot select face without face_id/id: {face}")
|
||||
return {"face_id": str(face_id)}
|
||||
|
||||
@staticmethod
|
||||
def _face_area(face: dict[str, Any]) -> float:
|
||||
for key in ("bbox", "box"):
|
||||
value = face.get(key)
|
||||
if isinstance(value, list) and len(value) >= 4:
|
||||
third = float(value[2])
|
||||
fourth = float(value[3])
|
||||
width_height_area = max(third, 0.0) * max(fourth, 0.0)
|
||||
corner_width = third - float(value[0])
|
||||
corner_height = fourth - float(value[1])
|
||||
corner_area = (
|
||||
corner_width * corner_height
|
||||
if corner_width > 0 and corner_height > 0
|
||||
else 0.0
|
||||
)
|
||||
if corner_area and width_height_area:
|
||||
return min(corner_area, width_height_area)
|
||||
return corner_area or width_height_area
|
||||
if isinstance(value, dict):
|
||||
width = value.get("width") or value.get("w")
|
||||
height = value.get("height") or value.get("h")
|
||||
if width is not None and height is not None:
|
||||
return max(float(width), 0.0) * max(float(height), 0.0)
|
||||
width = face.get("width") or face.get("w")
|
||||
height = face.get("height") or face.get("h")
|
||||
if width is not None and height is not None:
|
||||
return max(float(width), 0.0) * max(float(height), 0.0)
|
||||
return 0.0
|
||||
|
||||
@staticmethod
|
||||
def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
|
||||
audio_id = str(inputs.get("audio_id") or "").strip()
|
||||
if audio_id:
|
||||
payload["audio_id"] = audio_id
|
||||
return {"type": "audio_id", "value": audio_id}
|
||||
|
||||
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
|
||||
sound_file = normalize_media_input(
|
||||
url=inputs.get("sound_file_url"),
|
||||
path=sound_path,
|
||||
value=inputs.get("sound_file"),
|
||||
label="Lip-sync audio file",
|
||||
)
|
||||
if not sound_file:
|
||||
raise ValueError("advanced_lip_sync requires audio_id, sound_file, sound_file_url, sound_file_path, or audio_path")
|
||||
payload["sound_file"] = sound_file
|
||||
return {
|
||||
"type": "sound_file",
|
||||
"source": inputs.get("sound_file_url") or sound_path or "inline",
|
||||
}
|
||||
|
||||
def _download_videos(
|
||||
self,
|
||||
client: KlingClient,
|
||||
outputs: list[dict[str, Any]],
|
||||
inputs: dict[str, Any],
|
||||
) -> list[Path]:
|
||||
if not outputs:
|
||||
raise ValueError("Kling lip-sync response contained no videos")
|
||||
base_path = Path(inputs.get("output_path", "kling_lip_sync.mp4"))
|
||||
paths: list[Path] = []
|
||||
for index, item in enumerate(outputs):
|
||||
url = self._output_url(item)
|
||||
suffix = extension_from_url(url, ".mp4")
|
||||
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
|
||||
client.download(url, output_path)
|
||||
paths.append(output_path)
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _output_url(item: dict[str, Any]) -> str:
|
||||
url = item.get("url") or item.get("video_url") or item.get("resource_url")
|
||||
if url:
|
||||
return str(url)
|
||||
resource = item.get("resource") or {}
|
||||
if isinstance(resource, dict) and resource.get("url"):
|
||||
return str(resource["url"])
|
||||
raise ValueError(f"Kling lip-sync response item contained no downloadable URL: {item}")
|
||||
|
||||
@staticmethod
|
||||
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
payload["external_task_id"] = inputs["external_task_id"]
|
||||
|
||||
@staticmethod
|
||||
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
callback_url = inputs.get("callback_url")
|
||||
if not callback_url:
|
||||
return {}
|
||||
return {
|
||||
"callback_url": str(callback_url),
|
||||
"callback_requested": True,
|
||||
"polling_used": True,
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
|
||||
if not inputs.get("include_account_usage"):
|
||||
return {}
|
||||
try:
|
||||
usage = get_account_costs(client=client)
|
||||
return {
|
||||
"account_usage": usage,
|
||||
"cost_source": "estimate_with_account_usage_context",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"account_usage_error": str(exc),
|
||||
"cost_source": "estimate",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
|
||||
def _write_faces_artifact(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
identify: dict[str, Any],
|
||||
selection: dict[str, Any] | None = None,
|
||||
) -> Path:
|
||||
if inputs.get("faces_artifact_path"):
|
||||
artifact_path = Path(inputs["faces_artifact_path"])
|
||||
elif inputs.get("output_path"):
|
||||
artifact_path = Path(inputs["output_path"]).with_name("kling_lip_sync_faces.json")
|
||||
else:
|
||||
artifact_path = Path("kling_lip_sync_faces.json")
|
||||
artifact_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
artifact = {
|
||||
"provider": self.provider,
|
||||
"operation": "identify_face",
|
||||
"session_id": identify["session_id"],
|
||||
"faces": identify["faces"],
|
||||
"face_count": len(identify["faces"]),
|
||||
"selection": selection,
|
||||
}
|
||||
artifact_path.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return artifact_path
|
||||
@@ -61,6 +61,14 @@ class ImageSelector(BaseTool):
|
||||
"type": "string",
|
||||
"description": "Resolution tier for providers that support named resolutions.",
|
||||
},
|
||||
"api_family": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific API family hint passed through when supported.",
|
||||
},
|
||||
"model_name": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific model name passed through when supported.",
|
||||
},
|
||||
"generation_mode": {
|
||||
"type": "string",
|
||||
"enum": ["generate", "edit"],
|
||||
@@ -79,6 +87,46 @@ class ImageSelector(BaseTool):
|
||||
"items": {"type": "string"},
|
||||
"description": "Multiple local source image paths for compositing edits.",
|
||||
},
|
||||
"image_list": {
|
||||
"type": "array",
|
||||
"description": "Provider-specific image reference list, e.g. Kling Official Image Omni.",
|
||||
},
|
||||
"element_list": {
|
||||
"type": "array",
|
||||
"description": "Provider-specific element references, e.g. Kling Official element_id objects.",
|
||||
},
|
||||
"image_reference": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific reference type, e.g. subject or face.",
|
||||
},
|
||||
"image_fidelity": {
|
||||
"type": "number",
|
||||
"description": "Provider-specific reference image fidelity hint.",
|
||||
},
|
||||
"human_fidelity": {
|
||||
"type": "number",
|
||||
"description": "Provider-specific human or face fidelity hint.",
|
||||
},
|
||||
"result_type": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific result type, e.g. single or series.",
|
||||
},
|
||||
"series_amount": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific series amount for image series generation.",
|
||||
},
|
||||
"watermark": {
|
||||
"type": "boolean",
|
||||
"description": "Provider-specific watermark toggle passed through when supported.",
|
||||
},
|
||||
"callback_url": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific callback URL. Current OpenMontage providers still poll by default.",
|
||||
},
|
||||
"external_task_id": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific idempotency/provenance task id.",
|
||||
},
|
||||
"preferred_provider": {
|
||||
"type": "string",
|
||||
"description": "Provider name or 'auto'. Valid values are discovered at runtime from the registry.",
|
||||
@@ -216,6 +264,18 @@ class ImageSelector(BaseTool):
|
||||
"image_path",
|
||||
"image_urls",
|
||||
"image_paths",
|
||||
"image_list",
|
||||
"element_list",
|
||||
"api_family",
|
||||
"model_name",
|
||||
"image_reference",
|
||||
"image_fidelity",
|
||||
"human_fidelity",
|
||||
"result_type",
|
||||
"series_amount",
|
||||
"watermark",
|
||||
"callback_url",
|
||||
"external_task_id",
|
||||
"workflow_json",
|
||||
"workflow_path",
|
||||
"output_node",
|
||||
|
||||
437
tools/graphics/kling_official_image.py
Normal file
437
tools/graphics/kling_official_image.py
Normal file
@@ -0,0 +1,437 @@
|
||||
"""Kling official API image generation provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools._kling.account import account_usage_hint_for_error, get_account_costs
|
||||
from tools._kling.callbacks import validate_callback_url
|
||||
from tools._kling.client import KlingClient
|
||||
from tools._kling.elements import element_ids, normalize_element_list
|
||||
from tools._kling.errors import KlingAPIError
|
||||
from tools._kling.media import (
|
||||
extension_from_url,
|
||||
normalize_image_input,
|
||||
numbered_output_path,
|
||||
output_path_with_suffix,
|
||||
)
|
||||
from tools._kling.omni import build_image_prompt_references
|
||||
from tools._kling.schemas import (
|
||||
IMAGE_ASPECT_RATIOS,
|
||||
IMAGE_GENERATION_MODELS,
|
||||
IMAGE_MODELS,
|
||||
IMAGE_REFERENCE_TYPES,
|
||||
IMAGE_RESOLUTIONS,
|
||||
IMAGE_RESULT_TYPES,
|
||||
OMNI_IMAGE_MODELS,
|
||||
)
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
DependencyError,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class KlingOfficialImage(BaseTool):
|
||||
name = "kling_official_image"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.GENERATE
|
||||
capability = "image_generation"
|
||||
provider = "kling_official"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:KLING_API_KEY"]
|
||||
install_instructions = (
|
||||
"Set KLING_API_KEY in .env for the official Kling API. "
|
||||
"Optionally set KLING_API_BASE_URL to override the default Singapore endpoint."
|
||||
)
|
||||
agent_skills = ["kling-official"]
|
||||
|
||||
capabilities = ["generate_image", "text_to_image", "image_edit"]
|
||||
supports = {
|
||||
"text_to_image": True,
|
||||
"image_edit": True,
|
||||
"negative_prompt": True,
|
||||
"aspect_ratio": True,
|
||||
}
|
||||
best_for = [
|
||||
"official Kling image generation",
|
||||
"subject or face reference generation",
|
||||
"Omni multi-reference image workflows",
|
||||
]
|
||||
not_good_for = ["offline generation", "free generation", "non-Kling model families"]
|
||||
fallback_tools = ["flux_image", "google_imagen", "openai_image", "recraft_image"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["prompt"],
|
||||
"properties": {
|
||||
"prompt": {"type": "string"},
|
||||
"negative_prompt": {"type": "string"},
|
||||
"operation": {"type": "string", "enum": ["generate", "omni"], "default": "generate"},
|
||||
"generation_mode": {"type": "string", "enum": ["generate", "edit"], "default": "generate"},
|
||||
"api_family": {"type": "string", "enum": ["generation", "omni"], "default": "generation"},
|
||||
"model_name": {"type": "string", "enum": IMAGE_MODELS, "default": "kling-v3"},
|
||||
"image_url": {"type": "string"},
|
||||
"image_path": {"type": "string"},
|
||||
"image_urls": {"type": "array", "items": {"type": "string"}},
|
||||
"image_paths": {"type": "array", "items": {"type": "string"}},
|
||||
"image_list": {"type": "array"},
|
||||
"image_reference": {"type": "string", "enum": IMAGE_REFERENCE_TYPES},
|
||||
"image_fidelity": {"type": "number", "default": 0.5},
|
||||
"human_fidelity": {"type": "number", "default": 0.45},
|
||||
"resolution": {"type": "string", "enum": IMAGE_RESOLUTIONS, "default": "1k"},
|
||||
"aspect_ratio": {"type": "string", "enum": IMAGE_ASPECT_RATIOS, "default": "16:9"},
|
||||
"n": {"type": "integer", "default": 1},
|
||||
"result_type": {"type": "string", "enum": IMAGE_RESULT_TYPES, "default": "single"},
|
||||
"series_amount": {"type": "string"},
|
||||
"element_list": {"type": "array"},
|
||||
"watermark": {"type": "boolean", "default": False},
|
||||
"callback_url": {"type": "string"},
|
||||
"external_task_id": {"type": "string"},
|
||||
"include_account_usage": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Optional low-frequency account usage diagnostic; not used by default.",
|
||||
},
|
||||
"timeout_seconds": {"type": "integer", "default": 600},
|
||||
"poll_interval": {"type": "number", "default": 3.0},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=200, network_required=True
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=2,
|
||||
backoff_seconds=2.0,
|
||||
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
|
||||
)
|
||||
idempotency_key_fields = [
|
||||
"prompt",
|
||||
"api_family",
|
||||
"model_name",
|
||||
"image_url",
|
||||
"image_path",
|
||||
"aspect_ratio",
|
||||
"resolution",
|
||||
"n",
|
||||
]
|
||||
side_effects = [
|
||||
"paid remote generation via official Kling API",
|
||||
"writes image file(s) to output_path",
|
||||
]
|
||||
user_visible_verification = ["Inspect generated image for quality, prompt adherence, and reference fidelity"]
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
n = int(inputs.get("n", 1) or 1)
|
||||
resolution = str(inputs.get("resolution", "1k"))
|
||||
api_family = str(inputs.get("api_family", "generation"))
|
||||
base = 0.04 if api_family == "generation" else 0.08
|
||||
if resolution == "2k":
|
||||
base *= 1.8
|
||||
if resolution == "4k":
|
||||
base *= 3.5
|
||||
if inputs.get("result_type") == "series":
|
||||
base *= 1.5
|
||||
amount = inputs.get("series_amount")
|
||||
if amount and str(amount).isdigit():
|
||||
base *= max(int(str(amount)), 1)
|
||||
if api_family == "omni":
|
||||
reference_count = sum(len(inputs.get(key) or []) for key in ("image_list", "image_urls", "image_paths", "element_list"))
|
||||
if inputs.get("image_url") or inputs.get("image_path"):
|
||||
reference_count += 1
|
||||
base *= 1 + (0.08 * reference_count)
|
||||
return round(base * max(n, 1), 4)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 90.0
|
||||
|
||||
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
result = super().dry_run(inputs)
|
||||
result.update(
|
||||
{
|
||||
"paid_api": True,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative OpenMontage estimate; official account usage reconciliation is planned for Phase 2.",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
try:
|
||||
self.check_dependencies()
|
||||
except DependencyError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
request = self._build_request(inputs)
|
||||
client = KlingClient()
|
||||
task_id = client.create_classic_task(request["path"], request["payload"])
|
||||
outputs = client.poll_classic(
|
||||
request["path"],
|
||||
task_id,
|
||||
"images",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 600)),
|
||||
poll_interval=float(inputs.get("poll_interval", 3.0)),
|
||||
)
|
||||
paths = self._download_images(client, outputs, inputs)
|
||||
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
|
||||
data: dict[str, Any] = {"provider": self.provider}
|
||||
if isinstance(exc, KlingAPIError):
|
||||
data.update(
|
||||
{
|
||||
"error_code": exc.code,
|
||||
"request_id": exc.request_id,
|
||||
"http_status": exc.http_status,
|
||||
}
|
||||
)
|
||||
data["account_usage_diagnostic"] = account_usage_hint_for_error(exc)
|
||||
return ToolResult(success=False, data=data, error=f"Kling official image generation failed: {exc}")
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official image generation failed: {exc}")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": request["model"],
|
||||
"task_id": task_id,
|
||||
"api_family": request["api_family"],
|
||||
"operation": request["operation"],
|
||||
"prompt": request["payload"]["prompt"],
|
||||
"remote_outputs": outputs,
|
||||
"output": str(paths[0]),
|
||||
"output_path": str(paths[0]),
|
||||
"image_paths": [str(path) for path in paths],
|
||||
"format": paths[0].suffix.lstrip(".") or "png",
|
||||
"references_used": request.get("references_used", []),
|
||||
"element_ids": request.get("element_ids", []),
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
|
||||
**self._account_usage_result(inputs, client),
|
||||
**self._callback_result_data(inputs, task_id),
|
||||
},
|
||||
artifacts=[str(path) for path in paths],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model=request["model"],
|
||||
)
|
||||
|
||||
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
api_family = str(inputs.get("api_family", "generation"))
|
||||
if inputs.get("operation") == "omni":
|
||||
api_family = "omni"
|
||||
if api_family == "omni":
|
||||
return self._build_omni_request(inputs)
|
||||
return self._build_generation_request(inputs)
|
||||
|
||||
def _build_generation_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
prompt = self._prompt(inputs)
|
||||
model_name = str(inputs.get("model_name") or "kling-v3")
|
||||
if model_name not in IMAGE_GENERATION_MODELS:
|
||||
raise ValueError(f"model_name {model_name!r} is not supported for api_family=generation")
|
||||
payload: dict[str, Any] = {
|
||||
"model_name": model_name,
|
||||
"prompt": prompt,
|
||||
"resolution": inputs.get("resolution", "1k"),
|
||||
"n": int(inputs.get("n", 1) or 1),
|
||||
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
|
||||
}
|
||||
if len(prompt) > 2500:
|
||||
raise ValueError("prompt exceeds Kling image generation limit of 2500 characters")
|
||||
if inputs.get("negative_prompt"):
|
||||
payload["negative_prompt"] = inputs["negative_prompt"]
|
||||
image = normalize_image_input(inputs.get("image_url"), inputs.get("image_path"))
|
||||
if image:
|
||||
payload["image"] = image
|
||||
if inputs.get("image_reference"):
|
||||
payload["image_reference"] = inputs["image_reference"]
|
||||
for key in ("image_fidelity", "human_fidelity"):
|
||||
if inputs.get(key) is not None:
|
||||
payload[key] = inputs[key]
|
||||
elements = normalize_element_list(inputs.get("element_list"))
|
||||
if elements:
|
||||
payload["element_list"] = elements
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/images/generations",
|
||||
"payload": payload,
|
||||
"api_family": "generation",
|
||||
"operation": "generate",
|
||||
"model": payload["model_name"],
|
||||
"references_used": self._reference_metadata_from_generation_payload(payload),
|
||||
"element_ids": element_ids(payload.get("element_list")),
|
||||
}
|
||||
|
||||
def _build_omni_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
model_name = str(inputs.get("model_name") or "kling-image-o1")
|
||||
if model_name not in OMNI_IMAGE_MODELS:
|
||||
raise ValueError(f"model_name {model_name!r} is not supported for api_family=omni")
|
||||
image_items, references_used = self._normalize_omni_image_list(inputs)
|
||||
prompt, prompt_references = build_image_prompt_references(self._prompt(inputs), image_items)
|
||||
references_used = prompt_references or references_used
|
||||
elements = normalize_element_list(inputs.get("element_list"))
|
||||
payload: dict[str, Any] = {
|
||||
"model_name": model_name,
|
||||
"prompt": prompt,
|
||||
"resolution": inputs.get("resolution", "1k"),
|
||||
"n": int(inputs.get("n", 1) or 1),
|
||||
"result_type": inputs.get("result_type", "single"),
|
||||
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
|
||||
}
|
||||
if image_items:
|
||||
payload["image_list"] = [{"image": item["image"]} for item in image_items]
|
||||
if elements:
|
||||
payload["element_list"] = elements
|
||||
if inputs.get("series_amount"):
|
||||
payload["series_amount"] = inputs["series_amount"]
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/images/omni-image",
|
||||
"payload": payload,
|
||||
"api_family": "omni",
|
||||
"operation": "generate",
|
||||
"model": model_name,
|
||||
"references_used": references_used,
|
||||
"element_ids": [item["element_id"] for item in elements],
|
||||
}
|
||||
|
||||
def _normalize_omni_image_list(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
image_list: list[dict[str, Any]] = []
|
||||
references_used: list[dict[str, Any]] = []
|
||||
|
||||
def add_image(value: str | None, *, source: str | None, source_type: str) -> None:
|
||||
if not value:
|
||||
return
|
||||
image_list.append({"image": value, "source": source or value, "source_type": source_type})
|
||||
references_used.append(
|
||||
{
|
||||
"kind": "image",
|
||||
"source": source or value,
|
||||
"source_type": source_type,
|
||||
"placeholder": f"<<<image_{len(image_list)}>>>",
|
||||
}
|
||||
)
|
||||
|
||||
for item in inputs.get("image_list") or []:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("image_list items must be objects")
|
||||
source = item.get("image") or item.get("image_url") or item.get("image_path")
|
||||
value = normalize_image_input(item.get("image") or item.get("image_url"), item.get("image_path"))
|
||||
if not value:
|
||||
raise ValueError("image_list items must include image, image_url, or image_path")
|
||||
add_image(value, source=source, source_type="image_list")
|
||||
for url in inputs.get("image_urls") or []:
|
||||
add_image(normalize_image_input(url=url), source=url, source_type="image_urls")
|
||||
for path in inputs.get("image_paths") or []:
|
||||
add_image(normalize_image_input(path=path), source=str(path), source_type="image_paths")
|
||||
if inputs.get("image_url") or inputs.get("image_path"):
|
||||
add_image(
|
||||
normalize_image_input(inputs.get("image_url"), inputs.get("image_path")),
|
||||
source=inputs.get("image_url") or inputs.get("image_path"),
|
||||
source_type="image",
|
||||
)
|
||||
|
||||
return image_list, references_used
|
||||
|
||||
def _download_images(self, client: KlingClient, outputs: list[dict[str, Any]], inputs: dict[str, Any]) -> list[Path]:
|
||||
if not outputs:
|
||||
raise ValueError("Kling image response contained no images")
|
||||
base_path = Path(inputs.get("output_path", "kling_official_image.png"))
|
||||
paths: list[Path] = []
|
||||
for index, item in enumerate(outputs):
|
||||
url = self._output_url(item)
|
||||
suffix = extension_from_url(url, ".png")
|
||||
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
|
||||
client.download(url, output_path)
|
||||
paths.append(output_path)
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _output_url(item: dict[str, Any]) -> str:
|
||||
url = item.get("url") or item.get("image_url") or item.get("resource_url")
|
||||
if url:
|
||||
return str(url)
|
||||
resource = item.get("resource") or {}
|
||||
if isinstance(resource, dict) and resource.get("url"):
|
||||
return str(resource["url"])
|
||||
raise ValueError(f"Kling image response item contained no downloadable URL: {item}")
|
||||
|
||||
@staticmethod
|
||||
def _prompt(inputs: dict[str, Any]) -> str:
|
||||
prompt = str(inputs.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
raise ValueError("prompt is required")
|
||||
return prompt
|
||||
|
||||
@staticmethod
|
||||
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
if "watermark" in inputs:
|
||||
payload["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
payload["external_task_id"] = inputs["external_task_id"]
|
||||
|
||||
@staticmethod
|
||||
def _reference_metadata_from_generation_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
references: list[dict[str, Any]] = []
|
||||
if payload.get("image"):
|
||||
references.append({"kind": "image", "source_type": "image"})
|
||||
if payload.get("element_list"):
|
||||
references.extend(
|
||||
{"kind": "element", "element_id": item["element_id"]}
|
||||
for item in normalize_element_list(payload.get("element_list"))
|
||||
)
|
||||
return references
|
||||
|
||||
@staticmethod
|
||||
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
callback_url = inputs.get("callback_url")
|
||||
if not callback_url:
|
||||
return {}
|
||||
return {
|
||||
"callback_url": str(callback_url),
|
||||
"callback_requested": True,
|
||||
"polling_used": True,
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
|
||||
if not inputs.get("include_account_usage"):
|
||||
return {}
|
||||
try:
|
||||
usage = get_account_costs(client=client)
|
||||
return {
|
||||
"account_usage": usage,
|
||||
"cost_source": "estimate_with_account_usage_context",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"account_usage_error": str(exc),
|
||||
"cost_source": "estimate",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
684
tools/video/kling_official_video.py
Normal file
684
tools/video/kling_official_video.py
Normal file
@@ -0,0 +1,684 @@
|
||||
"""Kling official API video generation provider."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools._kling.account import account_usage_hint_for_error, get_account_costs
|
||||
from tools._kling.callbacks import validate_callback_url
|
||||
from tools._kling.client import KlingClient
|
||||
from tools._kling.elements import element_ids, normalize_element_list
|
||||
from tools._kling.errors import KlingAPIError
|
||||
from tools._kling.media import (
|
||||
extension_from_url,
|
||||
normalize_image_input,
|
||||
numbered_output_path,
|
||||
output_path_with_suffix,
|
||||
)
|
||||
from tools._kling.schemas import (
|
||||
CLASSIC_VIDEO_MODELS,
|
||||
OMNI_VIDEO_MODELS,
|
||||
SOUND_VALUES,
|
||||
VIDEO_ASPECT_RATIOS,
|
||||
VIDEO_DURATIONS,
|
||||
VIDEO_MODES,
|
||||
VIDEO_MODELS,
|
||||
VIDEO_RESOLUTIONS,
|
||||
)
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
DependencyError,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolTier,
|
||||
)
|
||||
from tools.video._shared import probe_output
|
||||
|
||||
|
||||
class KlingOfficialVideo(BaseTool):
|
||||
name = "kling_official_video"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.GENERATE
|
||||
capability = "video_generation"
|
||||
provider = "kling_official"
|
||||
stability = ToolStability.EXPERIMENTAL
|
||||
execution_mode = ExecutionMode.SYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:KLING_API_KEY"]
|
||||
install_instructions = (
|
||||
"Set KLING_API_KEY in .env for the official Kling API. "
|
||||
"Optionally set KLING_API_BASE_URL to override the default Singapore endpoint."
|
||||
)
|
||||
agent_skills = ["ai-video-gen", "kling-official"]
|
||||
|
||||
capabilities = ["text_to_video", "image_to_video", "reference_to_video"]
|
||||
supports = {
|
||||
"text_to_video": True,
|
||||
"image_to_video": True,
|
||||
"reference_to_video": True,
|
||||
"reference_image": True,
|
||||
"negative_prompt": True,
|
||||
"aspect_ratio": True,
|
||||
}
|
||||
best_for = [
|
||||
"official Kling direct API access",
|
||||
"text-to-video and image-to-video with Kling model controls",
|
||||
"projects that need provider provenance separate from fal.ai Kling",
|
||||
]
|
||||
not_good_for = ["offline generation", "free generation", "non-Kling model families"]
|
||||
fallback_tools = ["kling_video", "seedance_video", "veo_video", "minimax_video"]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["prompt"],
|
||||
"properties": {
|
||||
"prompt": {"type": "string"},
|
||||
"operation": {
|
||||
"type": "string",
|
||||
"enum": ["text_to_video", "image_to_video", "reference_to_video", "omni_video"],
|
||||
"default": "text_to_video",
|
||||
},
|
||||
"api_family": {
|
||||
"type": "string",
|
||||
"enum": ["classic", "turbo", "omni"],
|
||||
"default": "classic",
|
||||
},
|
||||
"model_name": {"type": "string", "enum": VIDEO_MODELS, "default": "kling-v3"},
|
||||
"model_variant": {"type": "string", "description": "Compatibility alias for model_name."},
|
||||
"duration": {"type": "string", "enum": VIDEO_DURATIONS, "default": "5"},
|
||||
"aspect_ratio": {"type": "string", "enum": VIDEO_ASPECT_RATIOS, "default": "16:9"},
|
||||
"resolution": {"type": "string", "enum": VIDEO_RESOLUTIONS, "default": "720p"},
|
||||
"mode": {"type": "string", "enum": VIDEO_MODES, "default": "std"},
|
||||
"sound": {"type": "string", "enum": SOUND_VALUES, "default": "off"},
|
||||
"negative_prompt": {"type": "string"},
|
||||
"cfg_scale": {"type": "number", "default": 0.5},
|
||||
"reference_image_url": {"type": "string"},
|
||||
"reference_image_path": {"type": "string"},
|
||||
"reference_tail_image_url": {"type": "string"},
|
||||
"reference_tail_image_path": {"type": "string"},
|
||||
"reference_image_urls": {"type": "array", "items": {"type": "string"}},
|
||||
"reference_image_paths": {"type": "array", "items": {"type": "string"}},
|
||||
"reference_video_url": {"type": "string"},
|
||||
"reference_video_path": {"type": "string"},
|
||||
"video_urls": {"type": "array", "items": {"type": "string"}},
|
||||
"video_paths": {"type": "array", "items": {"type": "string"}},
|
||||
"image_list": {"type": "array"},
|
||||
"video_list": {"type": "array"},
|
||||
"element_list": {"type": "array"},
|
||||
"multi_shot": {"type": "boolean"},
|
||||
"shot_type": {"type": "string", "enum": ["customize", "intelligence"]},
|
||||
"multi_prompt": {"type": "array"},
|
||||
"camera_control": {"type": "object"},
|
||||
"watermark": {"type": "boolean", "default": False},
|
||||
"callback_url": {"type": "string"},
|
||||
"external_task_id": {"type": "string"},
|
||||
"include_account_usage": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Optional low-frequency account usage diagnostic; not used by default.",
|
||||
},
|
||||
"timeout_seconds": {"type": "integer", "default": 900},
|
||||
"poll_interval": {"type": "number", "default": 5.0},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=2,
|
||||
backoff_seconds=2.0,
|
||||
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
|
||||
)
|
||||
idempotency_key_fields = [
|
||||
"prompt",
|
||||
"operation",
|
||||
"api_family",
|
||||
"model_name",
|
||||
"reference_image_url",
|
||||
"reference_image_path",
|
||||
"duration",
|
||||
"aspect_ratio",
|
||||
]
|
||||
side_effects = [
|
||||
"paid remote generation via official Kling API",
|
||||
"writes video file to output_path",
|
||||
]
|
||||
user_visible_verification = ["Watch generated clip for motion coherence and prompt adherence"]
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
duration = int(str(inputs.get("duration", "5")))
|
||||
mode = str(inputs.get("mode", "std"))
|
||||
api_family = str(inputs.get("api_family", "classic"))
|
||||
base = 0.18
|
||||
if api_family == "turbo":
|
||||
base = 0.22
|
||||
if api_family == "omni":
|
||||
base = 0.30
|
||||
if mode == "pro":
|
||||
base *= 1.6
|
||||
if mode == "4k":
|
||||
base *= 3.0
|
||||
if inputs.get("sound") == "on":
|
||||
base += 0.05
|
||||
if api_family == "omni":
|
||||
reference_count = self._estimate_reference_count(inputs)
|
||||
base *= 1 + (0.12 * reference_count)
|
||||
multi_prompt = inputs.get("multi_prompt") or []
|
||||
if multi_prompt:
|
||||
base *= 1 + (0.10 * len(multi_prompt))
|
||||
return round(base * max(duration, 3) / 5, 4)
|
||||
|
||||
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
|
||||
return 180.0
|
||||
|
||||
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
result = super().dry_run(inputs)
|
||||
result.update(
|
||||
{
|
||||
"paid_api": True,
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative OpenMontage estimate; official account usage reconciliation is planned for Phase 2.",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
try:
|
||||
self.check_dependencies()
|
||||
except DependencyError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
request = self._build_request(inputs)
|
||||
client = KlingClient()
|
||||
if request["protocol"] == "turbo":
|
||||
task_id = client.create_turbo(request["path"], request["payload"])
|
||||
outputs = client.poll_turbo(
|
||||
task_id,
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
|
||||
poll_interval=float(inputs.get("poll_interval", 5.0)),
|
||||
)
|
||||
else:
|
||||
task_id = client.create_classic_task(request["path"], request["payload"])
|
||||
outputs = client.poll_classic(
|
||||
request["path"],
|
||||
task_id,
|
||||
"videos",
|
||||
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
|
||||
poll_interval=float(inputs.get("poll_interval", 5.0)),
|
||||
)
|
||||
paths = self._download_videos(client, outputs, inputs)
|
||||
video_url = self._first_output_url(outputs)
|
||||
probed = probe_output(paths[0])
|
||||
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
|
||||
data: dict[str, Any] = {"provider": self.provider}
|
||||
if isinstance(exc, KlingAPIError):
|
||||
data.update(
|
||||
{
|
||||
"error_code": exc.code,
|
||||
"request_id": exc.request_id,
|
||||
"http_status": exc.http_status,
|
||||
}
|
||||
)
|
||||
data["account_usage_diagnostic"] = account_usage_hint_for_error(exc)
|
||||
return ToolResult(success=False, data=data, error=f"Kling official video generation failed: {exc}")
|
||||
except Exception as exc:
|
||||
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official video generation failed: {exc}")
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": self.provider,
|
||||
"model": request["model"],
|
||||
"task_id": task_id,
|
||||
"operation": request["operation"],
|
||||
"api_family": request["api_family"],
|
||||
"prompt": inputs["prompt"],
|
||||
"remote_url": video_url,
|
||||
"remote_outputs": outputs,
|
||||
"output": str(paths[0]),
|
||||
"output_path": str(paths[0]),
|
||||
"video_paths": [str(path) for path in paths],
|
||||
"format": "mp4",
|
||||
"references_used": request.get("references_used", []),
|
||||
"element_ids": request.get("element_ids", []),
|
||||
"cost_estimate_confidence": "low",
|
||||
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
|
||||
**self._account_usage_result(inputs, client),
|
||||
**self._callback_result_data(inputs, task_id),
|
||||
**probed,
|
||||
},
|
||||
artifacts=[str(path) for path in paths],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - start, 2),
|
||||
model=request["model"],
|
||||
)
|
||||
|
||||
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
operation = str(inputs.get("operation", "text_to_video"))
|
||||
api_family = str(inputs.get("api_family", "classic"))
|
||||
if operation == "omni_video":
|
||||
operation = "reference_to_video"
|
||||
api_family = "omni"
|
||||
if api_family == "turbo":
|
||||
return self._build_turbo_request(inputs, operation)
|
||||
if api_family == "omni":
|
||||
return self._build_omni_request(inputs, operation)
|
||||
return self._build_classic_request(inputs, operation)
|
||||
|
||||
def _build_classic_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
|
||||
if operation == "text_to_video":
|
||||
payload = self._base_classic_payload(inputs)
|
||||
payload["prompt"] = self._prompt(inputs)
|
||||
if inputs.get("negative_prompt"):
|
||||
payload["negative_prompt"] = inputs["negative_prompt"]
|
||||
if inputs.get("aspect_ratio"):
|
||||
payload["aspect_ratio"] = inputs.get("aspect_ratio", "16:9")
|
||||
self._copy_multi_shot_fields(inputs, payload)
|
||||
path = "/v1/videos/text2video"
|
||||
elif operation == "image_to_video":
|
||||
image = normalize_image_input(inputs.get("reference_image_url"), inputs.get("reference_image_path"))
|
||||
if not image:
|
||||
raise ValueError("image_to_video requires reference_image_url or reference_image_path")
|
||||
payload = self._base_classic_payload(inputs)
|
||||
payload["image"] = image
|
||||
if inputs.get("prompt"):
|
||||
payload["prompt"] = inputs["prompt"]
|
||||
if inputs.get("negative_prompt"):
|
||||
payload["negative_prompt"] = inputs["negative_prompt"]
|
||||
tail = normalize_image_input(inputs.get("reference_tail_image_url"), inputs.get("reference_tail_image_path"))
|
||||
if tail:
|
||||
payload["image_tail"] = tail
|
||||
if inputs.get("element_list"):
|
||||
payload["element_list"] = normalize_element_list(inputs.get("element_list"))
|
||||
self._copy_multi_shot_fields(inputs, payload)
|
||||
path = "/v1/videos/image2video"
|
||||
else:
|
||||
raise ValueError(f"Unsupported classic video operation: {operation}")
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": path,
|
||||
"payload": payload,
|
||||
"operation": operation,
|
||||
"api_family": "classic",
|
||||
"model": payload["model_name"],
|
||||
"references_used": self._reference_metadata_from_classic_payload(payload),
|
||||
"element_ids": element_ids(payload.get("element_list")),
|
||||
}
|
||||
|
||||
def _build_turbo_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
|
||||
settings = {
|
||||
"resolution": inputs.get("resolution", "720p"),
|
||||
"duration": int(str(inputs.get("duration", "5"))),
|
||||
}
|
||||
options = self._options_payload(inputs)
|
||||
if operation == "text_to_video":
|
||||
settings["aspect_ratio"] = inputs.get("aspect_ratio", "16:9")
|
||||
payload = {"prompt": self._prompt(inputs), "settings": settings}
|
||||
if options:
|
||||
payload["options"] = options
|
||||
path = "/text-to-video/kling-3.0-turbo"
|
||||
elif operation == "image_to_video":
|
||||
if inputs.get("reference_image_path") and not inputs.get("reference_image_url"):
|
||||
raise ValueError("Turbo image_to_video requires reference_image_url; local paths cannot be silently uploaded.")
|
||||
image_url = inputs.get("reference_image_url")
|
||||
if not image_url:
|
||||
raise ValueError("image_to_video requires reference_image_url for api_family=turbo")
|
||||
contents = [{"type": "prompt", "text": self._prompt(inputs)}, {"type": "first_frame", "url": image_url}]
|
||||
payload = {"contents": contents, "settings": settings}
|
||||
if options:
|
||||
payload["options"] = options
|
||||
path = "/image-to-video/kling-3.0-turbo"
|
||||
else:
|
||||
raise ValueError(f"Unsupported turbo video operation: {operation}")
|
||||
return {
|
||||
"protocol": "turbo",
|
||||
"path": path,
|
||||
"payload": payload,
|
||||
"operation": operation,
|
||||
"api_family": "turbo",
|
||||
"model": "kling-3.0-turbo",
|
||||
}
|
||||
|
||||
def _build_omni_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
|
||||
explicit_model = inputs.get("model_name") or inputs.get("model_variant")
|
||||
model_name = str(explicit_model or "kling-video-o1")
|
||||
if model_name not in OMNI_VIDEO_MODELS:
|
||||
raise ValueError(f"model_name {model_name!r} is not supported for api_family=omni")
|
||||
payload, references_used, element_id_values = self._build_omni_payload(inputs, operation, model_name)
|
||||
return {
|
||||
"protocol": "classic",
|
||||
"path": "/v1/videos/omni-video",
|
||||
"payload": payload,
|
||||
"operation": operation,
|
||||
"api_family": "omni",
|
||||
"model": model_name,
|
||||
"references_used": references_used,
|
||||
"element_ids": element_id_values,
|
||||
}
|
||||
|
||||
def _build_omni_payload(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
operation: str,
|
||||
model_name: str,
|
||||
) -> tuple[dict[str, Any], list[dict[str, Any]], list[int]]:
|
||||
payload: dict[str, Any] = {
|
||||
"model_name": model_name,
|
||||
"prompt": self._prompt(inputs),
|
||||
"mode": inputs.get("mode", "pro"),
|
||||
"duration": str(inputs.get("duration", "5")),
|
||||
}
|
||||
if inputs.get("sound"):
|
||||
payload["sound"] = inputs["sound"]
|
||||
if inputs.get("aspect_ratio"):
|
||||
payload["aspect_ratio"] = inputs["aspect_ratio"]
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
self._copy_multi_shot_fields(inputs, payload)
|
||||
|
||||
references_used: list[dict[str, Any]] = []
|
||||
image_list, image_refs = self._normalize_omni_image_list(inputs)
|
||||
references_used.extend(image_refs)
|
||||
if image_list:
|
||||
payload["image_list"] = image_list
|
||||
|
||||
video_list, video_refs = self._normalize_omni_video_list(inputs)
|
||||
references_used.extend(video_refs)
|
||||
if video_list:
|
||||
payload["video_list"] = video_list
|
||||
|
||||
elements = normalize_element_list(inputs.get("element_list"))
|
||||
element_id_values = [item["element_id"] for item in elements]
|
||||
if elements:
|
||||
payload["element_list"] = elements
|
||||
references_used.extend(
|
||||
{"kind": "element", "element_id": item["element_id"]}
|
||||
for item in elements
|
||||
)
|
||||
if operation == "reference_to_video" and not any(payload.get(k) for k in ("image_list", "video_list", "element_list")):
|
||||
raise ValueError("reference_to_video with api_family=omni requires image_list, video_list, element_list, or reference image URLs.")
|
||||
return payload, references_used, element_id_values
|
||||
|
||||
def _base_classic_payload(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
model_name = str(inputs.get("model_name") or inputs.get("model_variant") or "kling-v3")
|
||||
if model_name not in CLASSIC_VIDEO_MODELS:
|
||||
raise ValueError(f"model_name {model_name!r} is not supported for api_family=classic")
|
||||
payload: dict[str, Any] = {
|
||||
"model_name": model_name,
|
||||
"duration": str(inputs.get("duration", "5")),
|
||||
"mode": inputs.get("mode", "std"),
|
||||
"sound": inputs.get("sound", "off"),
|
||||
}
|
||||
if inputs.get("cfg_scale") is not None:
|
||||
payload["cfg_scale"] = inputs["cfg_scale"]
|
||||
if inputs.get("camera_control"):
|
||||
payload["camera_control"] = inputs["camera_control"]
|
||||
self._copy_common_task_fields(inputs, payload)
|
||||
return payload
|
||||
|
||||
def _options_payload(self, inputs: dict[str, Any]) -> dict[str, Any]:
|
||||
options: dict[str, Any] = {}
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
options["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
options["external_task_id"] = inputs["external_task_id"]
|
||||
if "watermark" in inputs:
|
||||
options["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
|
||||
return options
|
||||
|
||||
def _copy_common_task_fields(self, inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
if "watermark" in inputs:
|
||||
payload["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
|
||||
callback_url = validate_callback_url(inputs.get("callback_url"))
|
||||
if callback_url:
|
||||
payload["callback_url"] = callback_url
|
||||
if inputs.get("external_task_id"):
|
||||
payload["external_task_id"] = inputs["external_task_id"]
|
||||
|
||||
def _copy_multi_shot_fields(self, inputs: dict[str, Any], payload: dict[str, Any]) -> None:
|
||||
if inputs.get("multi_shot") is None and not inputs.get("multi_prompt"):
|
||||
return
|
||||
payload["multi_shot"] = bool(inputs.get("multi_shot", True))
|
||||
shot_type = str(inputs.get("shot_type") or "customize")
|
||||
if shot_type not in {"customize", "intelligence"}:
|
||||
raise ValueError("shot_type must be one of: customize, intelligence")
|
||||
payload["shot_type"] = shot_type
|
||||
if inputs.get("multi_prompt"):
|
||||
if not isinstance(inputs["multi_prompt"], list):
|
||||
raise ValueError("multi_prompt must be a list")
|
||||
normalized: list[dict[str, Any]] = []
|
||||
for item in inputs["multi_prompt"]:
|
||||
if not isinstance(item, dict) or not item.get("prompt"):
|
||||
raise ValueError("each multi_prompt item must be an object with prompt")
|
||||
allowed = {
|
||||
key: item[key]
|
||||
for key in ("prompt", "duration", "camera_control", "image_refs", "element_refs")
|
||||
if key in item
|
||||
}
|
||||
normalized.append(allowed)
|
||||
payload["multi_prompt"] = normalized
|
||||
|
||||
def _normalize_omni_image_list(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
image_list: list[dict[str, Any]] = []
|
||||
references_used: list[dict[str, Any]] = []
|
||||
|
||||
def add_image(value: str | None, *, kind: str, item_type: str | None = None) -> None:
|
||||
if not value:
|
||||
return
|
||||
record = {"image_url": value}
|
||||
if item_type:
|
||||
record["type"] = item_type
|
||||
image_list.append(record)
|
||||
references_used.append(
|
||||
{
|
||||
"kind": "image",
|
||||
"source": value,
|
||||
"source_type": kind,
|
||||
"type": item_type,
|
||||
}
|
||||
)
|
||||
|
||||
for item in inputs.get("image_list") or []:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("image_list items must be objects")
|
||||
value = normalize_image_input(item.get("image_url") or item.get("image"), item.get("image_path"))
|
||||
if not value:
|
||||
raise ValueError("image_list items must include image_url, image, or image_path")
|
||||
record = {"image_url": value}
|
||||
if item.get("type"):
|
||||
record["type"] = item["type"]
|
||||
image_list.append(record)
|
||||
references_used.append(
|
||||
{
|
||||
"kind": "image",
|
||||
"source": item.get("image_url") or item.get("image_path") or item.get("image"),
|
||||
"source_type": "image_list",
|
||||
"type": item.get("type"),
|
||||
}
|
||||
)
|
||||
|
||||
add_image(
|
||||
normalize_image_input(inputs.get("reference_image_url"), inputs.get("reference_image_path")),
|
||||
kind="reference_image",
|
||||
item_type="first_frame",
|
||||
)
|
||||
add_image(
|
||||
normalize_image_input(inputs.get("reference_tail_image_url"), inputs.get("reference_tail_image_path")),
|
||||
kind="reference_tail_image",
|
||||
item_type="end_frame",
|
||||
)
|
||||
for url in inputs.get("reference_image_urls") or []:
|
||||
add_image(normalize_image_input(url=url), kind="reference_image_urls")
|
||||
for path in inputs.get("reference_image_paths") or []:
|
||||
add_image(normalize_image_input(path=path), kind="reference_image_paths")
|
||||
return image_list, references_used
|
||||
|
||||
def _normalize_omni_video_list(
|
||||
self,
|
||||
inputs: dict[str, Any],
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
if inputs.get("reference_video_path") or inputs.get("video_paths"):
|
||||
raise ValueError("Video Omni requires video URLs; local video paths cannot be silently uploaded.")
|
||||
|
||||
video_list: list[dict[str, Any]] = []
|
||||
references_used: list[dict[str, Any]] = []
|
||||
|
||||
def add_video(item: dict[str, Any], source_type: str) -> None:
|
||||
if item.get("video_path"):
|
||||
raise ValueError("Video Omni requires video URLs; local video paths cannot be silently uploaded.")
|
||||
video_url = item.get("video_url") or item.get("url")
|
||||
if not video_url:
|
||||
raise ValueError("video_list items must include video_url")
|
||||
record = {"video_url": video_url}
|
||||
if item.get("refer_type"):
|
||||
record["refer_type"] = item["refer_type"]
|
||||
if "keep_original_sound" in item:
|
||||
value = item["keep_original_sound"]
|
||||
record["keep_original_sound"] = "yes" if value is True else "no" if value is False else value
|
||||
video_list.append(record)
|
||||
references_used.append(
|
||||
{
|
||||
"kind": "video",
|
||||
"source": video_url,
|
||||
"source_type": source_type,
|
||||
"refer_type": record.get("refer_type"),
|
||||
"keep_original_sound": record.get("keep_original_sound"),
|
||||
}
|
||||
)
|
||||
|
||||
for item in inputs.get("video_list") or []:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("video_list items must be objects")
|
||||
add_video(item, "video_list")
|
||||
if inputs.get("reference_video_url"):
|
||||
add_video({"video_url": inputs["reference_video_url"]}, "reference_video_url")
|
||||
for url in inputs.get("video_urls") or []:
|
||||
add_video({"video_url": url}, "video_urls")
|
||||
return video_list, references_used
|
||||
|
||||
def _download_videos(
|
||||
self,
|
||||
client: KlingClient,
|
||||
outputs: list[dict[str, Any]],
|
||||
inputs: dict[str, Any],
|
||||
) -> list[Path]:
|
||||
if not outputs:
|
||||
raise ValueError("Kling video response contained no videos")
|
||||
base_path = Path(inputs.get("output_path", "kling_official_video.mp4"))
|
||||
paths: list[Path] = []
|
||||
for index, item in enumerate(outputs):
|
||||
url = self._output_url(item)
|
||||
suffix = extension_from_url(url, ".mp4")
|
||||
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
|
||||
client.download(url, output_path)
|
||||
paths.append(output_path)
|
||||
return paths
|
||||
|
||||
@staticmethod
|
||||
def _output_url(item: dict[str, Any]) -> str:
|
||||
url = item.get("url") or item.get("video_url") or item.get("resource_url")
|
||||
if url:
|
||||
return str(url)
|
||||
resource = item.get("resource") or {}
|
||||
if isinstance(resource, dict) and resource.get("url"):
|
||||
return str(resource["url"])
|
||||
raise ValueError(f"Kling video response contained no downloadable URL: {item}")
|
||||
|
||||
@staticmethod
|
||||
def _reference_metadata_from_classic_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
references: list[dict[str, Any]] = []
|
||||
if payload.get("image"):
|
||||
references.append({"kind": "image", "source_type": "reference_image"})
|
||||
if payload.get("image_tail"):
|
||||
references.append({"kind": "image", "source_type": "reference_tail_image"})
|
||||
if payload.get("element_list"):
|
||||
references.extend(
|
||||
{"kind": "element", "element_id": item["element_id"]}
|
||||
for item in normalize_element_list(payload.get("element_list"))
|
||||
)
|
||||
return references
|
||||
|
||||
@staticmethod
|
||||
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
|
||||
callback_url = inputs.get("callback_url")
|
||||
if not callback_url:
|
||||
return {}
|
||||
return {
|
||||
"callback_url": str(callback_url),
|
||||
"callback_requested": True,
|
||||
"polling_used": True,
|
||||
"task_id": task_id,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
|
||||
if not inputs.get("include_account_usage"):
|
||||
return {}
|
||||
try:
|
||||
usage = get_account_costs(client=client)
|
||||
return {
|
||||
"account_usage": usage,
|
||||
"cost_source": "estimate_with_account_usage_context",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
except Exception as exc:
|
||||
return {
|
||||
"account_usage_error": str(exc),
|
||||
"cost_source": "estimate",
|
||||
"reconciled_cost_usd": None,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _estimate_reference_count(inputs: dict[str, Any]) -> int:
|
||||
count = 0
|
||||
for key in (
|
||||
"image_list",
|
||||
"video_list",
|
||||
"element_list",
|
||||
"reference_image_urls",
|
||||
"reference_image_paths",
|
||||
"video_urls",
|
||||
):
|
||||
count += len(inputs.get(key) or [])
|
||||
for key in (
|
||||
"reference_image_url",
|
||||
"reference_image_path",
|
||||
"reference_tail_image_url",
|
||||
"reference_tail_image_path",
|
||||
"reference_video_url",
|
||||
):
|
||||
if inputs.get(key):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
@staticmethod
|
||||
def _prompt(inputs: dict[str, Any]) -> str:
|
||||
prompt = str(inputs.get("prompt") or "").strip()
|
||||
if not prompt:
|
||||
raise ValueError("prompt is required")
|
||||
return prompt
|
||||
|
||||
@staticmethod
|
||||
def _first_output_url(outputs: list[dict[str, Any]]) -> str:
|
||||
for item in outputs:
|
||||
try:
|
||||
return KlingOfficialVideo._output_url(item)
|
||||
except ValueError:
|
||||
continue
|
||||
raise ValueError(f"Kling video response contained no downloadable URL: {outputs}")
|
||||
@@ -88,6 +88,38 @@ class VideoSelector(BaseTool):
|
||||
"items": {"type": "string"},
|
||||
"description": "Local reference image paths for providers that support reference-conditioned video.",
|
||||
},
|
||||
"reference_video_url": {
|
||||
"type": "string",
|
||||
"description": "Reference video URL for providers that support video-conditioned generation.",
|
||||
},
|
||||
"reference_video_path": {
|
||||
"type": "string",
|
||||
"description": "Local reference video path. Providers that require URLs should reject this clearly.",
|
||||
},
|
||||
"image_list": {
|
||||
"type": "array",
|
||||
"description": "Provider-specific list of image references, e.g. Kling Official Video Omni.",
|
||||
},
|
||||
"video_list": {
|
||||
"type": "array",
|
||||
"description": "Provider-specific list of video references, e.g. Kling Official Video Omni.",
|
||||
},
|
||||
"element_list": {
|
||||
"type": "array",
|
||||
"description": "Provider-specific element references, e.g. Kling Official element_id objects.",
|
||||
},
|
||||
"multi_shot": {
|
||||
"type": "boolean",
|
||||
"description": "Provider-specific multi-shot mode.",
|
||||
},
|
||||
"shot_type": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific multi-shot type.",
|
||||
},
|
||||
"multi_prompt": {
|
||||
"type": "array",
|
||||
"description": "Structured multi-shot prompts; not inferred from prose.",
|
||||
},
|
||||
"image_url": {
|
||||
"type": "string",
|
||||
"description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).",
|
||||
@@ -96,6 +128,34 @@ class VideoSelector(BaseTool):
|
||||
"type": "string",
|
||||
"description": "Resolution hint for providers that support named output resolutions.",
|
||||
},
|
||||
"api_family": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific API family hint passed through when supported, e.g. classic/turbo/omni.",
|
||||
},
|
||||
"model_name": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific model name passed through when supported.",
|
||||
},
|
||||
"mode": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific quality mode passed through when supported.",
|
||||
},
|
||||
"sound": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific native audio toggle passed through when supported.",
|
||||
},
|
||||
"watermark": {
|
||||
"type": "boolean",
|
||||
"description": "Provider-specific watermark toggle passed through when supported.",
|
||||
},
|
||||
"callback_url": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific callback URL. Current OpenMontage providers still poll by default.",
|
||||
},
|
||||
"external_task_id": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific idempotency/provenance task id.",
|
||||
},
|
||||
"workflow_json": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
|
||||
Reference in New Issue
Block a user