Add official Kling API providers

This commit is contained in:
xucailiang
2026-07-07 14:56:40 +08:00
parent 0c202b507a
commit 7c5dfdd31a
43 changed files with 9470 additions and 16 deletions

6
tools/_kling/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
"""Shared helpers for Kling official API providers."""
from .client import KlingClient
from .errors import KlingAPIError, is_retryable_kling_error
__all__ = ["KlingAPIError", "KlingClient", "is_retryable_kling_error"]

121
tools/_kling/account.py Normal file
View File

@@ -0,0 +1,121 @@
"""Account usage diagnostics for Kling official API.
This module is a low-frequency helper, not an OpenMontage registry tool.
"""
from __future__ import annotations
import time
from hashlib import sha256
from typing import Any
from .client import KlingClient
from .errors import KlingAPIError
_CACHE: dict[tuple[tuple[str, str], ...], dict[str, Any]] = {}
_LAST_QUERY_AT = 0.0
def reset_account_usage_cache() -> None:
"""Clear in-process account usage cache. Intended for tests."""
global _LAST_QUERY_AT
_CACHE.clear()
_LAST_QUERY_AT = 0.0
def get_account_costs(
*,
start_time: str | None = None,
end_time: str | None = None,
resource_pack_name: str | None = None,
client: KlingClient | None = None,
ttl_seconds: float = 300.0,
min_interval_seconds: float = 10.0,
now: float | None = None,
) -> dict[str, Any]:
"""Read `/account/costs` with in-process cache and throttle protection."""
global _LAST_QUERY_AT
timestamp = time.time() if now is None else now
params = {
key: value
for key, value in {
"start_time": start_time,
"end_time": end_time,
"resource_pack_name": resource_pack_name,
}.items()
if value
}
api = client or KlingClient()
key = _cache_key(api, params)
cached = _CACHE.get(key)
if cached and timestamp - float(cached["fetched_at"]) <= ttl_seconds:
return {**cached["payload"], "cached": True, "throttle_status": "cache_hit"}
if _LAST_QUERY_AT and timestamp - _LAST_QUERY_AT < min_interval_seconds:
if cached:
return {**cached["payload"], "cached": True, "throttle_status": "throttled_cache"}
return {
"provider": "kling_official",
"queried_range": {
"start_time": start_time,
"end_time": end_time,
"resource_pack_name": resource_pack_name,
},
"cached": False,
"throttle_status": "throttled_no_cache",
"message": "Account Usage is rate-limited; retry after the local throttle window.",
}
raw = api.get("/account/costs", params=params)
data = raw.get("data") if isinstance(raw, dict) else {}
if not isinstance(data, dict):
data = {}
payload = {
"provider": "kling_official",
"queried_range": {
"start_time": start_time,
"end_time": end_time,
"resource_pack_name": resource_pack_name,
},
"resource_pack_subscribe_infos": data.get("resource_pack_subscribe_infos", []),
"raw": raw,
"cached": False,
"throttle_status": "fresh",
}
_CACHE[key] = {"fetched_at": timestamp, "payload": payload}
_LAST_QUERY_AT = timestamp
return payload
def _cache_key(client: Any, params: dict[str, Any]) -> tuple[tuple[str, str], ...]:
"""Scope Account Usage cache by request params and account endpoint identity."""
api_key = getattr(client, "api_key", None) or ""
api_key_hash = sha256(str(api_key).encode("utf-8")).hexdigest() if api_key else ""
scope = {
"base_url": getattr(client, "base_url", ""),
"api_key_sha256": api_key_hash,
**{name: str(value) for name, value in params.items()},
}
return tuple(sorted((name, str(value)) for name, value in scope.items()))
def account_usage_hint_for_error(error: KlingAPIError) -> dict[str, Any]:
"""Return a diagnostic hint for balance/resource-pack related errors."""
code = str(error.code) if error.code is not None else ""
if code not in {"1101", "1102"}:
return {}
return {
"provider": "kling_official",
"reason": "account_balance_or_resource_pack",
"message": (
"Kling returned an account/resource-pack error. Use tools._kling.account.get_account_costs() "
"for a low-frequency account usage diagnostic, or check the Kling Open Platform console."
),
"error_code": error.code,
"request_id": error.request_id,
}

17
tools/_kling/callbacks.py Normal file
View File

@@ -0,0 +1,17 @@
"""Callback validation helpers for Kling official providers."""
from __future__ import annotations
from urllib.parse import urlparse
def validate_callback_url(callback_url: str | None) -> str | None:
"""Return a normalized callback URL or raise for obviously invalid input."""
if not callback_url:
return None
value = str(callback_url).strip()
parsed = urlparse(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("callback_url must be an absolute http(s) URL")
return value

216
tools/_kling/client.py Normal file
View File

@@ -0,0 +1,216 @@
"""HTTP client and task parsers for Kling official API providers."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from urllib.parse import urljoin
import requests
from .errors import KlingAPIError, is_retryable_kling_error
from .schemas import (
CLASSIC_FAILURE_STATUS,
CLASSIC_PENDING_STATUSES,
CLASSIC_SUCCESS_STATUS,
DEFAULT_API_BASE_URL,
TURBO_FAILURE_STATUS,
TURBO_PENDING_STATUSES,
TURBO_SUCCESS_STATUS,
)
class KlingClient:
"""Small synchronous client for the official Kling API."""
def __init__(
self,
api_key: str | None = None,
base_url: str | None = None,
session: Any | None = None,
max_retries: int = 2,
) -> None:
self.api_key = api_key if api_key is not None else os.environ.get("KLING_API_KEY")
self.base_url = (base_url or os.environ.get("KLING_API_BASE_URL") or DEFAULT_API_BASE_URL).rstrip("/")
self.session = session or requests.Session()
self.max_retries = max_retries
@property
def headers(self) -> dict[str, str]:
if not self.api_key:
raise KlingAPIError(
"KLING_API_KEY is not set. Configure KLING_API_KEY for official Kling API access.",
http_status=401,
)
return {
"Authorization": f"Bearer {self.api_key}",
"Accept": "application/json",
"Content-Type": "application/json",
}
def post(self, path: str, payload: dict[str, Any]) -> dict[str, Any]:
return self._request("post", path, json=payload)
def get(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
return self._request("get", path, params=params)
def download(self, url: str, output_path: Path, timeout: int = 180) -> Path:
output_path.parent.mkdir(parents=True, exist_ok=True)
response = self.session.get(url, timeout=timeout)
self._raise_for_http_error(response)
content = getattr(response, "content", None)
if content is None and hasattr(response, "iter_content"):
content = b"".join(chunk for chunk in response.iter_content(chunk_size=1024 * 128) if chunk)
output_path.write_bytes(content or b"")
return output_path
def create_classic_task(self, path: str, payload: dict[str, Any]) -> str:
data = self.post(path, payload)
task_id = ((data.get("data") or {}).get("task_id"))
if not task_id:
raise KlingAPIError(f"Kling Classic create response missing data.task_id: {data}")
return str(task_id)
def poll_classic(
self,
path: str,
task_id: str,
result_key: str,
timeout_seconds: int = 900,
poll_interval: float = 5.0,
) -> list[dict[str, Any]]:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
data = self.get(f"{path.rstrip('/')}/{task_id}")
payload = data.get("data") or {}
status = payload.get("task_status") or payload.get("status")
if status == CLASSIC_SUCCESS_STATUS:
task_result = payload.get("task_result") or {}
outputs = task_result.get(result_key) or []
if not isinstance(outputs, list):
raise KlingAPIError(f"Kling Classic result path data.task_result.{result_key} is not a list")
return outputs
if status == CLASSIC_FAILURE_STATUS:
message = payload.get("task_status_msg") or payload.get("message") or "Kling Classic task failed"
raise KlingAPIError(str(message), code=payload.get("task_status"), response=data)
if status not in CLASSIC_PENDING_STATUSES:
raise KlingAPIError(f"Unexpected Kling Classic task status {status!r}", response=data)
time.sleep(min(poll_interval, max(0.0, deadline - time.time())))
raise TimeoutError(f"Kling Classic task {task_id} timed out after {timeout_seconds}s")
def create_turbo(self, path: str, payload: dict[str, Any]) -> str:
data = self.post(path, payload)
task_id = ((data.get("data") or {}).get("id"))
if not task_id:
raise KlingAPIError(f"Kling Turbo create response missing data.id: {data}")
return str(task_id)
def poll_turbo(
self,
task_id: str,
timeout_seconds: int = 900,
poll_interval: float = 5.0,
) -> list[dict[str, Any]]:
deadline = time.time() + timeout_seconds
while time.time() < deadline:
data = self.get("/tasks", params={"task_ids": task_id})
records = data.get("data") or []
if not records:
raise KlingAPIError(f"Kling Turbo poll response missing data[0]: {data}")
record = records[0]
status = record.get("status") or record.get("task_status")
if status == TURBO_SUCCESS_STATUS:
outputs = record.get("outputs") or []
if not isinstance(outputs, list):
raise KlingAPIError("Kling Turbo result path data[0].outputs is not a list")
return outputs
if status == TURBO_FAILURE_STATUS:
message = record.get("message") or record.get("error") or "Kling Turbo task failed"
raise KlingAPIError(str(message), code=record.get("code"), request_id=record.get("request_id"), response=data)
if status not in TURBO_PENDING_STATUSES:
raise KlingAPIError(f"Unexpected Kling Turbo task status {status!r}", response=data)
time.sleep(min(poll_interval, max(0.0, deadline - time.time())))
raise TimeoutError(f"Kling Turbo task {task_id} timed out after {timeout_seconds}s")
def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
url = self._url(path)
last_error: KlingAPIError | None = None
for attempt in range(self.max_retries + 1):
try:
response = getattr(self.session, method)(url, headers=self.headers, timeout=30, **kwargs)
self._raise_for_http_error(response)
data = response.json()
self._raise_for_business_error(data)
return data
except KlingAPIError as error:
last_error = error
if attempt >= self.max_retries or not is_retryable_kling_error(error):
raise
time.sleep(min(2.0 * (attempt + 1), 8.0))
except requests.RequestException as exc:
last_error = KlingAPIError(str(exc))
if attempt >= self.max_retries:
raise last_error from exc
time.sleep(min(2.0 * (attempt + 1), 8.0))
raise last_error or KlingAPIError("Kling API request failed")
def _url(self, path: str) -> str:
if path.startswith("http://") or path.startswith("https://"):
return path
return urljoin(f"{self.base_url}/", path.lstrip("/"))
def _raise_for_http_error(self, response: Any) -> None:
status = getattr(response, "status_code", None)
if status is not None and 200 <= int(status) < 300:
return
code = None
message = None
request_id = None
body: dict[str, Any] | None = None
try:
body = response.json()
code = body.get("code")
message = body.get("message") or body.get("msg")
request_id = body.get("request_id") or body.get("requestId")
except Exception:
text = getattr(response, "text", "")
message = text[:500] if text else f"HTTP {status}"
raise self._format_error(
code=code,
message=message or f"HTTP {status}",
request_id=request_id,
http_status=int(status) if status is not None else None,
response=body,
)
def _raise_for_business_error(self, data: dict[str, Any]) -> None:
code = data.get("code")
if code in (None, 0, "0"):
return
raise self._format_error(
code=code,
message=str(data.get("message") or data.get("msg") or "Kling API returned an error"),
request_id=data.get("request_id") or data.get("requestId"),
response=data,
)
@staticmethod
def _format_error(
*,
code: str | int | None,
message: str,
request_id: str | None = None,
http_status: int | None = None,
response: dict[str, Any] | None = None,
) -> KlingAPIError:
if str(code) == "1303" and "并发/资源包限制" not in message:
message = f"{message} (并发/资源包限制: parallel task over resource pack limit)"
return KlingAPIError(
message=message,
code=code,
request_id=request_id,
http_status=http_status,
response=response,
)

86
tools/_kling/elements.py Normal file
View File

@@ -0,0 +1,86 @@
"""Element reference helpers for Kling official Omni providers.
These helpers intentionally do not inherit from BaseTool. Elements are an
internal provider reference mechanism in Phase 2, not a registry capability.
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from .client import KlingClient
def normalize_element_list(element_list: Any | None) -> list[dict[str, int]]:
"""Normalize official Kling element references to element_list objects."""
if not element_list:
return []
if not isinstance(element_list, list):
raise ValueError("element_list must be a list of element ids or objects")
normalized: list[dict[str, int]] = []
for item in element_list:
raw_id: Any
if isinstance(item, dict):
raw_id = item.get("element_id", item.get("id"))
else:
raw_id = item
if raw_id is None:
raise ValueError("each element_list item must include element_id")
try:
element_id = int(raw_id)
except (TypeError, ValueError) as exc:
raise ValueError(f"element_id must be an integer-compatible value: {raw_id!r}") from exc
if element_id <= 0:
raise ValueError("element_id must be positive")
normalized.append({"element_id": element_id})
return normalized
def element_ids(element_list: Any | None) -> list[int]:
"""Return normalized element ids from an element reference list."""
return [item["element_id"] for item in normalize_element_list(element_list)]
def get_custom_element(element_id: int, client: KlingClient | None = None) -> dict[str, Any]:
"""Fetch one custom element for validation or diagnostics."""
api = client or KlingClient()
return api.get(f"/v1/general/advanced-custom-elements/{int(element_id)}")
def list_custom_elements(
client: KlingClient | None = None,
params: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""List custom elements without creating or deleting assets."""
api = client or KlingClient()
return api.get("/v1/general/advanced-custom-elements", params=params)
def list_preset_elements(
client: KlingClient | None = None,
params: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""List official preset elements without entering the tool registry."""
api = client or KlingClient()
return api.get("/v1/general/advanced-presets-elements", params=params)
def write_elements_artifact(
artifact_path: str | Path,
elements: list[dict[str, Any]],
) -> Path:
"""Write element metadata in the Phase 2 reproducibility artifact shape."""
path = Path(artifact_path)
path.parent.mkdir(parents=True, exist_ok=True)
payload = {"provider": "kling_official", "elements": elements}
path.write_text(json.dumps(payload, indent=2, ensure_ascii=False), encoding="utf-8")
return path

46
tools/_kling/errors.py Normal file
View File

@@ -0,0 +1,46 @@
"""Error types and retry policy for Kling official API calls."""
from __future__ import annotations
from dataclasses import dataclass
from typing import Any
@dataclass
class KlingAPIError(Exception):
"""Structured error returned by the Kling official API."""
message: str
code: str | int | None = None
request_id: str | None = None
http_status: int | None = None
response: dict[str, Any] | None = None
def __str__(self) -> str:
parts = [self.message]
if self.code is not None:
parts.append(f"code={self.code}")
if self.request_id:
parts.append(f"request_id={self.request_id}")
if self.http_status is not None:
parts.append(f"http_status={self.http_status}")
return " | ".join(parts)
_RETRYABLE_CODES = {"1302", "1303", "5000", "5001", "5002"}
_RETRYABLE_HTTP = {500, 503, 504}
def _code_str(code: str | int | None) -> str | None:
if code is None:
return None
return str(code)
def is_retryable_kling_error(error: KlingAPIError) -> bool:
"""Return whether an official Kling error is safe for limited retry."""
code = _code_str(error.code)
if code in _RETRYABLE_CODES:
return True
return error.http_status in _RETRYABLE_HTTP

111
tools/_kling/media.py Normal file
View File

@@ -0,0 +1,111 @@
"""Media normalization and download helpers for Kling official providers."""
from __future__ import annotations
import base64
import mimetypes
from pathlib import Path
from urllib.parse import urlparse
def strip_data_uri_prefix(value: str | None) -> str | None:
"""Return raw base64/content by removing a data URI prefix if present."""
if value is None:
return None
marker = ";base64,"
if value.startswith("data:") and marker in value:
return value.split(marker, 1)[1]
return value
def image_file_to_raw_base64(path: str | Path) -> str:
"""Read a local image file and return raw base64 without data URI prefix."""
image_path = Path(path)
if not image_path.is_file():
raise FileNotFoundError(f"Image not found: {image_path}")
return base64.b64encode(image_path.read_bytes()).decode("ascii")
def file_to_raw_base64(path: str | Path, *, label: str = "File") -> str:
"""Read a local media file and return raw base64 without a data URI prefix."""
media_path = Path(path)
if not media_path.is_file():
raise FileNotFoundError(f"{label} not found: {media_path}")
return base64.b64encode(media_path.read_bytes()).decode("ascii")
def normalize_image_input(url: str | None = None, path: str | Path | None = None) -> str | None:
"""Normalize a Kling image input to either URL or raw base64."""
if url:
return strip_data_uri_prefix(url)
if path:
return image_file_to_raw_base64(path)
return None
def normalize_media_input(
url: str | None = None,
path: str | Path | None = None,
value: str | None = None,
*,
label: str = "Media file",
) -> str | None:
"""Normalize a generic Kling media input to URL, raw base64, or raw provided value."""
if value:
return strip_data_uri_prefix(value)
if url:
return strip_data_uri_prefix(url)
if path:
return file_to_raw_base64(path, label=label)
return None
def extension_from_url(url: str | None, default: str = ".png") -> str:
"""Infer a file extension from a URL path."""
if not url:
return default
suffix = Path(urlparse(url).path).suffix.lower()
if suffix in {
".png",
".jpg",
".jpeg",
".webp",
".gif",
".mp4",
".mov",
".m4v",
".mp3",
".wav",
".m4a",
".aac",
".ogg",
".opus",
}:
return suffix
return default
def extension_from_content_type(content_type: str | None, default: str = ".png") -> str:
if not content_type:
return default
ext = mimetypes.guess_extension(content_type.split(";", 1)[0].strip())
return ext or default
def output_path_with_suffix(path: str | Path, suffix: str) -> Path:
output_path = Path(path)
if output_path.suffix:
return output_path
return output_path.with_suffix(suffix)
def numbered_output_path(first_path: Path, index: int, suffix: str) -> Path:
if index == 0:
return output_path_with_suffix(first_path, suffix)
return first_path.with_name(f"{first_path.stem}_{index + 1}{suffix}")

40
tools/_kling/omni.py Normal file
View File

@@ -0,0 +1,40 @@
"""Omni reference helpers for Kling official providers."""
from __future__ import annotations
import re
from typing import Any
PLACEHOLDER_RE = re.compile(r"<<<image_(\d+)>>>")
def build_image_prompt_references(
prompt: str,
image_list: list[dict[str, Any]],
) -> tuple[str, list[dict[str, Any]]]:
"""Bind image_list entries to stable Image Omni placeholders."""
references = [
{
"index": index,
"placeholder": f"<<<image_{index}>>>",
"source": item.get("source") or item.get("image") or item.get("image_url"),
"source_type": item.get("source_type", "unknown"),
}
for index, item in enumerate(image_list, start=1)
]
if not references:
return prompt, []
existing_numbers = [int(value) for value in PLACEHOLDER_RE.findall(prompt)]
if existing_numbers:
if max(existing_numbers) > len(references):
raise ValueError(
f"prompt references <<<image_{max(existing_numbers)}>>> but only {len(references)} image(s) were provided"
)
if min(existing_numbers) < 1:
raise ValueError("Image Omni prompt placeholders must start at <<<image_1>>>")
return prompt, references
placeholders = " ".join(item["placeholder"] for item in references)
return f"{prompt}\nReferences: {placeholders}", references

121
tools/_kling/schemas.py Normal file
View File

@@ -0,0 +1,121 @@
"""Lightweight schema constants for Kling official providers."""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
from typing import Any
DEFAULT_API_BASE_URL = "https://api-singapore.klingai.com"
class KlingProtocol(str, Enum):
CLASSIC = "classic"
TURBO = "turbo"
CLASSIC_PENDING_STATUSES = {"submitted", "processing"}
CLASSIC_SUCCESS_STATUS = "succeed"
CLASSIC_FAILURE_STATUS = "failed"
CLASSIC_STATUSES = [
"submitted",
"processing",
"succeed",
"failed",
]
TURBO_PENDING_STATUSES = {"submitted", "processing"}
TURBO_SUCCESS_STATUS = "succeeded"
TURBO_FAILURE_STATUS = "failed"
TURBO_STATUSES = [
"submitted",
"processing",
"succeeded",
"failed",
]
VIDEO_MODELS = [
"kling-v1",
"kling-v1-5",
"kling-v1-6",
"kling-v2-master",
"kling-v2-1",
"kling-v2-1-master",
"kling-v2-5-turbo",
"kling-v2-6",
"kling-v3",
"kling-video-o1",
"kling-v3-omni",
]
CLASSIC_VIDEO_MODELS = [
"kling-v1",
"kling-v1-5",
"kling-v1-6",
"kling-v2-master",
"kling-v2-1",
"kling-v2-1-master",
"kling-v2-5-turbo",
"kling-v2-6",
"kling-v3",
]
OMNI_VIDEO_MODELS = ["kling-video-o1", "kling-v3-omni"]
IMAGE_MODELS = [
"kling-v1",
"kling-v1-5",
"kling-v2",
"kling-v2-new",
"kling-v2-1",
"kling-v3",
"kling-image-o1",
"kling-v3-omni",
]
IMAGE_GENERATION_MODELS = [
"kling-v1",
"kling-v1-5",
"kling-v2",
"kling-v2-new",
"kling-v2-1",
"kling-v3",
]
OMNI_IMAGE_MODELS = ["kling-image-o1", "kling-v3-omni"]
VIDEO_DURATIONS = [str(value) for value in range(3, 16)]
VIDEO_ASPECT_RATIOS = ["16:9", "9:16", "1:1"]
VIDEO_RESOLUTIONS = ["720p", "1080p"]
VIDEO_MODES = ["std", "pro", "4k"]
SOUND_VALUES = ["on", "off"]
IMAGE_RESOLUTIONS = ["1k", "2k", "4k"]
IMAGE_ASPECT_RATIOS = ["16:9", "9:16", "1:1", "4:3", "3:4", "3:2", "2:3", "21:9", "auto"]
IMAGE_REFERENCE_TYPES = ["subject", "face"]
IMAGE_RESULT_TYPES = ["single", "series"]
RESULT_PATHS = {
"classic_video": "data.task_result.videos[]",
"classic_image": "data.task_result.images[]",
"classic_audio": "data.task_result.audios[]",
"turbo": "data[0].outputs[]",
}
TTS_LANGUAGES = ["zh", "en"]
TTS_SPEED_MIN = 0.5
TTS_SPEED_MAX = 2.0
AVATAR_MODES = ["std", "pro"]
LIP_SYNC_OPERATIONS = ["identify_face", "advanced_lip_sync", "full_lip_sync"]
@dataclass
class ClassicTaskResult:
task_id: str
status: str
outputs: list[dict[str, Any]]
@dataclass
class TurboTaskResult:
task_id: str
status: str
outputs: list[dict[str, Any]]

340
tools/audio/kling_tts.py Normal file
View File

@@ -0,0 +1,340 @@
"""Kling official API text-to-speech provider."""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools._kling.account import account_usage_hint_for_error, get_account_costs
from tools._kling.callbacks import validate_callback_url
from tools._kling.client import KlingClient
from tools._kling.errors import KlingAPIError
from tools._kling.media import extension_from_url, numbered_output_path, output_path_with_suffix
from tools._kling.schemas import TTS_LANGUAGES, TTS_SPEED_MAX, TTS_SPEED_MIN
from tools.analysis.audio_probe import probe_duration
from tools.base_tool import (
BaseTool,
DependencyError,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolTier,
)
class KlingTTS(BaseTool):
name = "kling_tts"
version = "0.1.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "kling_official"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:KLING_API_KEY"]
install_instructions = (
"Set KLING_API_KEY in .env for the official Kling API. "
"Pass voice_id explicitly; OpenMontage does not guess Kling voice IDs."
)
agent_skills = ["kling-official", "text-to-speech"]
capabilities = ["text_to_speech", "voice_selection", "multilingual"]
supports = {
"multilingual": True,
"voice_selection": True,
"offline": False,
"native_audio": True,
}
best_for = [
"official Kling text-to-speech",
"Chinese or English narration when a Kling voice_id is known",
"keeping narration provider provenance inside the Kling official account",
]
not_good_for = [
"fully offline narration",
"voice cloning without a configured official voice_id",
"auto-discovering voices",
]
fallback_tools = ["doubao_tts", "elevenlabs_tts", "openai_tts", "google_tts", "piper_tts"]
input_schema = {
"type": "object",
"required": ["text", "voice_id"],
"properties": {
"text": {"type": "string"},
"voice_id": {
"type": "string",
"description": "Official Kling voice ID. Required; do not rely on an unknown default.",
},
"voice_language": {"type": "string", "enum": TTS_LANGUAGES, "default": "en"},
"voice_speed": {
"type": "number",
"minimum": TTS_SPEED_MIN,
"maximum": TTS_SPEED_MAX,
"default": 1.0,
},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
"include_account_usage": {
"type": "boolean",
"default": False,
"description": "Optional low-frequency account usage diagnostic; not used by default.",
},
"timeout_seconds": {"type": "integer", "default": 300},
"poll_interval": {"type": "number", "default": 3.0},
"output_path": {"type": "string"},
},
}
output_schema = {
"type": "object",
"properties": {
"output": {"type": "string"},
"output_path": {"type": "string"},
"audio_paths": {"type": "array"},
"task_id": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=256, vram_mb=0, disk_mb=100, network_required=True
)
retry_policy = RetryPolicy(
max_retries=2,
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = ["text", "voice_id", "voice_language", "voice_speed"]
side_effects = ["paid remote generation via official Kling API", "writes audio file to output_path"]
user_visible_verification = ["Listen to generated audio for voice, language, and pacing"]
quality_score = 0.78
latency_p50_seconds = 20.0
def estimate_cost(self, inputs: dict[str, Any]) -> float:
text_length = len(str(inputs.get("text") or ""))
return round(max(text_length, 1) * 0.000018, 4)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 30.0
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
result = super().dry_run(inputs)
result.update(
{
"paid_api": True,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative character-based OpenMontage estimate pending official account-usage reconciliation.",
}
)
return result
def execute(self, inputs: dict[str, Any]) -> ToolResult:
try:
self.check_dependencies()
except DependencyError as exc:
return ToolResult(success=False, error=str(exc))
start = time.time()
try:
request = self._build_request(inputs)
client = KlingClient()
task_id, outputs = self._create_and_collect_audios(client, request, inputs)
paths = self._download_audios(client, outputs, inputs)
audio_duration = probe_duration(paths[0])
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
data: dict[str, Any] = {"provider": self.provider}
if isinstance(exc, KlingAPIError):
data.update(
{
"error_code": exc.code,
"request_id": exc.request_id,
"http_status": exc.http_status,
"account_usage_diagnostic": account_usage_hint_for_error(exc),
}
)
return ToolResult(success=False, data=data, error=f"Kling official TTS failed: {exc}")
except Exception as exc:
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official TTS failed: {exc}")
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": "kling-official-tts",
"task_id": task_id,
"operation": "text_to_speech",
"text_length": len(request["payload"]["text"]),
"voice_id": request["payload"]["voice_id"],
"voice_language": request["payload"].get("voice_language"),
"voice_speed": request["payload"].get("voice_speed"),
"remote_outputs": outputs,
"output": str(paths[0]),
"output_path": str(paths[0]),
"audio_paths": [str(path) for path in paths],
"format": paths[0].suffix.lstrip(".") or "mp3",
"audio_duration_seconds": round(audio_duration, 2) if audio_duration else None,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
**self._account_usage_result(inputs, client),
**self._callback_result_data(inputs, task_id),
},
artifacts=[str(path) for path in paths],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model="kling-official-tts",
)
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
text = str(inputs.get("text") or "").strip()
if not text:
raise ValueError("text is required")
if len(text) > 5000:
raise ValueError("text exceeds Kling TTS safety limit of 5000 characters")
voice_id = str(inputs.get("voice_id") or "").strip()
if not voice_id:
raise ValueError("voice_id is required for Kling official TTS")
voice_language = str(inputs.get("voice_language") or "en")
if voice_language not in TTS_LANGUAGES:
raise ValueError(f"voice_language must be one of: {', '.join(TTS_LANGUAGES)}")
voice_speed = float(inputs.get("voice_speed", 1.0))
if voice_speed < TTS_SPEED_MIN or voice_speed > TTS_SPEED_MAX:
raise ValueError(f"voice_speed must be between {TTS_SPEED_MIN} and {TTS_SPEED_MAX}")
payload: dict[str, Any] = {
"text": text,
"voice_id": voice_id,
"voice_language": voice_language,
"voice_speed": voice_speed,
}
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
"path": "/v1/audio/tts",
"payload": payload,
"operation": "text_to_speech",
"model": "kling-official-tts",
}
@staticmethod
def _create_and_collect_audios(
client: KlingClient,
request: dict[str, Any],
inputs: dict[str, Any],
) -> tuple[str, list[dict[str, Any]]]:
"""Create a TTS task and return audio outputs.
Official TTS may return a completed task and task_result.audios[]
directly from POST /v1/audio/tts. Older/async behavior still requires
polling GET /v1/audio/tts/{task_id}, so support both shapes.
"""
if hasattr(client, "post"):
data = client.post(request["path"], request["payload"])
payload = data.get("data") or {}
task_id = payload.get("task_id")
if not task_id:
raise KlingAPIError(f"Kling TTS create response missing data.task_id: {data}")
task_result = payload.get("task_result") or {}
outputs = task_result.get("audios")
if outputs is not None:
if not isinstance(outputs, list):
raise KlingAPIError("Kling TTS result path data.task_result.audios is not a list")
return str(task_id), outputs
status = payload.get("task_status") or payload.get("status")
if status == "failed":
message = payload.get("task_status_msg") or payload.get("message") or "Kling TTS task failed"
raise KlingAPIError(str(message), code=payload.get("task_status"), response=data)
return str(task_id), client.poll_classic(
request["path"],
str(task_id),
"audios",
timeout_seconds=int(inputs.get("timeout_seconds", 300)),
poll_interval=float(inputs.get("poll_interval", 3.0)),
)
task_id = client.create_classic_task(request["path"], request["payload"])
return task_id, client.poll_classic(
request["path"],
task_id,
"audios",
timeout_seconds=int(inputs.get("timeout_seconds", 300)),
poll_interval=float(inputs.get("poll_interval", 3.0)),
)
def _download_audios(
self,
client: KlingClient,
outputs: list[dict[str, Any]],
inputs: dict[str, Any],
) -> list[Path]:
if not outputs:
raise ValueError("Kling TTS response contained no audios")
base_path = Path(inputs.get("output_path", "kling_tts.mp3"))
paths: list[Path] = []
for index, item in enumerate(outputs):
url = self._output_url(item)
suffix = extension_from_url(url, ".mp3")
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
client.download(url, output_path)
paths.append(output_path)
return paths
@staticmethod
def _output_url(item: dict[str, Any]) -> str:
url = item.get("url") or item.get("audio_url") or item.get("resource_url")
if url:
return str(url)
resource = item.get("resource") or {}
if isinstance(resource, dict) and resource.get("url"):
return str(resource["url"])
raise ValueError(f"Kling TTS response item contained no downloadable URL: {item}")
@staticmethod
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
payload["callback_url"] = callback_url
if inputs.get("external_task_id"):
payload["external_task_id"] = inputs["external_task_id"]
@staticmethod
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
callback_url = inputs.get("callback_url")
if not callback_url:
return {}
return {
"callback_url": str(callback_url),
"callback_requested": True,
"polling_used": True,
"task_id": task_id,
}
@staticmethod
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
if not inputs.get("include_account_usage"):
return {}
try:
usage = get_account_costs(client=client)
return {
"account_usage": usage,
"cost_source": "estimate_with_account_usage_context",
"reconciled_cost_usd": None,
}
except Exception as exc:
return {
"account_usage_error": str(exc),
"cost_source": "estimate",
"reconciled_cost_usd": None,
}

View File

@@ -45,6 +45,17 @@ class TTSSelector(BaseTool):
"type": "string",
"description": "Provider-specific voice ID. Passed through to the selected TTS provider.",
},
"voice_language": {
"type": "string",
"enum": ["zh", "en"],
"description": "Kling official voice language. Passed through when selected provider supports it.",
},
"voice_speed": {
"type": "number",
"minimum": 0.5,
"maximum": 2.0,
"description": "Kling official voice speed. Use speed for OpenAI/ElevenLabs-style controls.",
},
"model_id": {
"type": "string",
"description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.",

View File

@@ -0,0 +1,332 @@
"""Kling official API avatar image-to-video provider."""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools._kling.account import account_usage_hint_for_error, get_account_costs
from tools._kling.callbacks import validate_callback_url
from tools._kling.client import KlingClient
from tools._kling.errors import KlingAPIError
from tools._kling.media import (
extension_from_url,
normalize_image_input,
normalize_media_input,
numbered_output_path,
output_path_with_suffix,
)
from tools._kling.schemas import AVATAR_MODES
from tools.base_tool import (
BaseTool,
DependencyError,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolTier,
)
from tools.video._shared import probe_output
class KlingAvatar(BaseTool):
name = "kling_avatar"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "avatar"
provider = "kling_official"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:KLING_API_KEY"]
install_instructions = (
"Set KLING_API_KEY in .env for the official Kling API. "
"Provide an avatar image plus either audio_id or sound_file/audio_path."
)
agent_skills = ["kling-official", "avatar-video"]
capabilities = ["photo_to_video", "avatar_video", "audio_driven_avatar"]
supports = {
"photo_to_video": True,
"audio_driven_animation": True,
"offline": False,
"cloud_render": True,
}
best_for = [
"official Kling cloud avatar presenter clips",
"high-quality image-to-video avatar generation from a supplied portrait",
"projects already using a Kling official account and resource pack",
]
not_good_for = [
"fully offline avatar generation",
"free local drafts",
"silently replacing the local talking_head provider",
]
fallback_tools = ["talking_head", "lip_sync"]
input_schema = {
"type": "object",
"anyOf": [
{"required": ["image_url"]},
{"required": ["image_path"]},
],
"allOf": [
{
"anyOf": [
{"required": ["audio_id"]},
{"required": ["sound_file"]},
{"required": ["sound_file_url"]},
{"required": ["sound_file_path"]},
{"required": ["audio_path"]},
]
}
],
"properties": {
"image_url": {"type": "string"},
"image_path": {"type": "string"},
"audio_id": {"type": "string"},
"sound_file": {
"type": "string",
"description": "Official Kling sound_file value or raw base64 audio.",
},
"sound_file_url": {"type": "string"},
"sound_file_path": {"type": "string"},
"audio_path": {
"type": "string",
"description": "Alias for sound_file_path for compatibility with local avatar tools.",
},
"prompt": {"type": "string"},
"mode": {"type": "string", "enum": AVATAR_MODES, "default": "std"},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
"include_account_usage": {
"type": "boolean",
"default": False,
"description": "Optional low-frequency account usage diagnostic; not used by default.",
},
"timeout_seconds": {"type": "integer", "default": 900},
"poll_interval": {"type": "number", "default": 5.0},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(
max_retries=2,
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = ["image_url", "image_path", "audio_id", "sound_file", "sound_file_path", "mode"]
side_effects = ["paid remote generation via official Kling API", "writes avatar video to output_path"]
user_visible_verification = ["Watch generated avatar video for identity preservation and mouth motion"]
quality_score = 0.82
latency_p50_seconds = 240.0
def estimate_cost(self, inputs: dict[str, Any]) -> float:
base = 0.35
if inputs.get("mode") == "pro":
base *= 1.7
if inputs.get("sound_file_path") or inputs.get("audio_path"):
base += 0.04
return round(base, 4)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 240.0
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
result = super().dry_run(inputs)
result.update(
{
"paid_api": True,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative OpenMontage avatar estimate pending official account-usage reconciliation.",
}
)
return result
def execute(self, inputs: dict[str, Any]) -> ToolResult:
try:
self.check_dependencies()
except DependencyError as exc:
return ToolResult(success=False, error=str(exc))
start = time.time()
try:
request = self._build_request(inputs)
client = KlingClient()
task_id = client.create_classic_task(request["path"], request["payload"])
outputs = client.poll_classic(
request["path"],
task_id,
"videos",
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
poll_interval=float(inputs.get("poll_interval", 5.0)),
)
paths = self._download_videos(client, outputs, inputs)
probed = probe_output(paths[0])
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
data: dict[str, Any] = {"provider": self.provider}
if isinstance(exc, KlingAPIError):
data.update(
{
"error_code": exc.code,
"request_id": exc.request_id,
"http_status": exc.http_status,
"account_usage_diagnostic": account_usage_hint_for_error(exc),
}
)
return ToolResult(success=False, data=data, error=f"Kling official avatar generation failed: {exc}")
except Exception as exc:
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official avatar generation failed: {exc}")
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": "kling-official-avatar",
"task_id": task_id,
"operation": "image_to_avatar_video",
"mode": request["payload"].get("mode"),
"prompt": request["payload"].get("prompt"),
"avatar_source": request["avatar_source"],
"audio_source": request["audio_source"],
"remote_outputs": outputs,
"output": str(paths[0]),
"output_path": str(paths[0]),
"video_paths": [str(path) for path in paths],
"format": "mp4",
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
**self._account_usage_result(inputs, client),
**self._callback_result_data(inputs, task_id),
**probed,
},
artifacts=[str(path) for path in paths],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model="kling-official-avatar",
)
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
image = normalize_image_input(inputs.get("image_url"), inputs.get("image_path"))
if not image:
raise ValueError("Kling avatar requires image_url or image_path")
mode = str(inputs.get("mode") or "std")
if mode not in AVATAR_MODES:
raise ValueError(f"mode must be one of: {', '.join(AVATAR_MODES)}")
payload: dict[str, Any] = {
"image": image,
"mode": mode,
}
if inputs.get("prompt"):
payload["prompt"] = str(inputs["prompt"])
audio_source = self._copy_audio_input(inputs, payload)
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
"path": "/v1/videos/avatar/image2video",
"payload": payload,
"operation": "image_to_avatar_video",
"model": "kling-official-avatar",
"avatar_source": inputs.get("image_url") or inputs.get("image_path"),
"audio_source": audio_source,
}
@staticmethod
def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
audio_id = str(inputs.get("audio_id") or "").strip()
if audio_id:
payload["audio_id"] = audio_id
return {"type": "audio_id", "value": audio_id}
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
sound_file = normalize_media_input(
url=inputs.get("sound_file_url"),
path=sound_path,
value=inputs.get("sound_file"),
label="Avatar audio file",
)
if not sound_file:
raise ValueError("Kling avatar requires audio_id, sound_file, sound_file_url, sound_file_path, or audio_path")
payload["sound_file"] = sound_file
return {
"type": "sound_file",
"source": inputs.get("sound_file_url") or sound_path or "inline",
}
def _download_videos(
self,
client: KlingClient,
outputs: list[dict[str, Any]],
inputs: dict[str, Any],
) -> list[Path]:
if not outputs:
raise ValueError("Kling avatar response contained no videos")
base_path = Path(inputs.get("output_path", "kling_avatar.mp4"))
paths: list[Path] = []
for index, item in enumerate(outputs):
url = self._output_url(item)
suffix = extension_from_url(url, ".mp4")
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
client.download(url, output_path)
paths.append(output_path)
return paths
@staticmethod
def _output_url(item: dict[str, Any]) -> str:
url = item.get("url") or item.get("video_url") or item.get("resource_url")
if url:
return str(url)
resource = item.get("resource") or {}
if isinstance(resource, dict) and resource.get("url"):
return str(resource["url"])
raise ValueError(f"Kling avatar response item contained no downloadable URL: {item}")
@staticmethod
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
payload["callback_url"] = callback_url
if inputs.get("external_task_id"):
payload["external_task_id"] = inputs["external_task_id"]
@staticmethod
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
callback_url = inputs.get("callback_url")
if not callback_url:
return {}
return {
"callback_url": str(callback_url),
"callback_requested": True,
"polling_used": True,
"task_id": task_id,
}
@staticmethod
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
if not inputs.get("include_account_usage"):
return {}
try:
usage = get_account_costs(client=client)
return {
"account_usage": usage,
"cost_source": "estimate_with_account_usage_context",
"reconciled_cost_usd": None,
}
except Exception as exc:
return {
"account_usage_error": str(exc),
"cost_source": "estimate",
"reconciled_cost_usd": None,
}

View File

@@ -0,0 +1,551 @@
"""Kling official API lip-sync provider."""
from __future__ import annotations
import json
import time
from pathlib import Path
from typing import Any
from tools._kling.account import account_usage_hint_for_error, get_account_costs
from tools._kling.callbacks import validate_callback_url
from tools._kling.client import KlingClient
from tools._kling.errors import KlingAPIError
from tools._kling.media import (
extension_from_url,
normalize_media_input,
numbered_output_path,
output_path_with_suffix,
)
from tools._kling.schemas import LIP_SYNC_OPERATIONS
from tools.base_tool import (
BaseTool,
DependencyError,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolTier,
)
from tools.video._shared import probe_output
class KlingLipSync(BaseTool):
name = "kling_lip_sync"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "avatar"
provider = "kling_official"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:KLING_API_KEY"]
install_instructions = (
"Set KLING_API_KEY in .env for the official Kling API. "
"Use identify_face first for multi-person clips, then pass face_choose or face_id."
)
agent_skills = ["kling-official", "avatar-video"]
capabilities = ["lip_sync", "identify_face", "audio_video_alignment"]
supports = {
"lip_sync": True,
"face_selection": True,
"offline": False,
"cloud_render": True,
}
best_for = [
"official Kling cloud lip-sync for existing presenter video",
"dubbing workflows that can use Kling face identification",
"manual or explicit automatic face selection before paid lip-sync generation",
]
not_good_for = [
"fully offline lip-sync",
"silent first-face selection in multi-person footage",
"replacing local lip_sync behavior implicitly",
]
fallback_tools = ["lip_sync"]
input_schema = {
"type": "object",
"properties": {
"operation": {"type": "string", "enum": LIP_SYNC_OPERATIONS, "default": "advanced_lip_sync"},
"video_id": {"type": "string"},
"video_url": {"type": "string"},
"video_path": {
"type": "string",
"description": "Not silently uploaded. Provide video_url unless an official upload path is added.",
},
"session_id": {"type": "string"},
"face_id": {"type": "string"},
"face_choose": {"type": "array"},
"auto_select_face": {
"type": "boolean",
"default": False,
"description": "Explicitly allow largest-face automatic selection after identify_face.",
},
"audio_id": {"type": "string"},
"sound_file": {
"type": "string",
"description": "Official Kling sound_file value or raw base64 audio.",
},
"sound_file_url": {"type": "string"},
"sound_file_path": {"type": "string"},
"audio_path": {
"type": "string",
"description": "Alias for sound_file_path for compatibility with local lip_sync.",
},
"faces_artifact_path": {"type": "string"},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
"include_account_usage": {
"type": "boolean",
"default": False,
"description": "Optional low-frequency account usage diagnostic; not used by default.",
},
"timeout_seconds": {"type": "integer", "default": 900},
"poll_interval": {"type": "number", "default": 5.0},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(
max_retries=2,
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = ["video_id", "video_url", "session_id", "face_id", "audio_id", "sound_file_path"]
side_effects = [
"paid remote generation via official Kling API",
"writes face selection artifact",
"writes lip-synced video to output_path",
]
user_visible_verification = ["Watch output video to verify the selected face matches the new audio"]
quality_score = 0.80
latency_p50_seconds = 240.0
def estimate_cost(self, inputs: dict[str, Any]) -> float:
operation = str(inputs.get("operation", "advanced_lip_sync"))
if operation == "identify_face":
return 0.02
if operation == "full_lip_sync":
return 0.34
return 0.32
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
if inputs.get("operation") == "identify_face":
return 15.0
return 240.0
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
result = super().dry_run(inputs)
result.update(
{
"paid_api": True,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative OpenMontage estimate for identify-face plus advanced lip-sync.",
}
)
return result
def execute(self, inputs: dict[str, Any]) -> ToolResult:
try:
self.check_dependencies()
except DependencyError as exc:
return ToolResult(success=False, error=str(exc))
operation = str(inputs.get("operation") or "advanced_lip_sync")
start = time.time()
client = KlingClient()
try:
if operation == "identify_face":
identify = self._identify_faces(client, inputs)
return self._identify_result(inputs, identify, start)
if operation == "full_lip_sync":
identify = self._identify_faces(client, inputs)
artifact_path = self._write_faces_artifact(inputs, identify)
face_choose, selection = self._face_selection(identify["faces"], inputs)
artifact_path = self._write_faces_artifact(inputs, identify, selection=selection)
if selection["selection_method"] == "requires_user_selection":
return ToolResult(
success=False,
data={
"provider": self.provider,
"operation": operation,
"session_id": identify["session_id"],
"faces": identify["faces"],
"requires_face_selection": True,
"selection_reason": selection["selection_reason"],
"faces_artifact_path": str(artifact_path),
},
artifacts=[str(artifact_path)],
error="Multiple faces detected. Pass face_id/face_choose or set auto_select_face=True.",
cost_usd=self.estimate_cost({"operation": "identify_face"}),
duration_seconds=round(time.time() - start, 2),
model="kling-official-lip-sync",
)
merged = {**inputs, "session_id": identify["session_id"], "face_choose": face_choose}
request = self._build_advanced_request(merged)
result = self._run_advanced_lip_sync(client, merged, request, start)
result.data["faces_artifact_path"] = str(artifact_path)
result.data["face_selection"] = selection
result.artifacts.append(str(artifact_path))
return result
if operation == "advanced_lip_sync":
request = self._build_advanced_request(inputs)
return self._run_advanced_lip_sync(client, inputs, request, start)
raise ValueError(f"Unsupported Kling lip-sync operation: {operation}")
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
data: dict[str, Any] = {"provider": self.provider}
if isinstance(exc, KlingAPIError):
data.update(
{
"error_code": exc.code,
"request_id": exc.request_id,
"http_status": exc.http_status,
"account_usage_diagnostic": account_usage_hint_for_error(exc),
}
)
return ToolResult(success=False, data=data, error=f"Kling official lip-sync failed: {exc}")
except Exception as exc:
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official lip-sync failed: {exc}")
def _identify_faces(self, client: KlingClient, inputs: dict[str, Any]) -> dict[str, Any]:
request = self._build_identify_request(inputs)
data = client.post(request["path"], request["payload"])
payload = data.get("data") or {}
session_id = payload.get("session_id")
if not session_id:
raise ValueError(f"Kling identify-face response missing data.session_id: {data}")
faces = (
payload.get("faces")
or payload.get("face_list")
or payload.get("face_infos")
or payload.get("faces_info")
or []
)
if not isinstance(faces, list):
raise ValueError("Kling identify-face response face list is not a list")
if not faces:
raise ValueError("Kling identify-face response contained no faces")
return {
"session_id": str(session_id),
"faces": faces,
"raw_response": data,
"request": request,
}
def _identify_result(self, inputs: dict[str, Any], identify: dict[str, Any], start: float) -> ToolResult:
artifact_path = self._write_faces_artifact(inputs, identify)
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": "kling-official-lip-sync",
"operation": "identify_face",
"session_id": identify["session_id"],
"faces": identify["faces"],
"face_count": len(identify["faces"]),
"faces_artifact_path": str(artifact_path),
},
artifacts=[str(artifact_path)],
cost_usd=self.estimate_cost({"operation": "identify_face"}),
duration_seconds=round(time.time() - start, 2),
model="kling-official-lip-sync",
)
def _run_advanced_lip_sync(
self,
client: KlingClient,
inputs: dict[str, Any],
request: dict[str, Any],
start: float,
) -> ToolResult:
task_id = client.create_classic_task(request["path"], request["payload"])
outputs = client.poll_classic(
request["path"],
task_id,
"videos",
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
poll_interval=float(inputs.get("poll_interval", 5.0)),
)
paths = self._download_videos(client, outputs, inputs)
probed = probe_output(paths[0])
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": "kling-official-lip-sync",
"task_id": task_id,
"operation": request["operation"],
"session_id": request["payload"]["session_id"],
"face_choose": request["payload"]["face_choose"],
"audio_source": request["audio_source"],
"remote_outputs": outputs,
"output": str(paths[0]),
"output_path": str(paths[0]),
"video_paths": [str(path) for path in paths],
"format": "mp4",
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
**self._account_usage_result(inputs, client),
**self._callback_result_data(inputs, task_id),
**probed,
},
artifacts=[str(path) for path in paths],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model="kling-official-lip-sync",
)
def _build_identify_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
if inputs.get("video_path") and not (inputs.get("video_url") or inputs.get("video_id")):
raise ValueError("Kling identify_face requires video_url or video_id; local video paths cannot be silently uploaded.")
payload: dict[str, Any] = {}
if inputs.get("video_id"):
payload["video_id"] = str(inputs["video_id"])
if inputs.get("video_url"):
payload["video_url"] = str(inputs["video_url"])
if not payload:
raise ValueError("Kling identify_face requires video_id or video_url")
return {
"path": "/v1/videos/identify-face",
"payload": payload,
"operation": "identify_face",
}
def _build_advanced_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
session_id = str(inputs.get("session_id") or "").strip()
if not session_id:
raise ValueError("advanced_lip_sync requires session_id")
face_choose = self._normalize_face_choose(inputs)
if not face_choose:
raise ValueError("advanced_lip_sync requires face_choose or face_id")
payload: dict[str, Any] = {
"session_id": session_id,
"face_choose": face_choose,
}
audio_source = self._copy_audio_input(inputs, payload)
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
"path": "/v1/videos/advanced-lip-sync",
"payload": payload,
"operation": "advanced_lip_sync",
"model": "kling-official-lip-sync",
"audio_source": audio_source,
}
@staticmethod
def _normalize_face_choose(inputs: dict[str, Any]) -> list[dict[str, Any]]:
if inputs.get("face_choose"):
raw = inputs["face_choose"]
if isinstance(raw, dict):
raw = [raw]
if not isinstance(raw, list):
raise ValueError("face_choose must be a list of face choice objects")
normalized: list[dict[str, Any]] = []
for item in raw:
if isinstance(item, str):
normalized.append({"face_id": item})
elif isinstance(item, dict):
if not (item.get("face_id") or item.get("id")):
raise ValueError("face_choose items must include face_id")
record = dict(item)
if "face_id" not in record and record.get("id"):
record["face_id"] = record.pop("id")
normalized.append(record)
else:
raise ValueError("face_choose items must be strings or objects")
return normalized
if inputs.get("face_id"):
return [{"face_id": str(inputs["face_id"])}]
return []
def _face_selection(
self,
faces: list[dict[str, Any]],
inputs: dict[str, Any],
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
explicit = self._normalize_face_choose(inputs)
if explicit:
return explicit, {
"selection_method": "user_selected",
"selection_reason": "face_choose or face_id was provided",
"selected_face": explicit,
}
if len(faces) == 1:
choice = [self._face_to_choice(faces[0])]
return choice, {
"selection_method": "single_face",
"selection_reason": "Only one face was returned by identify_face",
"selected_face": choice,
}
if not inputs.get("auto_select_face"):
return [], {
"selection_method": "requires_user_selection",
"selection_reason": "Multiple faces detected and auto_select_face was not enabled",
"face_count": len(faces),
}
selected = max(faces, key=self._face_area)
choice = [self._face_to_choice(selected)]
return choice, {
"selection_method": "auto_selected",
"selection_reason": "auto_select_face=True selected the largest detected face area",
"selected_face": choice,
}
@staticmethod
def _face_to_choice(face: dict[str, Any]) -> dict[str, Any]:
face_id = face.get("face_id") or face.get("id")
if not face_id:
raise ValueError(f"Cannot select face without face_id/id: {face}")
return {"face_id": str(face_id)}
@staticmethod
def _face_area(face: dict[str, Any]) -> float:
for key in ("bbox", "box"):
value = face.get(key)
if isinstance(value, list) and len(value) >= 4:
third = float(value[2])
fourth = float(value[3])
width_height_area = max(third, 0.0) * max(fourth, 0.0)
corner_width = third - float(value[0])
corner_height = fourth - float(value[1])
corner_area = (
corner_width * corner_height
if corner_width > 0 and corner_height > 0
else 0.0
)
if corner_area and width_height_area:
return min(corner_area, width_height_area)
return corner_area or width_height_area
if isinstance(value, dict):
width = value.get("width") or value.get("w")
height = value.get("height") or value.get("h")
if width is not None and height is not None:
return max(float(width), 0.0) * max(float(height), 0.0)
width = face.get("width") or face.get("w")
height = face.get("height") or face.get("h")
if width is not None and height is not None:
return max(float(width), 0.0) * max(float(height), 0.0)
return 0.0
@staticmethod
def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
audio_id = str(inputs.get("audio_id") or "").strip()
if audio_id:
payload["audio_id"] = audio_id
return {"type": "audio_id", "value": audio_id}
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
sound_file = normalize_media_input(
url=inputs.get("sound_file_url"),
path=sound_path,
value=inputs.get("sound_file"),
label="Lip-sync audio file",
)
if not sound_file:
raise ValueError("advanced_lip_sync requires audio_id, sound_file, sound_file_url, sound_file_path, or audio_path")
payload["sound_file"] = sound_file
return {
"type": "sound_file",
"source": inputs.get("sound_file_url") or sound_path or "inline",
}
def _download_videos(
self,
client: KlingClient,
outputs: list[dict[str, Any]],
inputs: dict[str, Any],
) -> list[Path]:
if not outputs:
raise ValueError("Kling lip-sync response contained no videos")
base_path = Path(inputs.get("output_path", "kling_lip_sync.mp4"))
paths: list[Path] = []
for index, item in enumerate(outputs):
url = self._output_url(item)
suffix = extension_from_url(url, ".mp4")
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
client.download(url, output_path)
paths.append(output_path)
return paths
@staticmethod
def _output_url(item: dict[str, Any]) -> str:
url = item.get("url") or item.get("video_url") or item.get("resource_url")
if url:
return str(url)
resource = item.get("resource") or {}
if isinstance(resource, dict) and resource.get("url"):
return str(resource["url"])
raise ValueError(f"Kling lip-sync response item contained no downloadable URL: {item}")
@staticmethod
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
payload["callback_url"] = callback_url
if inputs.get("external_task_id"):
payload["external_task_id"] = inputs["external_task_id"]
@staticmethod
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
callback_url = inputs.get("callback_url")
if not callback_url:
return {}
return {
"callback_url": str(callback_url),
"callback_requested": True,
"polling_used": True,
"task_id": task_id,
}
@staticmethod
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
if not inputs.get("include_account_usage"):
return {}
try:
usage = get_account_costs(client=client)
return {
"account_usage": usage,
"cost_source": "estimate_with_account_usage_context",
"reconciled_cost_usd": None,
}
except Exception as exc:
return {
"account_usage_error": str(exc),
"cost_source": "estimate",
"reconciled_cost_usd": None,
}
def _write_faces_artifact(
self,
inputs: dict[str, Any],
identify: dict[str, Any],
selection: dict[str, Any] | None = None,
) -> Path:
if inputs.get("faces_artifact_path"):
artifact_path = Path(inputs["faces_artifact_path"])
elif inputs.get("output_path"):
artifact_path = Path(inputs["output_path"]).with_name("kling_lip_sync_faces.json")
else:
artifact_path = Path("kling_lip_sync_faces.json")
artifact_path.parent.mkdir(parents=True, exist_ok=True)
artifact = {
"provider": self.provider,
"operation": "identify_face",
"session_id": identify["session_id"],
"faces": identify["faces"],
"face_count": len(identify["faces"]),
"selection": selection,
}
artifact_path.write_text(json.dumps(artifact, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return artifact_path

View File

@@ -61,6 +61,14 @@ class ImageSelector(BaseTool):
"type": "string",
"description": "Resolution tier for providers that support named resolutions.",
},
"api_family": {
"type": "string",
"description": "Provider-specific API family hint passed through when supported.",
},
"model_name": {
"type": "string",
"description": "Provider-specific model name passed through when supported.",
},
"generation_mode": {
"type": "string",
"enum": ["generate", "edit"],
@@ -79,6 +87,46 @@ class ImageSelector(BaseTool):
"items": {"type": "string"},
"description": "Multiple local source image paths for compositing edits.",
},
"image_list": {
"type": "array",
"description": "Provider-specific image reference list, e.g. Kling Official Image Omni.",
},
"element_list": {
"type": "array",
"description": "Provider-specific element references, e.g. Kling Official element_id objects.",
},
"image_reference": {
"type": "string",
"description": "Provider-specific reference type, e.g. subject or face.",
},
"image_fidelity": {
"type": "number",
"description": "Provider-specific reference image fidelity hint.",
},
"human_fidelity": {
"type": "number",
"description": "Provider-specific human or face fidelity hint.",
},
"result_type": {
"type": "string",
"description": "Provider-specific result type, e.g. single or series.",
},
"series_amount": {
"type": "string",
"description": "Provider-specific series amount for image series generation.",
},
"watermark": {
"type": "boolean",
"description": "Provider-specific watermark toggle passed through when supported.",
},
"callback_url": {
"type": "string",
"description": "Provider-specific callback URL. Current OpenMontage providers still poll by default.",
},
"external_task_id": {
"type": "string",
"description": "Provider-specific idempotency/provenance task id.",
},
"preferred_provider": {
"type": "string",
"description": "Provider name or 'auto'. Valid values are discovered at runtime from the registry.",
@@ -216,6 +264,18 @@ class ImageSelector(BaseTool):
"image_path",
"image_urls",
"image_paths",
"image_list",
"element_list",
"api_family",
"model_name",
"image_reference",
"image_fidelity",
"human_fidelity",
"result_type",
"series_amount",
"watermark",
"callback_url",
"external_task_id",
"workflow_json",
"workflow_path",
"output_node",

View File

@@ -0,0 +1,437 @@
"""Kling official API image generation provider."""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools._kling.account import account_usage_hint_for_error, get_account_costs
from tools._kling.callbacks import validate_callback_url
from tools._kling.client import KlingClient
from tools._kling.elements import element_ids, normalize_element_list
from tools._kling.errors import KlingAPIError
from tools._kling.media import (
extension_from_url,
normalize_image_input,
numbered_output_path,
output_path_with_suffix,
)
from tools._kling.omni import build_image_prompt_references
from tools._kling.schemas import (
IMAGE_ASPECT_RATIOS,
IMAGE_GENERATION_MODELS,
IMAGE_MODELS,
IMAGE_REFERENCE_TYPES,
IMAGE_RESOLUTIONS,
IMAGE_RESULT_TYPES,
OMNI_IMAGE_MODELS,
)
from tools.base_tool import (
BaseTool,
DependencyError,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolTier,
)
class KlingOfficialImage(BaseTool):
name = "kling_official_image"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "image_generation"
provider = "kling_official"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:KLING_API_KEY"]
install_instructions = (
"Set KLING_API_KEY in .env for the official Kling API. "
"Optionally set KLING_API_BASE_URL to override the default Singapore endpoint."
)
agent_skills = ["kling-official"]
capabilities = ["generate_image", "text_to_image", "image_edit"]
supports = {
"text_to_image": True,
"image_edit": True,
"negative_prompt": True,
"aspect_ratio": True,
}
best_for = [
"official Kling image generation",
"subject or face reference generation",
"Omni multi-reference image workflows",
]
not_good_for = ["offline generation", "free generation", "non-Kling model families"]
fallback_tools = ["flux_image", "google_imagen", "openai_image", "recraft_image"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"negative_prompt": {"type": "string"},
"operation": {"type": "string", "enum": ["generate", "omni"], "default": "generate"},
"generation_mode": {"type": "string", "enum": ["generate", "edit"], "default": "generate"},
"api_family": {"type": "string", "enum": ["generation", "omni"], "default": "generation"},
"model_name": {"type": "string", "enum": IMAGE_MODELS, "default": "kling-v3"},
"image_url": {"type": "string"},
"image_path": {"type": "string"},
"image_urls": {"type": "array", "items": {"type": "string"}},
"image_paths": {"type": "array", "items": {"type": "string"}},
"image_list": {"type": "array"},
"image_reference": {"type": "string", "enum": IMAGE_REFERENCE_TYPES},
"image_fidelity": {"type": "number", "default": 0.5},
"human_fidelity": {"type": "number", "default": 0.45},
"resolution": {"type": "string", "enum": IMAGE_RESOLUTIONS, "default": "1k"},
"aspect_ratio": {"type": "string", "enum": IMAGE_ASPECT_RATIOS, "default": "16:9"},
"n": {"type": "integer", "default": 1},
"result_type": {"type": "string", "enum": IMAGE_RESULT_TYPES, "default": "single"},
"series_amount": {"type": "string"},
"element_list": {"type": "array"},
"watermark": {"type": "boolean", "default": False},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
"include_account_usage": {
"type": "boolean",
"default": False,
"description": "Optional low-frequency account usage diagnostic; not used by default.",
},
"timeout_seconds": {"type": "integer", "default": 600},
"poll_interval": {"type": "number", "default": 3.0},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=200, network_required=True
)
retry_policy = RetryPolicy(
max_retries=2,
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = [
"prompt",
"api_family",
"model_name",
"image_url",
"image_path",
"aspect_ratio",
"resolution",
"n",
]
side_effects = [
"paid remote generation via official Kling API",
"writes image file(s) to output_path",
]
user_visible_verification = ["Inspect generated image for quality, prompt adherence, and reference fidelity"]
def estimate_cost(self, inputs: dict[str, Any]) -> float:
n = int(inputs.get("n", 1) or 1)
resolution = str(inputs.get("resolution", "1k"))
api_family = str(inputs.get("api_family", "generation"))
base = 0.04 if api_family == "generation" else 0.08
if resolution == "2k":
base *= 1.8
if resolution == "4k":
base *= 3.5
if inputs.get("result_type") == "series":
base *= 1.5
amount = inputs.get("series_amount")
if amount and str(amount).isdigit():
base *= max(int(str(amount)), 1)
if api_family == "omni":
reference_count = sum(len(inputs.get(key) or []) for key in ("image_list", "image_urls", "image_paths", "element_list"))
if inputs.get("image_url") or inputs.get("image_path"):
reference_count += 1
base *= 1 + (0.08 * reference_count)
return round(base * max(n, 1), 4)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 90.0
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
result = super().dry_run(inputs)
result.update(
{
"paid_api": True,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative OpenMontage estimate; official account usage reconciliation is planned for Phase 2.",
}
)
return result
def execute(self, inputs: dict[str, Any]) -> ToolResult:
try:
self.check_dependencies()
except DependencyError as exc:
return ToolResult(success=False, error=str(exc))
start = time.time()
try:
request = self._build_request(inputs)
client = KlingClient()
task_id = client.create_classic_task(request["path"], request["payload"])
outputs = client.poll_classic(
request["path"],
task_id,
"images",
timeout_seconds=int(inputs.get("timeout_seconds", 600)),
poll_interval=float(inputs.get("poll_interval", 3.0)),
)
paths = self._download_images(client, outputs, inputs)
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
data: dict[str, Any] = {"provider": self.provider}
if isinstance(exc, KlingAPIError):
data.update(
{
"error_code": exc.code,
"request_id": exc.request_id,
"http_status": exc.http_status,
}
)
data["account_usage_diagnostic"] = account_usage_hint_for_error(exc)
return ToolResult(success=False, data=data, error=f"Kling official image generation failed: {exc}")
except Exception as exc:
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official image generation failed: {exc}")
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": request["model"],
"task_id": task_id,
"api_family": request["api_family"],
"operation": request["operation"],
"prompt": request["payload"]["prompt"],
"remote_outputs": outputs,
"output": str(paths[0]),
"output_path": str(paths[0]),
"image_paths": [str(path) for path in paths],
"format": paths[0].suffix.lstrip(".") or "png",
"references_used": request.get("references_used", []),
"element_ids": request.get("element_ids", []),
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
**self._account_usage_result(inputs, client),
**self._callback_result_data(inputs, task_id),
},
artifacts=[str(path) for path in paths],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=request["model"],
)
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
api_family = str(inputs.get("api_family", "generation"))
if inputs.get("operation") == "omni":
api_family = "omni"
if api_family == "omni":
return self._build_omni_request(inputs)
return self._build_generation_request(inputs)
def _build_generation_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
prompt = self._prompt(inputs)
model_name = str(inputs.get("model_name") or "kling-v3")
if model_name not in IMAGE_GENERATION_MODELS:
raise ValueError(f"model_name {model_name!r} is not supported for api_family=generation")
payload: dict[str, Any] = {
"model_name": model_name,
"prompt": prompt,
"resolution": inputs.get("resolution", "1k"),
"n": int(inputs.get("n", 1) or 1),
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
}
if len(prompt) > 2500:
raise ValueError("prompt exceeds Kling image generation limit of 2500 characters")
if inputs.get("negative_prompt"):
payload["negative_prompt"] = inputs["negative_prompt"]
image = normalize_image_input(inputs.get("image_url"), inputs.get("image_path"))
if image:
payload["image"] = image
if inputs.get("image_reference"):
payload["image_reference"] = inputs["image_reference"]
for key in ("image_fidelity", "human_fidelity"):
if inputs.get(key) is not None:
payload[key] = inputs[key]
elements = normalize_element_list(inputs.get("element_list"))
if elements:
payload["element_list"] = elements
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
"path": "/v1/images/generations",
"payload": payload,
"api_family": "generation",
"operation": "generate",
"model": payload["model_name"],
"references_used": self._reference_metadata_from_generation_payload(payload),
"element_ids": element_ids(payload.get("element_list")),
}
def _build_omni_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
model_name = str(inputs.get("model_name") or "kling-image-o1")
if model_name not in OMNI_IMAGE_MODELS:
raise ValueError(f"model_name {model_name!r} is not supported for api_family=omni")
image_items, references_used = self._normalize_omni_image_list(inputs)
prompt, prompt_references = build_image_prompt_references(self._prompt(inputs), image_items)
references_used = prompt_references or references_used
elements = normalize_element_list(inputs.get("element_list"))
payload: dict[str, Any] = {
"model_name": model_name,
"prompt": prompt,
"resolution": inputs.get("resolution", "1k"),
"n": int(inputs.get("n", 1) or 1),
"result_type": inputs.get("result_type", "single"),
"aspect_ratio": inputs.get("aspect_ratio", "16:9"),
}
if image_items:
payload["image_list"] = [{"image": item["image"]} for item in image_items]
if elements:
payload["element_list"] = elements
if inputs.get("series_amount"):
payload["series_amount"] = inputs["series_amount"]
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
"path": "/v1/images/omni-image",
"payload": payload,
"api_family": "omni",
"operation": "generate",
"model": model_name,
"references_used": references_used,
"element_ids": [item["element_id"] for item in elements],
}
def _normalize_omni_image_list(
self,
inputs: dict[str, Any],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
image_list: list[dict[str, Any]] = []
references_used: list[dict[str, Any]] = []
def add_image(value: str | None, *, source: str | None, source_type: str) -> None:
if not value:
return
image_list.append({"image": value, "source": source or value, "source_type": source_type})
references_used.append(
{
"kind": "image",
"source": source or value,
"source_type": source_type,
"placeholder": f"<<<image_{len(image_list)}>>>",
}
)
for item in inputs.get("image_list") or []:
if not isinstance(item, dict):
raise ValueError("image_list items must be objects")
source = item.get("image") or item.get("image_url") or item.get("image_path")
value = normalize_image_input(item.get("image") or item.get("image_url"), item.get("image_path"))
if not value:
raise ValueError("image_list items must include image, image_url, or image_path")
add_image(value, source=source, source_type="image_list")
for url in inputs.get("image_urls") or []:
add_image(normalize_image_input(url=url), source=url, source_type="image_urls")
for path in inputs.get("image_paths") or []:
add_image(normalize_image_input(path=path), source=str(path), source_type="image_paths")
if inputs.get("image_url") or inputs.get("image_path"):
add_image(
normalize_image_input(inputs.get("image_url"), inputs.get("image_path")),
source=inputs.get("image_url") or inputs.get("image_path"),
source_type="image",
)
return image_list, references_used
def _download_images(self, client: KlingClient, outputs: list[dict[str, Any]], inputs: dict[str, Any]) -> list[Path]:
if not outputs:
raise ValueError("Kling image response contained no images")
base_path = Path(inputs.get("output_path", "kling_official_image.png"))
paths: list[Path] = []
for index, item in enumerate(outputs):
url = self._output_url(item)
suffix = extension_from_url(url, ".png")
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
client.download(url, output_path)
paths.append(output_path)
return paths
@staticmethod
def _output_url(item: dict[str, Any]) -> str:
url = item.get("url") or item.get("image_url") or item.get("resource_url")
if url:
return str(url)
resource = item.get("resource") or {}
if isinstance(resource, dict) and resource.get("url"):
return str(resource["url"])
raise ValueError(f"Kling image response item contained no downloadable URL: {item}")
@staticmethod
def _prompt(inputs: dict[str, Any]) -> str:
prompt = str(inputs.get("prompt") or "").strip()
if not prompt:
raise ValueError("prompt is required")
return prompt
@staticmethod
def _copy_common_task_fields(inputs: dict[str, Any], payload: dict[str, Any]) -> None:
if "watermark" in inputs:
payload["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
payload["callback_url"] = callback_url
if inputs.get("external_task_id"):
payload["external_task_id"] = inputs["external_task_id"]
@staticmethod
def _reference_metadata_from_generation_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
references: list[dict[str, Any]] = []
if payload.get("image"):
references.append({"kind": "image", "source_type": "image"})
if payload.get("element_list"):
references.extend(
{"kind": "element", "element_id": item["element_id"]}
for item in normalize_element_list(payload.get("element_list"))
)
return references
@staticmethod
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
callback_url = inputs.get("callback_url")
if not callback_url:
return {}
return {
"callback_url": str(callback_url),
"callback_requested": True,
"polling_used": True,
"task_id": task_id,
}
@staticmethod
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
if not inputs.get("include_account_usage"):
return {}
try:
usage = get_account_costs(client=client)
return {
"account_usage": usage,
"cost_source": "estimate_with_account_usage_context",
"reconciled_cost_usd": None,
}
except Exception as exc:
return {
"account_usage_error": str(exc),
"cost_source": "estimate",
"reconciled_cost_usd": None,
}

View File

@@ -0,0 +1,684 @@
"""Kling official API video generation provider."""
from __future__ import annotations
import time
from pathlib import Path
from typing import Any
from tools._kling.account import account_usage_hint_for_error, get_account_costs
from tools._kling.callbacks import validate_callback_url
from tools._kling.client import KlingClient
from tools._kling.elements import element_ids, normalize_element_list
from tools._kling.errors import KlingAPIError
from tools._kling.media import (
extension_from_url,
normalize_image_input,
numbered_output_path,
output_path_with_suffix,
)
from tools._kling.schemas import (
CLASSIC_VIDEO_MODELS,
OMNI_VIDEO_MODELS,
SOUND_VALUES,
VIDEO_ASPECT_RATIOS,
VIDEO_DURATIONS,
VIDEO_MODES,
VIDEO_MODELS,
VIDEO_RESOLUTIONS,
)
from tools.base_tool import (
BaseTool,
DependencyError,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolTier,
)
from tools.video._shared import probe_output
class KlingOfficialVideo(BaseTool):
name = "kling_official_video"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "video_generation"
provider = "kling_official"
stability = ToolStability.EXPERIMENTAL
execution_mode = ExecutionMode.SYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:KLING_API_KEY"]
install_instructions = (
"Set KLING_API_KEY in .env for the official Kling API. "
"Optionally set KLING_API_BASE_URL to override the default Singapore endpoint."
)
agent_skills = ["ai-video-gen", "kling-official"]
capabilities = ["text_to_video", "image_to_video", "reference_to_video"]
supports = {
"text_to_video": True,
"image_to_video": True,
"reference_to_video": True,
"reference_image": True,
"negative_prompt": True,
"aspect_ratio": True,
}
best_for = [
"official Kling direct API access",
"text-to-video and image-to-video with Kling model controls",
"projects that need provider provenance separate from fal.ai Kling",
]
not_good_for = ["offline generation", "free generation", "non-Kling model families"]
fallback_tools = ["kling_video", "seedance_video", "veo_video", "minimax_video"]
input_schema = {
"type": "object",
"required": ["prompt"],
"properties": {
"prompt": {"type": "string"},
"operation": {
"type": "string",
"enum": ["text_to_video", "image_to_video", "reference_to_video", "omni_video"],
"default": "text_to_video",
},
"api_family": {
"type": "string",
"enum": ["classic", "turbo", "omni"],
"default": "classic",
},
"model_name": {"type": "string", "enum": VIDEO_MODELS, "default": "kling-v3"},
"model_variant": {"type": "string", "description": "Compatibility alias for model_name."},
"duration": {"type": "string", "enum": VIDEO_DURATIONS, "default": "5"},
"aspect_ratio": {"type": "string", "enum": VIDEO_ASPECT_RATIOS, "default": "16:9"},
"resolution": {"type": "string", "enum": VIDEO_RESOLUTIONS, "default": "720p"},
"mode": {"type": "string", "enum": VIDEO_MODES, "default": "std"},
"sound": {"type": "string", "enum": SOUND_VALUES, "default": "off"},
"negative_prompt": {"type": "string"},
"cfg_scale": {"type": "number", "default": 0.5},
"reference_image_url": {"type": "string"},
"reference_image_path": {"type": "string"},
"reference_tail_image_url": {"type": "string"},
"reference_tail_image_path": {"type": "string"},
"reference_image_urls": {"type": "array", "items": {"type": "string"}},
"reference_image_paths": {"type": "array", "items": {"type": "string"}},
"reference_video_url": {"type": "string"},
"reference_video_path": {"type": "string"},
"video_urls": {"type": "array", "items": {"type": "string"}},
"video_paths": {"type": "array", "items": {"type": "string"}},
"image_list": {"type": "array"},
"video_list": {"type": "array"},
"element_list": {"type": "array"},
"multi_shot": {"type": "boolean"},
"shot_type": {"type": "string", "enum": ["customize", "intelligence"]},
"multi_prompt": {"type": "array"},
"camera_control": {"type": "object"},
"watermark": {"type": "boolean", "default": False},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
"include_account_usage": {
"type": "boolean",
"default": False,
"description": "Optional low-frequency account usage diagnostic; not used by default.",
},
"timeout_seconds": {"type": "integer", "default": 900},
"poll_interval": {"type": "number", "default": 5.0},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1, ram_mb=512, vram_mb=0, disk_mb=500, network_required=True
)
retry_policy = RetryPolicy(
max_retries=2,
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = [
"prompt",
"operation",
"api_family",
"model_name",
"reference_image_url",
"reference_image_path",
"duration",
"aspect_ratio",
]
side_effects = [
"paid remote generation via official Kling API",
"writes video file to output_path",
]
user_visible_verification = ["Watch generated clip for motion coherence and prompt adherence"]
def estimate_cost(self, inputs: dict[str, Any]) -> float:
duration = int(str(inputs.get("duration", "5")))
mode = str(inputs.get("mode", "std"))
api_family = str(inputs.get("api_family", "classic"))
base = 0.18
if api_family == "turbo":
base = 0.22
if api_family == "omni":
base = 0.30
if mode == "pro":
base *= 1.6
if mode == "4k":
base *= 3.0
if inputs.get("sound") == "on":
base += 0.05
if api_family == "omni":
reference_count = self._estimate_reference_count(inputs)
base *= 1 + (0.12 * reference_count)
multi_prompt = inputs.get("multi_prompt") or []
if multi_prompt:
base *= 1 + (0.10 * len(multi_prompt))
return round(base * max(duration, 3) / 5, 4)
def estimate_runtime(self, inputs: dict[str, Any]) -> float:
return 180.0
def dry_run(self, inputs: dict[str, Any]) -> dict[str, Any]:
result = super().dry_run(inputs)
result.update(
{
"paid_api": True,
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative OpenMontage estimate; official account usage reconciliation is planned for Phase 2.",
}
)
return result
def execute(self, inputs: dict[str, Any]) -> ToolResult:
try:
self.check_dependencies()
except DependencyError as exc:
return ToolResult(success=False, error=str(exc))
start = time.time()
try:
request = self._build_request(inputs)
client = KlingClient()
if request["protocol"] == "turbo":
task_id = client.create_turbo(request["path"], request["payload"])
outputs = client.poll_turbo(
task_id,
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
poll_interval=float(inputs.get("poll_interval", 5.0)),
)
else:
task_id = client.create_classic_task(request["path"], request["payload"])
outputs = client.poll_classic(
request["path"],
task_id,
"videos",
timeout_seconds=int(inputs.get("timeout_seconds", 900)),
poll_interval=float(inputs.get("poll_interval", 5.0)),
)
paths = self._download_videos(client, outputs, inputs)
video_url = self._first_output_url(outputs)
probed = probe_output(paths[0])
except (KlingAPIError, TimeoutError, ValueError, KeyError, FileNotFoundError) as exc:
data: dict[str, Any] = {"provider": self.provider}
if isinstance(exc, KlingAPIError):
data.update(
{
"error_code": exc.code,
"request_id": exc.request_id,
"http_status": exc.http_status,
}
)
data["account_usage_diagnostic"] = account_usage_hint_for_error(exc)
return ToolResult(success=False, data=data, error=f"Kling official video generation failed: {exc}")
except Exception as exc:
return ToolResult(success=False, data={"provider": self.provider}, error=f"Kling official video generation failed: {exc}")
return ToolResult(
success=True,
data={
"provider": self.provider,
"model": request["model"],
"task_id": task_id,
"operation": request["operation"],
"api_family": request["api_family"],
"prompt": inputs["prompt"],
"remote_url": video_url,
"remote_outputs": outputs,
"output": str(paths[0]),
"output_path": str(paths[0]),
"video_paths": [str(path) for path in paths],
"format": "mp4",
"references_used": request.get("references_used", []),
"element_ids": request.get("element_ids", []),
"cost_estimate_confidence": "low",
"cost_estimate_basis": "Conservative estimate pending official account-usage reconciliation.",
**self._account_usage_result(inputs, client),
**self._callback_result_data(inputs, task_id),
**probed,
},
artifacts=[str(path) for path in paths],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - start, 2),
model=request["model"],
)
def _build_request(self, inputs: dict[str, Any]) -> dict[str, Any]:
operation = str(inputs.get("operation", "text_to_video"))
api_family = str(inputs.get("api_family", "classic"))
if operation == "omni_video":
operation = "reference_to_video"
api_family = "omni"
if api_family == "turbo":
return self._build_turbo_request(inputs, operation)
if api_family == "omni":
return self._build_omni_request(inputs, operation)
return self._build_classic_request(inputs, operation)
def _build_classic_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
if operation == "text_to_video":
payload = self._base_classic_payload(inputs)
payload["prompt"] = self._prompt(inputs)
if inputs.get("negative_prompt"):
payload["negative_prompt"] = inputs["negative_prompt"]
if inputs.get("aspect_ratio"):
payload["aspect_ratio"] = inputs.get("aspect_ratio", "16:9")
self._copy_multi_shot_fields(inputs, payload)
path = "/v1/videos/text2video"
elif operation == "image_to_video":
image = normalize_image_input(inputs.get("reference_image_url"), inputs.get("reference_image_path"))
if not image:
raise ValueError("image_to_video requires reference_image_url or reference_image_path")
payload = self._base_classic_payload(inputs)
payload["image"] = image
if inputs.get("prompt"):
payload["prompt"] = inputs["prompt"]
if inputs.get("negative_prompt"):
payload["negative_prompt"] = inputs["negative_prompt"]
tail = normalize_image_input(inputs.get("reference_tail_image_url"), inputs.get("reference_tail_image_path"))
if tail:
payload["image_tail"] = tail
if inputs.get("element_list"):
payload["element_list"] = normalize_element_list(inputs.get("element_list"))
self._copy_multi_shot_fields(inputs, payload)
path = "/v1/videos/image2video"
else:
raise ValueError(f"Unsupported classic video operation: {operation}")
return {
"protocol": "classic",
"path": path,
"payload": payload,
"operation": operation,
"api_family": "classic",
"model": payload["model_name"],
"references_used": self._reference_metadata_from_classic_payload(payload),
"element_ids": element_ids(payload.get("element_list")),
}
def _build_turbo_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
settings = {
"resolution": inputs.get("resolution", "720p"),
"duration": int(str(inputs.get("duration", "5"))),
}
options = self._options_payload(inputs)
if operation == "text_to_video":
settings["aspect_ratio"] = inputs.get("aspect_ratio", "16:9")
payload = {"prompt": self._prompt(inputs), "settings": settings}
if options:
payload["options"] = options
path = "/text-to-video/kling-3.0-turbo"
elif operation == "image_to_video":
if inputs.get("reference_image_path") and not inputs.get("reference_image_url"):
raise ValueError("Turbo image_to_video requires reference_image_url; local paths cannot be silently uploaded.")
image_url = inputs.get("reference_image_url")
if not image_url:
raise ValueError("image_to_video requires reference_image_url for api_family=turbo")
contents = [{"type": "prompt", "text": self._prompt(inputs)}, {"type": "first_frame", "url": image_url}]
payload = {"contents": contents, "settings": settings}
if options:
payload["options"] = options
path = "/image-to-video/kling-3.0-turbo"
else:
raise ValueError(f"Unsupported turbo video operation: {operation}")
return {
"protocol": "turbo",
"path": path,
"payload": payload,
"operation": operation,
"api_family": "turbo",
"model": "kling-3.0-turbo",
}
def _build_omni_request(self, inputs: dict[str, Any], operation: str) -> dict[str, Any]:
explicit_model = inputs.get("model_name") or inputs.get("model_variant")
model_name = str(explicit_model or "kling-video-o1")
if model_name not in OMNI_VIDEO_MODELS:
raise ValueError(f"model_name {model_name!r} is not supported for api_family=omni")
payload, references_used, element_id_values = self._build_omni_payload(inputs, operation, model_name)
return {
"protocol": "classic",
"path": "/v1/videos/omni-video",
"payload": payload,
"operation": operation,
"api_family": "omni",
"model": model_name,
"references_used": references_used,
"element_ids": element_id_values,
}
def _build_omni_payload(
self,
inputs: dict[str, Any],
operation: str,
model_name: str,
) -> tuple[dict[str, Any], list[dict[str, Any]], list[int]]:
payload: dict[str, Any] = {
"model_name": model_name,
"prompt": self._prompt(inputs),
"mode": inputs.get("mode", "pro"),
"duration": str(inputs.get("duration", "5")),
}
if inputs.get("sound"):
payload["sound"] = inputs["sound"]
if inputs.get("aspect_ratio"):
payload["aspect_ratio"] = inputs["aspect_ratio"]
self._copy_common_task_fields(inputs, payload)
self._copy_multi_shot_fields(inputs, payload)
references_used: list[dict[str, Any]] = []
image_list, image_refs = self._normalize_omni_image_list(inputs)
references_used.extend(image_refs)
if image_list:
payload["image_list"] = image_list
video_list, video_refs = self._normalize_omni_video_list(inputs)
references_used.extend(video_refs)
if video_list:
payload["video_list"] = video_list
elements = normalize_element_list(inputs.get("element_list"))
element_id_values = [item["element_id"] for item in elements]
if elements:
payload["element_list"] = elements
references_used.extend(
{"kind": "element", "element_id": item["element_id"]}
for item in elements
)
if operation == "reference_to_video" and not any(payload.get(k) for k in ("image_list", "video_list", "element_list")):
raise ValueError("reference_to_video with api_family=omni requires image_list, video_list, element_list, or reference image URLs.")
return payload, references_used, element_id_values
def _base_classic_payload(self, inputs: dict[str, Any]) -> dict[str, Any]:
model_name = str(inputs.get("model_name") or inputs.get("model_variant") or "kling-v3")
if model_name not in CLASSIC_VIDEO_MODELS:
raise ValueError(f"model_name {model_name!r} is not supported for api_family=classic")
payload: dict[str, Any] = {
"model_name": model_name,
"duration": str(inputs.get("duration", "5")),
"mode": inputs.get("mode", "std"),
"sound": inputs.get("sound", "off"),
}
if inputs.get("cfg_scale") is not None:
payload["cfg_scale"] = inputs["cfg_scale"]
if inputs.get("camera_control"):
payload["camera_control"] = inputs["camera_control"]
self._copy_common_task_fields(inputs, payload)
return payload
def _options_payload(self, inputs: dict[str, Any]) -> dict[str, Any]:
options: dict[str, Any] = {}
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
options["callback_url"] = callback_url
if inputs.get("external_task_id"):
options["external_task_id"] = inputs["external_task_id"]
if "watermark" in inputs:
options["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
return options
def _copy_common_task_fields(self, inputs: dict[str, Any], payload: dict[str, Any]) -> None:
if "watermark" in inputs:
payload["watermark_info"] = {"enabled": bool(inputs.get("watermark"))}
callback_url = validate_callback_url(inputs.get("callback_url"))
if callback_url:
payload["callback_url"] = callback_url
if inputs.get("external_task_id"):
payload["external_task_id"] = inputs["external_task_id"]
def _copy_multi_shot_fields(self, inputs: dict[str, Any], payload: dict[str, Any]) -> None:
if inputs.get("multi_shot") is None and not inputs.get("multi_prompt"):
return
payload["multi_shot"] = bool(inputs.get("multi_shot", True))
shot_type = str(inputs.get("shot_type") or "customize")
if shot_type not in {"customize", "intelligence"}:
raise ValueError("shot_type must be one of: customize, intelligence")
payload["shot_type"] = shot_type
if inputs.get("multi_prompt"):
if not isinstance(inputs["multi_prompt"], list):
raise ValueError("multi_prompt must be a list")
normalized: list[dict[str, Any]] = []
for item in inputs["multi_prompt"]:
if not isinstance(item, dict) or not item.get("prompt"):
raise ValueError("each multi_prompt item must be an object with prompt")
allowed = {
key: item[key]
for key in ("prompt", "duration", "camera_control", "image_refs", "element_refs")
if key in item
}
normalized.append(allowed)
payload["multi_prompt"] = normalized
def _normalize_omni_image_list(
self,
inputs: dict[str, Any],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
image_list: list[dict[str, Any]] = []
references_used: list[dict[str, Any]] = []
def add_image(value: str | None, *, kind: str, item_type: str | None = None) -> None:
if not value:
return
record = {"image_url": value}
if item_type:
record["type"] = item_type
image_list.append(record)
references_used.append(
{
"kind": "image",
"source": value,
"source_type": kind,
"type": item_type,
}
)
for item in inputs.get("image_list") or []:
if not isinstance(item, dict):
raise ValueError("image_list items must be objects")
value = normalize_image_input(item.get("image_url") or item.get("image"), item.get("image_path"))
if not value:
raise ValueError("image_list items must include image_url, image, or image_path")
record = {"image_url": value}
if item.get("type"):
record["type"] = item["type"]
image_list.append(record)
references_used.append(
{
"kind": "image",
"source": item.get("image_url") or item.get("image_path") or item.get("image"),
"source_type": "image_list",
"type": item.get("type"),
}
)
add_image(
normalize_image_input(inputs.get("reference_image_url"), inputs.get("reference_image_path")),
kind="reference_image",
item_type="first_frame",
)
add_image(
normalize_image_input(inputs.get("reference_tail_image_url"), inputs.get("reference_tail_image_path")),
kind="reference_tail_image",
item_type="end_frame",
)
for url in inputs.get("reference_image_urls") or []:
add_image(normalize_image_input(url=url), kind="reference_image_urls")
for path in inputs.get("reference_image_paths") or []:
add_image(normalize_image_input(path=path), kind="reference_image_paths")
return image_list, references_used
def _normalize_omni_video_list(
self,
inputs: dict[str, Any],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
if inputs.get("reference_video_path") or inputs.get("video_paths"):
raise ValueError("Video Omni requires video URLs; local video paths cannot be silently uploaded.")
video_list: list[dict[str, Any]] = []
references_used: list[dict[str, Any]] = []
def add_video(item: dict[str, Any], source_type: str) -> None:
if item.get("video_path"):
raise ValueError("Video Omni requires video URLs; local video paths cannot be silently uploaded.")
video_url = item.get("video_url") or item.get("url")
if not video_url:
raise ValueError("video_list items must include video_url")
record = {"video_url": video_url}
if item.get("refer_type"):
record["refer_type"] = item["refer_type"]
if "keep_original_sound" in item:
value = item["keep_original_sound"]
record["keep_original_sound"] = "yes" if value is True else "no" if value is False else value
video_list.append(record)
references_used.append(
{
"kind": "video",
"source": video_url,
"source_type": source_type,
"refer_type": record.get("refer_type"),
"keep_original_sound": record.get("keep_original_sound"),
}
)
for item in inputs.get("video_list") or []:
if not isinstance(item, dict):
raise ValueError("video_list items must be objects")
add_video(item, "video_list")
if inputs.get("reference_video_url"):
add_video({"video_url": inputs["reference_video_url"]}, "reference_video_url")
for url in inputs.get("video_urls") or []:
add_video({"video_url": url}, "video_urls")
return video_list, references_used
def _download_videos(
self,
client: KlingClient,
outputs: list[dict[str, Any]],
inputs: dict[str, Any],
) -> list[Path]:
if not outputs:
raise ValueError("Kling video response contained no videos")
base_path = Path(inputs.get("output_path", "kling_official_video.mp4"))
paths: list[Path] = []
for index, item in enumerate(outputs):
url = self._output_url(item)
suffix = extension_from_url(url, ".mp4")
output_path = numbered_output_path(output_path_with_suffix(base_path, suffix), index, suffix)
client.download(url, output_path)
paths.append(output_path)
return paths
@staticmethod
def _output_url(item: dict[str, Any]) -> str:
url = item.get("url") or item.get("video_url") or item.get("resource_url")
if url:
return str(url)
resource = item.get("resource") or {}
if isinstance(resource, dict) and resource.get("url"):
return str(resource["url"])
raise ValueError(f"Kling video response contained no downloadable URL: {item}")
@staticmethod
def _reference_metadata_from_classic_payload(payload: dict[str, Any]) -> list[dict[str, Any]]:
references: list[dict[str, Any]] = []
if payload.get("image"):
references.append({"kind": "image", "source_type": "reference_image"})
if payload.get("image_tail"):
references.append({"kind": "image", "source_type": "reference_tail_image"})
if payload.get("element_list"):
references.extend(
{"kind": "element", "element_id": item["element_id"]}
for item in normalize_element_list(payload.get("element_list"))
)
return references
@staticmethod
def _callback_result_data(inputs: dict[str, Any], task_id: str) -> dict[str, Any]:
callback_url = inputs.get("callback_url")
if not callback_url:
return {}
return {
"callback_url": str(callback_url),
"callback_requested": True,
"polling_used": True,
"task_id": task_id,
}
@staticmethod
def _account_usage_result(inputs: dict[str, Any], client: KlingClient) -> dict[str, Any]:
if not inputs.get("include_account_usage"):
return {}
try:
usage = get_account_costs(client=client)
return {
"account_usage": usage,
"cost_source": "estimate_with_account_usage_context",
"reconciled_cost_usd": None,
}
except Exception as exc:
return {
"account_usage_error": str(exc),
"cost_source": "estimate",
"reconciled_cost_usd": None,
}
@staticmethod
def _estimate_reference_count(inputs: dict[str, Any]) -> int:
count = 0
for key in (
"image_list",
"video_list",
"element_list",
"reference_image_urls",
"reference_image_paths",
"video_urls",
):
count += len(inputs.get(key) or [])
for key in (
"reference_image_url",
"reference_image_path",
"reference_tail_image_url",
"reference_tail_image_path",
"reference_video_url",
):
if inputs.get(key):
count += 1
return count
@staticmethod
def _prompt(inputs: dict[str, Any]) -> str:
prompt = str(inputs.get("prompt") or "").strip()
if not prompt:
raise ValueError("prompt is required")
return prompt
@staticmethod
def _first_output_url(outputs: list[dict[str, Any]]) -> str:
for item in outputs:
try:
return KlingOfficialVideo._output_url(item)
except ValueError:
continue
raise ValueError(f"Kling video response contained no downloadable URL: {outputs}")

View File

@@ -88,6 +88,38 @@ class VideoSelector(BaseTool):
"items": {"type": "string"},
"description": "Local reference image paths for providers that support reference-conditioned video.",
},
"reference_video_url": {
"type": "string",
"description": "Reference video URL for providers that support video-conditioned generation.",
},
"reference_video_path": {
"type": "string",
"description": "Local reference video path. Providers that require URLs should reject this clearly.",
},
"image_list": {
"type": "array",
"description": "Provider-specific list of image references, e.g. Kling Official Video Omni.",
},
"video_list": {
"type": "array",
"description": "Provider-specific list of video references, e.g. Kling Official Video Omni.",
},
"element_list": {
"type": "array",
"description": "Provider-specific element references, e.g. Kling Official element_id objects.",
},
"multi_shot": {
"type": "boolean",
"description": "Provider-specific multi-shot mode.",
},
"shot_type": {
"type": "string",
"description": "Provider-specific multi-shot type.",
},
"multi_prompt": {
"type": "array",
"description": "Structured multi-shot prompts; not inferred from prose.",
},
"image_url": {
"type": "string",
"description": "Alias for reference_image_url (used by some providers like Kling via fal.ai).",
@@ -96,6 +128,34 @@ class VideoSelector(BaseTool):
"type": "string",
"description": "Resolution hint for providers that support named output resolutions.",
},
"api_family": {
"type": "string",
"description": "Provider-specific API family hint passed through when supported, e.g. classic/turbo/omni.",
},
"model_name": {
"type": "string",
"description": "Provider-specific model name passed through when supported.",
},
"mode": {
"type": "string",
"description": "Provider-specific quality mode passed through when supported.",
},
"sound": {
"type": "string",
"description": "Provider-specific native audio toggle passed through when supported.",
},
"watermark": {
"type": "boolean",
"description": "Provider-specific watermark toggle passed through when supported.",
},
"callback_url": {
"type": "string",
"description": "Provider-specific callback URL. Current OpenMontage providers still poll by default.",
},
"external_task_id": {
"type": "string",
"description": "Provider-specific idempotency/provenance task id.",
},
"workflow_json": {
"type": "string",
"description": (