zlei6/OpenMontage
1
0
Fork 0
mirror of https://github.com/calesthio/OpenMontage.git synced 2026-08-24 00:56:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: harden Kling integration verification

Browse Source 
Isolate Kling contract tests from the singleton registry so discovery state cannot leak into later selector tests. Align lip-sync face, audio, and timing payloads with the current official API and extend the live smoke coverage.
This commit is contained in:
xucailiang
2026-07-10 20:53:45 +08:00
parent 2b6d717f00
commit b9b9b82b64
11 changed files with 676 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.agents/skills/kling-official/SKILL.md
View File

@@ -141,8 +141,8 @@ Keep it separate from local `talking_head`. Pipelines that want Kling avatar out
`kling_lip_sync` has two steps:
1. `POST /v1/videos/identify-face` with `video_id` or `video_url`
2. `POST /v1/videos/advanced-lip-sync` with `session_id`, `face_choose[]`, and `audio_id` or `sound_file`
1. `POST /v1/videos/identify-face` with `video_id` or `video_url`; read faces from `data.face_data[]`
2. `POST /v1/videos/advanced-lip-sync` with `session_id` and one `face_choose[]` item containing `face_id`, `audio_id` or `sound_file`, and the sound start/end/insert times
Local video paths must not be silently uploaded through fal.ai or any other provider. If multiple faces are returned and the user did not pass `face_id` or `face_choose`, stop and return the face list for confirmation unless `auto_select_face=True` was explicitly set. If auto-selecting, record the selection reason and selected face in the result/artifact.

241
scripts/kling_official_animated_explainer_e2e.py
View File

@@ -4,8 +4,9 @@
This script validates the official Kling provider path through OpenMontage
selectors and the animated-explainer asset/compose surface.
Default mode is a no-cost dry run. Use --live-tts for one paid TTS sample, or
--live-full for TTS + image + image-to-video + local FFmpeg compose.
Default mode is a no-cost dry run. Use --live-tts for one paid TTS sample,
--live-full for TTS + image + image-to-video + local FFmpeg compose, or
--live-all to add avatar and lip-sync provider smokes.
"""
from __future__ import annotations
@@ -113,6 +114,8 @@ def _tool_statuses() -> dict[str, str]:
"kling_tts",
"kling_official_image",
"kling_official_video",
"kling_avatar",
"kling_lip_sync",
]
statuses: dict[str, str] = {}
for name in names:
@@ -123,7 +126,7 @@ def _tool_statuses() -> dict[str, str]:
def _capability_summary() -> dict[str, Any]:
summary = registry.provider_menu_summary()
wanted = {"tts", "image_generation", "video_generation", "video_post"}
wanted = {"tts", "image_generation", "video_generation", "avatar", "video_post"}
return {
"composition_runtimes": summary.get("composition_runtimes", {}),
"capabilities": [
@@ -454,6 +457,198 @@ def _run_live_full(
}
def _first_remote_url(result_data: dict[str, Any]) -> str:
direct = result_data.get("remote_url")
if direct:
return str(direct)
for item in result_data.get("remote_outputs") or []:
if not isinstance(item, dict):
continue
url = item.get("url") or item.get("video_url") or item.get("resource_url")
if url:
return str(url)
raise RuntimeError("Kling result did not include a remote video URL for lip-sync input.")
def _run_live_avatar_suite(
project_dir: Path,
*,
timeout_seconds: int,
poll_interval: float,
) -> dict[str, Any]:
image = registry.get("image_selector")
avatar = registry.get("kling_avatar")
lip_sync = registry.get("kling_lip_sync")
assert image and avatar and lip_sync
assets_dir = project_dir / "assets"
image_dir = assets_dir / "images"
video_dir = assets_dir / "video"
artifacts_dir = project_dir / "artifacts"
narration_path = assets_dir / "audio" / "narration.mp3"
if not narration_path.is_file():
raise RuntimeError(
f"Avatar smoke requires an existing narration file: {narration_path}. "
"Run --live-full or --live-all first."
)
_announce_paid_call(
"image_selector -> kling_official_image",
"kling_official",
"kling-v3 generation",
"Create one synthetic single-face portrait for the avatar provider smoke.",
"sample",
)
portrait_result = image.execute(
{
"preferred_provider": "kling_official",
"allowed_providers": ["kling_official"],
"prompt": (
"Photorealistic studio portrait of one fictional adult presenter, front-facing, "
"head and shoulders centered, neutral expression, mouth closed, even soft lighting, "
"plain background, no text, no watermark."
),
"negative_prompt": "multiple people, profile view, open mouth, obscured face, text, watermark",
"api_family": "generation",
"model_name": "kling-v3",
"resolution": "1k",
"aspect_ratio": "1:1",
"n": 1,
"output_path": str(image_dir / "avatar_portrait.png"),
}
)
_require_success("avatar portrait", portrait_result)
portrait_path = Path(portrait_result.data["output_path"])
_announce_paid_call(
"kling_avatar",
"kling_official",
"kling-official-avatar std",
"Validate photo-and-audio to avatar video through the official provider.",
"sample",
)
avatar_result = avatar.execute(
{
"image_path": str(portrait_path),
"audio_path": str(narration_path),
"prompt": "Natural presenter delivery with subtle head movement and stable identity.",
"mode": "std",
"timeout_seconds": max(timeout_seconds, 900),
"poll_interval": poll_interval,
"output_path": str(video_dir / "kling_avatar_smoke.mp4"),
}
)
_require_success("kling_avatar", avatar_result)
avatar_path = Path(avatar_result.data["output_path"])
partial_report_path = artifacts_dir / "kling_avatar_live_partial.json"
_write_json(
partial_report_path,
{
"portrait": portrait_result.data,
"avatar": avatar_result.data,
"artifacts": {
"narration": str(narration_path),
"portrait": str(portrait_path),
"avatar": str(avatar_path),
},
"ffprobe": {
"portrait": _probe_media(portrait_path),
"avatar": _probe_media(avatar_path),
},
},
)
_announce_paid_call(
"kling_lip_sync",
"kling_official",
"kling-official-lip-sync full_lip_sync",
"Validate identify-face, explicit auto-selection, and advanced lip-sync as one smoke.",
"sample",
)
lip_sync_result = lip_sync.execute(
{
"operation": "full_lip_sync",
"video_url": _first_remote_url(avatar_result.data),
"audio_path": str(narration_path),
"auto_select_face": True,
"faces_artifact_path": str(artifacts_dir / "kling_lip_sync_faces.json"),
"timeout_seconds": max(timeout_seconds, 900),
"poll_interval": poll_interval,
"output_path": str(video_dir / "kling_lip_sync_smoke.mp4"),
}
)
_require_success("kling_lip_sync", lip_sync_result)
lip_sync_path = Path(lip_sync_result.data["output_path"])
estimated_cost = sum(
float(getattr(result, "cost_usd", 0) or 0)
for result in (portrait_result, avatar_result, lip_sync_result)
)
return {
"portrait": portrait_result.data,
"avatar": avatar_result.data,
"lip_sync": lip_sync_result.data,
"artifacts": {
"narration": str(narration_path),
"portrait": str(portrait_path),
"avatar": str(avatar_path),
"lip_sync": str(lip_sync_path),
"faces": str(artifacts_dir / "kling_lip_sync_faces.json"),
"avatar_partial_report": str(partial_report_path),
},
"ffprobe": {
"portrait": _probe_media(portrait_path),
"avatar": _probe_media(avatar_path),
"lip_sync": _probe_media(lip_sync_path),
},
"estimated_cost_usd": estimated_cost,
}
def _run_live_all(
project_dir: Path,
*,
voice_id: str,
voice_language: str,
voice_speed: float,
text: str,
timeout_seconds: int,
poll_interval: float,
include_account_usage: bool,
video_duration: str,
) -> dict[str, Any]:
core = _run_live_full(
project_dir,
voice_id=voice_id,
voice_language=voice_language,
voice_speed=voice_speed,
text=text,
timeout_seconds=timeout_seconds,
poll_interval=poll_interval,
include_account_usage=include_account_usage,
video_duration=video_duration,
)
avatar_suite = _run_live_avatar_suite(
project_dir,
timeout_seconds=timeout_seconds,
poll_interval=poll_interval,
)
return {
"core": core,
"avatar_suite": avatar_suite,
"artifacts": {
**core["artifacts"],
**avatar_suite["artifacts"],
},
"ffprobe": {
"core": core["ffprobe"],
"avatar_suite": avatar_suite["ffprobe"],
},
"estimated_cost_usd": float(core.get("estimated_cost_usd") or 0)
+ float(avatar_suite.get("estimated_cost_usd") or 0),
}
def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
live = parser.add_mutually_exclusive_group()
@@ -465,6 +660,16 @@ def _parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
action="store_true",
help="Run paid Kling TTS, image, video, and local compose.",
)
live.add_argument(
"--live-avatar",
action="store_true",
help="Use existing narration to run paid Kling portrait, avatar, and lip-sync samples.",
)
live.add_argument(
"--live-all",
action="store_true",
help="Run the full smoke plus paid Kling avatar and lip-sync samples.",
)
parser.add_argument("--voice-id", default=DEFAULT_VOICE_ID)
parser.add_argument("--voice-language", choices=["en", "zh"], default="en")
parser.add_argument("--voice-speed", type=float, default=1.0)
@@ -482,6 +687,10 @@ def _execution_mode(args: argparse.Namespace) -> str:
return "live_tts"
if getattr(args, "live_full", False):
return "live_full"
if getattr(args, "live_avatar", False):
return "live_avatar"
if getattr(args, "live_all", False):
return "live_all"
return "dry_run"
@@ -545,10 +754,30 @@ def main(argv: Sequence[str] | None = None) -> int:
include_account_usage=args.include_account_usage,
video_duration=args.video_duration,
)
elif mode == "live_avatar":
report["live_avatar_result"] = _run_live_avatar_suite(
project_dir,
timeout_seconds=args.timeout_seconds,
poll_interval=args.poll_interval,
)
elif mode == "live_all":
report["live_all_result"] = _run_live_all(
project_dir,
voice_id=args.voice_id,
voice_language=args.voice_language,
voice_speed=args.voice_speed,
text=args.text,
timeout_seconds=args.timeout_seconds,
poll_interval=args.poll_interval,
include_account_usage=args.include_account_usage,
video_duration=args.video_duration,
)
else:
report["next_steps"] = [
"Run with --live-tts to make one paid Kling TTS sample call.",
"Run with --live-full to make paid Kling TTS/image/video calls and compose final_kling_e2e_smoke.mp4.",
"Run with --live-avatar to reuse narration for paid Kling avatar and lip-sync provider smokes.",
"Run with --live-all to add paid Kling avatar and lip-sync provider smokes.",
]
except Exception as exc:
report["failed"] = {"error": str(exc)}
@@ -563,6 +792,12 @@ def main(argv: Sequence[str] | None = None) -> int:
print(f"narration: {report['live_tts_result']['artifacts']['narration']}")
elif mode == "live_full":
print(f"final: {report['live_full_result']['artifacts']['final']}")
elif mode == "live_avatar":
print(f"avatar: {report['live_avatar_result']['artifacts']['avatar']}")
print(f"lip_sync: {report['live_avatar_result']['artifacts']['lip_sync']}")
elif mode == "live_all":
print(f"avatar: {report['live_all_result']['artifacts']['avatar']}")
print(f"lip_sync: {report['live_all_result']['artifacts']['lip_sync']}")
return 0

15
tests/contracts/conftest.py Normal file
View File

@@ -0,0 +1,15 @@
"""Shared fixtures for contract tests."""
from __future__ import annotations
import pytest
from tools.tool_registry import ToolRegistry
@pytest.fixture()
def isolated_tool_registry(monkeypatch) -> ToolRegistry:
"""Provide a registry singleton replacement scoped to one test."""
test_registry = ToolRegistry()
monkeypatch.setattr("tools.tool_registry.registry", test_registry)
return test_registry

175
tests/contracts/test_kling_official_avatar.py
View File

@@ -7,6 +7,8 @@ import json
import sys
from pathlib import Path
import pytest
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
@@ -14,14 +16,12 @@ from tools.avatar.kling_avatar import KlingAvatar
from tools.avatar.kling_lip_sync import KlingLipSync
from tools.avatar.lip_sync import LipSync
from tools.avatar.talking_head import TalkingHead
from tools.tool_registry import registry
def test_registry_discovers_kling_avatar(monkeypatch):
def test_registry_discovers_kling_avatar(monkeypatch, isolated_tool_registry):
monkeypatch.delenv("KLING_API_KEY", raising=False)
registry.clear()
registry.discover("tools")
tool = registry.get("kling_avatar")
isolated_tool_registry.discover("tools")
tool = isolated_tool_registry.get("kling_avatar")
assert tool is not None
assert tool.capability == "avatar"
assert tool.provider == "kling_official"
@@ -123,11 +123,10 @@ def test_avatar_cost_estimate_is_not_zero():
assert tool.dry_run({"image_url": "x", "audio_id": "a"})["cost_estimate_confidence"] == "low"
def test_registry_discovers_kling_lip_sync(monkeypatch):
def test_registry_discovers_kling_lip_sync(monkeypatch, isolated_tool_registry):
monkeypatch.delenv("KLING_API_KEY", raising=False)
registry.clear()
registry.discover("tools")
tool = registry.get("kling_lip_sync")
isolated_tool_registry.discover("tools")
tool = isolated_tool_registry.get("kling_lip_sync")
assert tool is not None
assert tool.capability == "avatar"
assert tool.provider == "kling_official"
@@ -135,8 +134,12 @@ def test_registry_discovers_kling_lip_sync(monkeypatch):
def test_lip_sync_schema_and_local_tool_are_distinct():
tool = KlingLipSync()
props = tool.input_schema["properties"]
assert "kling-official" in tool.agent_skills
assert "avatar-video" in tool.agent_skills
assert "sound_start_time" in props
assert "sound_end_time" in props
assert "sound_insert_time" in props
assert tool.runtime.value == "api"
assert LipSync().provider == "wav2lip"
assert LipSync().runtime.value == "local_gpu"
@@ -165,17 +168,99 @@ def test_advanced_lip_sync_payload_uses_face_and_audio_path(tmp_path):
"session_id": "session-a",
"face_id": "face-a",
"audio_path": str(audio_path),
"sound_start_time": 0,
"sound_end_time": 2500,
"sound_insert_time": 500,
"callback_url": "https://example.com/kling/callback",
}
)
assert request["path"] == "/v1/videos/advanced-lip-sync"
assert request["payload"]["session_id"] == "session-a"
assert request["payload"]["face_choose"] == [{"face_id": "face-a"}]
assert request["payload"]["sound_file"] == base64.b64encode(b"audio").decode("ascii")
assert request["payload"]["face_choose"] == [
{
"face_id": "face-a",
"sound_file": base64.b64encode(b"audio").decode("ascii"),
"sound_start_time": 0,
"sound_end_time": 2500,
"sound_insert_time": 500,
}
]
assert "sound_file" not in request["payload"]
assert request["payload"]["callback_url"] == "https://example.com/kling/callback"
def test_advanced_lip_sync_accepts_official_nested_face_choose():
face_choose = {
"face_id": "face-a",
"audio_id": "audio-a",
"sound_start_time": 0,
"sound_end_time": 4000,
"sound_insert_time": 500,
}
request = KlingLipSync()._build_advanced_request(
{"session_id": "session-a", "face_choose": [face_choose]}
)
assert request["payload"]["face_choose"] == [face_choose]
assert request["audio_source"] == {"type": "audio_id", "value": "audio-a"}
def test_full_lip_sync_preserves_nested_face_timing_defaults():
inputs = {
"session_id": "session-a",
"face_choose": [
{
"face_id": "face-a",
"audio_id": "audio-a",
"sound_start_time": 250,
"sound_end_time": 4250,
"sound_insert_time": 750,
}
],
}
tool = KlingLipSync()
tool._apply_face_timing_defaults(
inputs, {"face_id": "face-a", "start_time": 0, "end_time": 0}
)
request = tool._build_advanced_request(inputs)
assert request["payload"]["face_choose"][0]["sound_start_time"] == 250
assert request["payload"]["face_choose"][0]["sound_end_time"] == 4250
assert request["payload"]["face_choose"][0]["sound_insert_time"] == 750
@pytest.mark.parametrize(
("top_level", "nested", "message"),
[
({"audio_id": "audio-top"}, {"audio_id": "audio-nested"}, "audio input"),
({"sound_end_time": 5000}, {"sound_end_time": 4000}, "sound_end_time"),
],
)
def test_advanced_lip_sync_rejects_conflicting_top_level_and_nested_values(
top_level, nested, message
):
inputs = {
"session_id": "session-a",
"face_choose": [
{
"face_id": "face-a",
"audio_id": "audio-a",
"sound_start_time": 0,
"sound_end_time": 4000,
"sound_insert_time": 0,
**nested,
}
],
**top_level,
}
with pytest.raises(ValueError, match=message):
KlingLipSync()._build_advanced_request(inputs)
def test_identify_face_execute_writes_faces_artifact(monkeypatch, tmp_path):
class FakeClient:
def post(self, path, payload):
@@ -184,7 +269,14 @@ def test_identify_face_execute_writes_faces_artifact(monkeypatch, tmp_path):
"code": 0,
"data": {
"session_id": "session-a",
"faces": [{"face_id": "face-a", "bbox": [0, 0, 100, 100]}],
"face_data": [
{
"face_id": "face-a",
"face_image": "https://example.com/face.png",
"start_time": 0,
"end_time": 5200,
}
],
},
}
@@ -214,9 +306,9 @@ def test_full_lip_sync_requires_confirmation_for_multiple_faces(monkeypatch, tmp
"code": 0,
"data": {
"session_id": "session-a",
"faces": [
{"face_id": "face-small", "bbox": [0, 0, 50, 50]},
{"face_id": "face-large", "bbox": [0, 0, 200, 200]},
"face_data": [
{"face_id": "face-small", "bbox": [0, 0, 50, 50], "start_time": 0, "end_time": 4000},
{"face_id": "face-large", "bbox": [0, 0, 200, 200], "start_time": 500, "end_time": 5500},
],
},
}
@@ -247,9 +339,9 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_
"code": 0,
"data": {
"session_id": "session-a",
"faces": [
{"face_id": "face-small", "bbox": [0, 0, 50, 50]},
{"face_id": "face-large", "bbox": [0, 0, 200, 200]},
"face_data": [
{"face_id": "face-small", "bbox": [0, 0, 50, 50], "start_time": 0, "end_time": 4000},
{"face_id": "face-large", "bbox": [0, 0, 200, 200], "start_time": 500, "end_time": 5500},
],
},
}
@@ -257,7 +349,15 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_
def create_classic_task(self, path, payload):
self.path = path
self.payload = payload
assert payload["face_choose"] == [{"face_id": "face-large"}]
assert payload["face_choose"] == [
{
"face_id": "face-large",
"audio_id": "audio-a",
"sound_start_time": 0,
"sound_end_time": 5000,
"sound_insert_time": 500,
}
]
return "lip-task-1"
def poll_classic(self, path, task_id, result_key, timeout_seconds, poll_interval):
@@ -285,7 +385,15 @@ def test_full_lip_sync_auto_selects_largest_face_and_downloads(monkeypatch, tmp_
assert result.success
assert result.data["task_id"] == "lip-task-1"
assert result.data["face_selection"]["selection_method"] == "auto_selected"
assert result.data["face_choose"] == [{"face_id": "face-large"}]
assert result.data["face_choose"] == [
{
"face_id": "face-large",
"audio_id": "audio-a",
"sound_start_time": 0,
"sound_end_time": 5000,
"sound_insert_time": 500,
}
]
assert Path(result.artifacts[0]).read_bytes() == b"video"
faces_artifact = next(Path(path) for path in result.artifacts if Path(path).name == "kling_lip_sync_faces.json")
assert json.loads(faces_artifact.read_text())["selection"]["selection_method"] == "auto_selected"
@@ -300,10 +408,21 @@ def test_auto_select_face_area_avoids_position_inflation():
def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path):
audio_path = tmp_path / "voice.mp3"
audio_path.write_bytes(b"audio")
class FakeClient:
def create_classic_task(self, path, payload):
assert path == "/v1/videos/advanced-lip-sync"
assert payload["face_choose"] == [{"face_id": "face-a"}]
assert payload["face_choose"] == [
{
"face_id": "face-a",
"sound_file": base64.b64encode(b"audio").decode("ascii"),
"sound_start_time": 0,
"sound_end_time": 4000,
"sound_insert_time": 0,
}
]
return "lip-task-1"
def poll_classic(self, path, task_id, result_key, timeout_seconds, poll_interval):
@@ -322,7 +441,10 @@ def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path):
"operation": "advanced_lip_sync",
"session_id": "session-a",
"face_id": "face-a",
"audio_id": "audio-a",
"audio_path": str(audio_path),
"sound_start_time": 0,
"sound_end_time": 4000,
"sound_insert_time": 0,
"output_path": str(tmp_path / "lip.mp4"),
}
)
@@ -331,6 +453,15 @@ def test_advanced_lip_sync_execute_downloads_video(monkeypatch, tmp_path):
assert result.data["provider"] == "kling_official"
assert result.data["task_id"] == "lip-task-1"
assert result.data["duration_seconds"] == 4.0
assert result.data["face_choose"] == [
{
"face_id": "face-a",
"sound_start_time": 0,
"sound_end_time": 4000,
"sound_insert_time": 0,
"sound_file_provided": True,
}
]
assert result.cost_usd > 0

26
tests/contracts/test_kling_official_core.py
View File

@@ -27,7 +27,6 @@ from tools.audio.kling_tts import KlingTTS
from tools.avatar.kling_avatar import KlingAvatar
from tools.avatar.kling_lip_sync import KlingLipSync
from tools.graphics.kling_official_image import KlingOfficialImage
from tools.tool_registry import registry
from tools.video.kling_official_video import KlingOfficialVideo
@@ -194,9 +193,9 @@ def test_turbo_create_and_poll_parse_result_paths(monkeypatch):
def test_schema_snapshot_contains_phase1_contract_fields():
fixture = PROJECT_ROOT / "tests/fixtures/kling_official/schema_snapshot.json"
data = json.loads(fixture.read_text())
assert data["build_id"] == "97344324"
assert "index-B9E4in0e.js" in data["chunk_names"]
assert "document-navigation-nxVgwiS5.js" in data["chunk_names"]
assert data["build_id"] == "97939672"
assert "index-0m3slU3p.js" in data["chunk_names"]
assert "document-navigation-Dk7H_V3n.js" in data["chunk_names"]
assert data["api_base"]["auth_env"] == "KLING_API_KEY"
assert data["task_statuses"]["classic"] == ["submitted", "processing", "succeed", "failed"]
assert data["task_statuses"]["turbo"] == ["submitted", "processing", "succeeded", "failed"]
@@ -211,6 +210,7 @@ def test_schema_snapshot_contains_phase1_contract_fields():
assert data["endpoints"]["video_effects"]["path"] == "/v1/videos/effects"
assert data["result_paths"]["classic_audio_results"] == "data.task_result.audios[]"
assert data["result_paths"]["identify_face_session"] == "data.session_id"
assert data["result_paths"]["identify_face_results"] == "data.face_data[]"
assert data["core_field_enums"]["tts_voice_language"] == ["zh", "en"]
assert data["core_field_enums"]["avatar_mode"] == ["std", "pro"]
@@ -251,7 +251,7 @@ def test_elements_helper_normalizes_and_records_metadata(tmp_path):
raise AssertionError("element_list items without element_id must be rejected")
def test_elements_helper_read_only_endpoints_do_not_enter_registry():
def test_elements_helper_read_only_endpoints_do_not_enter_registry(isolated_tool_registry):
fake = HelperFakeClient()
assert get_custom_element(123, client=fake)["data"]["element_id"] == 123
assert list_custom_elements(client=fake)["data"][0]["element_id"] == 456
@@ -266,10 +266,9 @@ def test_elements_helper_read_only_endpoints_do_not_enter_registry():
assert not hasattr(elements_module, "create_element")
assert not hasattr(elements_module, "delete_element")
registry.clear()
registry.discover("tools")
assert registry.get("kling_elements") is None
assert registry.get("kling_account_usage") is None
isolated_tool_registry.discover("tools")
assert isolated_tool_registry.get("kling_elements") is None
assert isolated_tool_registry.get("kling_account_usage") is None
def test_account_usage_helper_uses_endpoint_cache_and_throttle():
@@ -402,8 +401,7 @@ def test_provider_agent_skills_reference_kling_official():
assert "kling-official" in KlingLipSync().agent_skills
def test_phase3_does_not_register_audio_or_video_effect_tools():
registry.clear()
registry.discover("tools")
assert registry.get("kling_audio") is None
assert registry.get("kling_effects") is None
def test_phase3_does_not_register_audio_or_video_effect_tools(isolated_tool_registry):
isolated_tool_registry.discover("tools")
assert isolated_tool_registry.get("kling_audio") is None
assert isolated_tool_registry.get("kling_effects") is None

38
tests/contracts/test_kling_official_e2e_script.py
View File

@@ -43,6 +43,8 @@ def test_cli_modes_are_explicit_and_non_paid_by_default():
assert script._execution_mode(script._parse_args(["--live-tts"])) == "live_tts"
assert script._execution_mode(script._parse_args(["--live-full"])) == "live_full"
assert script._execution_mode(script._parse_args(["--live"])) == "live_full"
assert script._execution_mode(script._parse_args(["--live-avatar"])) == "live_avatar"
assert script._execution_mode(script._parse_args(["--live-all"])) == "live_all"
def test_video_duration_aligns_to_narration_within_kling_limits():
@@ -52,3 +54,39 @@ def test_video_duration_aligns_to_narration_within_kling_limits():
assert script._aligned_video_duration("10", 6.05) == "10"
assert script._aligned_video_duration("3", None) == "3"
assert script._aligned_video_duration("3", 30.0) == "15"
def test_live_all_combines_core_and_avatar_results(monkeypatch, tmp_path):
script = _load_script()
core = {
"artifacts": {"narration": "narration.mp3", "final": "final.mp4"},
"ffprobe": {"final": {"duration": 6.0}},
"estimated_cost_usd": 0.25,
}
avatar_suite = {
"artifacts": {"avatar": "avatar.mp4", "lip_sync": "lip.mp4"},
"ffprobe": {"avatar": {"duration": 7.0}, "lip_sync": {"duration": 7.0}},
"estimated_cost_usd": 0.75,
}
monkeypatch.setattr(script, "_run_live_full", lambda *args, **kwargs: core)
monkeypatch.setattr(
script, "_run_live_avatar_suite", lambda *args, **kwargs: avatar_suite
)
result = script._run_live_all(
tmp_path,
voice_id="voice-a",
voice_language="en",
voice_speed=1.0,
text="hello",
timeout_seconds=30,
poll_interval=1.0,
include_account_usage=False,
video_duration="3",
)
assert result["core"] is core
assert result["avatar_suite"] is avatar_suite
assert result["artifacts"]["final"] == "final.mp4"
assert result["artifacts"]["lip_sync"] == "lip.mp4"
assert result["estimated_cost_usd"] == 1.0

19
tests/contracts/test_kling_official_image.py
View File

@@ -9,16 +9,13 @@ from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
from tools.graphics.image_selector import ImageSelector
from tools.graphics.kling_official_image import KlingOfficialImage
from tools.tool_registry import registry
def test_registry_discovers_kling_official_image(monkeypatch):
def test_registry_discovers_kling_official_image(monkeypatch, isolated_tool_registry):
monkeypatch.delenv("KLING_API_KEY", raising=False)
registry.clear()
registry.discover("tools")
tool = registry.get("kling_official_image")
isolated_tool_registry.discover("tools")
tool = isolated_tool_registry.get("kling_official_image")
assert tool is not None
assert tool.capability == "image_generation"
assert tool.provider == "kling_official"
@@ -255,12 +252,9 @@ def test_execute_image_omni_series_records_references_callback_and_artifacts(mon
assert Path(result.artifacts[1]).name == "series_2.png"
def test_image_selector_prefers_official_provider(monkeypatch):
def test_image_selector_prefers_official_provider(monkeypatch, isolated_tool_registry):
monkeypatch.setenv("KLING_API_KEY", "test-key")
registry.clear()
registry.register(KlingOfficialImage())
registry.register(ImageSelector())
registry._discovered_packages.add("tools")
isolated_tool_registry.discover("tools")
def fake_execute(self, inputs):
from tools.base_tool import ToolResult
@@ -268,10 +262,11 @@ def test_image_selector_prefers_official_provider(monkeypatch):
return ToolResult(success=True, data={"output_path": "out.png"}, artifacts=["out.png"])
monkeypatch.setattr(KlingOfficialImage, "execute", fake_execute)
result = registry.get("image_selector").execute(
result = isolated_tool_registry.get("image_selector").execute(
{
"prompt": "official image",
"preferred_provider": "kling_official",
"allowed_providers": ["kling_official"],
"api_family": "omni",
"image_reference": "subject",
}

21
tests/contracts/test_kling_official_video.py
View File

@@ -10,17 +10,14 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
from tools._kling.account import reset_account_usage_cache
from tools.tool_registry import registry
from tools._kling.errors import KlingAPIError
from tools.video.kling_official_video import KlingOfficialVideo
from tools.video.video_selector import VideoSelector
def test_registry_discovers_kling_official_video(monkeypatch):
def test_registry_discovers_kling_official_video(monkeypatch, isolated_tool_registry):
monkeypatch.delenv("KLING_API_KEY", raising=False)
registry.clear()
registry.discover("tools")
tool = registry.get("kling_official_video")
isolated_tool_registry.discover("tools")
tool = isolated_tool_registry.get("kling_official_video")
assert tool is not None
assert tool.capability == "video_generation"
assert tool.provider == "kling_official"
@@ -355,14 +352,13 @@ def test_execute_downloads_all_omni_video_outputs_and_records_metadata(monkeypat
assert Path(result.artifacts[1]).name == "out_2.mp4"
def test_video_selector_prefers_official_provider_without_fal_upload(monkeypatch, tmp_path):
def test_video_selector_prefers_official_provider_without_fal_upload(
monkeypatch, tmp_path, isolated_tool_registry
):
monkeypatch.setenv("KLING_API_KEY", "test-key")
image_path = tmp_path / "ref.png"
image_path.write_bytes(b"fake")
registry.clear()
registry.register(KlingOfficialVideo())
registry.register(VideoSelector())
registry._discovered_packages.add("tools")
isolated_tool_registry.discover("tools")
seen = {}
@@ -377,11 +373,12 @@ def test_video_selector_prefers_official_provider_without_fal_upload(monkeypatch
monkeypatch.setattr(KlingOfficialVideo, "execute", fake_execute)
monkeypatch.setattr("tools.video._shared.upload_image_fal", fail_upload)
result = registry.get("video_selector").execute(
result = isolated_tool_registry.get("video_selector").execute(
{
"prompt": "animate",
"operation": "image_to_video",
"preferred_provider": "kling_official",
"allowed_providers": ["kling_official"],
"reference_image_path": str(image_path),
}
)

19
tests/contracts/test_kling_tts.py
View File

@@ -9,15 +9,12 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
from tools.audio.kling_tts import KlingTTS
from tools.audio.tts_selector import TTSSelector
from tools.tool_registry import registry
def test_registry_discovers_kling_tts(monkeypatch):
def test_registry_discovers_kling_tts(monkeypatch, isolated_tool_registry):
monkeypatch.delenv("KLING_API_KEY", raising=False)
registry.clear()
registry.discover("tools")
tool = registry.get("kling_tts")
isolated_tool_registry.discover("tools")
tool = isolated_tool_registry.get("kling_tts")
assert tool is not None
assert tool.capability == "tts"
assert tool.provider == "kling_official"
@@ -153,12 +150,9 @@ def test_execute_accepts_synchronous_create_response(monkeypatch, tmp_path):
assert result.data["audio_duration_seconds"] == 2.5
def test_tts_selector_prefers_kling_official(monkeypatch):
def test_tts_selector_prefers_kling_official(monkeypatch, isolated_tool_registry):
monkeypatch.setenv("KLING_API_KEY", "test-key")
registry.clear()
registry.register(KlingTTS())
registry.register(TTSSelector())
registry._discovered_packages.add("tools")
isolated_tool_registry.discover("tools")
def fake_execute(self, inputs):
from tools.base_tool import ToolResult
@@ -166,11 +160,12 @@ def test_tts_selector_prefers_kling_official(monkeypatch):
return ToolResult(success=True, data={"output_path": "out.mp3"}, artifacts=["out.mp3"])
monkeypatch.setattr(KlingTTS, "execute", fake_execute)
result = registry.get("tts_selector").execute(
result = isolated_tool_registry.get("tts_selector").execute(
{
"text": "official speech",
"voice_id": "voice-a",
"preferred_provider": "kling_official",
"allowed_providers": ["kling_official"],
}
)
assert result.success

12
tests/fixtures/kling_official/schema_snapshot.json vendored
View File

@@ -1,5 +1,5 @@
{
"build_id": "97344324",
"build_id": "97939672",
"source_urls": [
"https://kling.ai/document-api/api/get-started/authentication",
"https://kling.ai/document-api/api/get-started/error-codes",
@@ -18,10 +18,10 @@
"https://kling.ai/document-api/api/video/effects"
],
"chunk_names": [
"index-B9E4in0e.js",
"document-navigation-nxVgwiS5.js"
"index-0m3slU3p.js",
"document-navigation-Dk7H_V3n.js"
],
"extracted_at": "2026-07-03T08:12:55Z",
"extracted_at": "2026-07-10T08:34:20Z",
"api_base": {
"default": "https://api-singapore.klingai.com",
"env_override": "KLING_API_BASE_URL",
@@ -145,6 +145,7 @@
"classic_image_results": "data.task_result.images[]",
"classic_audio_results": "data.task_result.audios[]",
"identify_face_session": "data.session_id",
"identify_face_results": "data.face_data[]",
"turbo_results": "data[0].outputs[]"
},
"core_field_enums": {
@@ -220,7 +221,8 @@
]
},
"notes": [
"Official docs were fetched as a SPA on 2026-07-03. Current HTML exposes buildId 97344324.",
"Official docs were fetched as a SPA on 2026-07-10. Current HTML exposes buildId 97939672.",
"Advanced lip sync currently accepts one face_choose item. The item contains face_id, audio_id or sound_file, and sound_start_time, sound_end_time, and sound_insert_time.",
"The current entry/navigation bundles no longer expose the older api-*.js chunk names as literal references, so this fixture records the current entry assets and the Phase 1 core schema facts used by implementation and tests."
]
}

196
tools/avatar/kling_lip_sync.py
View File

@@ -99,6 +99,23 @@ class KlingLipSync(BaseTool):
"type": "string",
"description": "Alias for sound_file_path for compatibility with local lip_sync.",
},
"sound_start_time": {
"type": "integer",
"minimum": 0,
"description": "Audio crop start in milliseconds.",
},
"sound_end_time": {
"type": "integer",
"minimum": 0,
"description": "Audio crop end in milliseconds; inferred for full_lip_sync when possible.",
},
"sound_insert_time": {
"type": "integer",
"minimum": 0,
"description": "Video timeline insertion point in milliseconds.",
},
"sound_volume": {"type": "number", "minimum": 0, "maximum": 2},
"original_audio_volume": {"type": "number", "minimum": 0, "maximum": 2},
"faces_artifact_path": {"type": "string"},
"callback_url": {"type": "string"},
"external_task_id": {"type": "string"},
@@ -121,7 +138,17 @@ class KlingLipSync(BaseTool):
backoff_seconds=2.0,
retryable_errors=["1302", "1303", "5000", "5001", "5002"],
)
idempotency_key_fields = ["video_id", "video_url", "session_id", "face_id", "audio_id", "sound_file_path"]
idempotency_key_fields = [
"video_id",
"video_url",
"session_id",
"face_id",
"audio_id",
"sound_file_path",
"sound_start_time",
"sound_end_time",
"sound_insert_time",
]
side_effects = [
"paid remote generation via official Kling API",
"writes face selection artifact",
@@ -192,6 +219,8 @@ class KlingLipSync(BaseTool):
model="kling-official-lip-sync",
)
merged = {**inputs, "session_id": identify["session_id"], "face_choose": face_choose}
selected_face = self._selected_face_record(identify["faces"], face_choose)
self._apply_face_timing_defaults(merged, selected_face)
request = self._build_advanced_request(merged)
result = self._run_advanced_lip_sync(client, merged, request, start)
result.data["faces_artifact_path"] = str(artifact_path)
@@ -225,7 +254,8 @@ class KlingLipSync(BaseTool):
if not session_id:
raise ValueError(f"Kling identify-face response missing data.session_id: {data}")
faces = (
payload.get("faces")
payload.get("face_data")
or payload.get("faces")
or payload.get("face_list")
or payload.get("face_infos")
or payload.get("faces_info")
@@ -286,7 +316,9 @@ class KlingLipSync(BaseTool):
"task_id": task_id,
"operation": request["operation"],
"session_id": request["payload"]["session_id"],
"face_choose": request["payload"]["face_choose"],
"face_choose": self._face_choose_result_metadata(
request["payload"]["face_choose"]
),
"audio_source": request["audio_source"],
"remote_outputs": outputs,
"output": str(paths[0]),
@@ -328,11 +360,15 @@ class KlingLipSync(BaseTool):
face_choose = self._normalize_face_choose(inputs)
if not face_choose:
raise ValueError("advanced_lip_sync requires face_choose or face_id")
if len(face_choose) != 1:
raise ValueError("advanced_lip_sync currently supports exactly one face_choose item")
face_item = face_choose[0]
audio_source = self._copy_audio_input(inputs, face_item)
self._copy_timing_fields(inputs, face_item)
payload: dict[str, Any] = {
"session_id": session_id,
"face_choose": face_choose,
}
audio_source = self._copy_audio_input(inputs, payload)
self._copy_common_task_fields(inputs, payload)
return {
"protocol": "classic",
@@ -438,12 +474,156 @@ class KlingLipSync(BaseTool):
return max(float(width), 0.0) * max(float(height), 0.0)
return 0.0
@staticmethod
def _selected_face_record(
faces: list[dict[str, Any]], face_choose: list[dict[str, Any]]
) -> dict[str, Any]:
selected_id = str(face_choose[0].get("face_id") or "")
for face in faces:
if str(face.get("face_id") or face.get("id") or "") == selected_id:
return face
raise ValueError(f"Selected face_id {selected_id!r} was not returned by identify_face")
def _apply_face_timing_defaults(
self, inputs: dict[str, Any], face: dict[str, Any]
) -> None:
face_start = int(face.get("start_time") or 0)
face_end = int(face.get("end_time") or 0)
face_choose = self._normalize_face_choose(inputs)
face_item = face_choose[0] if face_choose else {}
if inputs.get("sound_start_time") is None and face_item.get("sound_start_time") is None:
inputs["sound_start_time"] = 0
if inputs.get("sound_insert_time") is None and face_item.get("sound_insert_time") is None:
inputs["sound_insert_time"] = face_start
if inputs.get("sound_end_time") is not None or face_item.get("sound_end_time") is not None:
return
candidates: list[int] = []
audio_duration = self._local_audio_duration_ms(inputs)
if audio_duration:
candidates.append(audio_duration)
if face_end > face_start:
candidates.append(face_end - face_start)
if not candidates:
raise ValueError(
"full_lip_sync could not infer sound_end_time; provide it explicitly"
)
inputs["sound_end_time"] = min(candidates)
@staticmethod
def _local_audio_duration_ms(inputs: dict[str, Any]) -> int | None:
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
if not sound_path:
return None
path = Path(sound_path)
if not path.is_file():
return None
seconds = probe_output(path).get("duration_seconds")
if not seconds:
return None
return int(round(float(seconds) * 1000))
@staticmethod
def _copy_timing_fields(inputs: dict[str, Any], face_item: dict[str, Any]) -> None:
for key, default in (("sound_start_time", 0), ("sound_insert_time", 0)):
nested = face_item.get(key)
top_level = inputs.get(key)
if nested is not None and top_level is not None and int(nested) != int(top_level):
raise ValueError(
f"Conflicting {key} values between top-level input and face_choose[0]"
)
value = nested if nested is not None else top_level
if value is None:
value = default
face_item[key] = int(value)
nested_end = face_item.get("sound_end_time")
top_level_end = inputs.get("sound_end_time")
if (
nested_end is not None
and top_level_end is not None
and int(nested_end) != int(top_level_end)
):
raise ValueError(
"Conflicting sound_end_time values between top-level input and face_choose[0]"
)
sound_end = nested_end if nested_end is not None else top_level_end
if sound_end is None:
raise ValueError("advanced_lip_sync requires sound_end_time")
face_item["sound_end_time"] = int(sound_end)
if face_item["sound_end_time"] - face_item["sound_start_time"] < 2000:
raise ValueError("advanced_lip_sync requires at least 2000ms of cropped audio")
for key in ("sound_volume", "original_audio_volume"):
nested = face_item.get(key)
top_level = inputs.get(key)
if nested is not None and top_level is not None and float(nested) != float(top_level):
raise ValueError(
f"Conflicting {key} values between top-level input and face_choose[0]"
)
value = nested if nested is not None else top_level
if value is not None:
face_item[key] = float(value)
@staticmethod
def _face_choose_result_metadata(
face_choose: list[dict[str, Any]],
) -> list[dict[str, Any]]:
metadata: list[dict[str, Any]] = []
for item in face_choose:
record = {key: value for key, value in item.items() if key != "sound_file"}
if item.get("sound_file"):
record["sound_file_provided"] = True
metadata.append(record)
return metadata
@staticmethod
def _copy_audio_input(inputs: dict[str, Any], payload: dict[str, Any]) -> dict[str, Any]:
audio_id = str(inputs.get("audio_id") or "").strip()
if audio_id:
payload["audio_id"] = audio_id
return {"type": "audio_id", "value": audio_id}
nested_audio_id = str(payload.get("audio_id") or "").strip()
nested_sound_file = payload.get("sound_file")
if nested_audio_id and nested_sound_file:
raise ValueError(
"Conflicting audio input in face_choose[0]; provide audio_id or sound_file, not both"
)
top_level_audio_id = str(inputs.get("audio_id") or "").strip()
top_level_sound_requested = any(
inputs.get(key)
for key in ("sound_file", "sound_file_url", "sound_file_path", "audio_path")
)
if nested_audio_id:
if (
(top_level_audio_id and top_level_audio_id != nested_audio_id)
or top_level_sound_requested
):
raise ValueError(
"Conflicting audio input between top-level fields and face_choose[0]"
)
payload["audio_id"] = nested_audio_id
return {"type": "audio_id", "value": nested_audio_id}
if nested_sound_file:
if top_level_audio_id:
raise ValueError(
"Conflicting audio input between top-level fields and face_choose[0]"
)
if top_level_sound_requested:
top_level_sound_file = normalize_media_input(
url=inputs.get("sound_file_url"),
path=inputs.get("sound_file_path") or inputs.get("audio_path"),
value=inputs.get("sound_file"),
label="Lip-sync audio file",
)
if top_level_sound_file != nested_sound_file:
raise ValueError(
"Conflicting audio input between top-level fields and face_choose[0]"
)
return {"type": "sound_file", "source": "face_choose[0]"}
if top_level_audio_id:
payload["audio_id"] = top_level_audio_id
return {"type": "audio_id", "value": top_level_audio_id}
sound_path = inputs.get("sound_file_path") or inputs.get("audio_path")
sound_file = normalize_media_input(