diff --git a/.agents/skills/elevenlabs/SKILL.md b/.agents/skills/elevenlabs/SKILL.md index cc9ddcef..d81dc390 100644 --- a/.agents/skills/elevenlabs/SKILL.md +++ b/.agents/skills/elevenlabs/SKILL.md @@ -5,7 +5,22 @@ description: Generate AI voiceovers, sound effects, and music using ElevenLabs A # ElevenLabs Audio Generation -Requires `ELEVENLABS_API_KEY` in `.env`. +## OpenMontage provider routing + +Inspect the OpenMontage registry before choosing an authentication path. + +- Prefer `fal_elevenlabs_tts` when it is available. It provides Eleven v3, + Multilingual v2, and Turbo v2.5 through the centrally managed fal.ai + connection; no separate ElevenLabs credential is needed. +- Use `elevenlabs_tts` only when that direct provider is already reported as + available by the registry. +- In a shared installation, never tell the user to create a `.env`, export a + key, or paste a credential. Report missing direct-provider access as an + administrator setup request. + +The direct API examples below require a centrally configured +`ELEVENLABS_API_KEY`; they are not the default path when the fal.ai provider is +available. ## Text-to-Speech diff --git a/AGENT_GUIDE.md b/AGENT_GUIDE.md index fe464643..e470f371 100644 --- a/AGENT_GUIDE.md +++ b/AGENT_GUIDE.md @@ -707,6 +707,8 @@ The `.agents/skills/` directory is large. When you're not coming in through a to - **Do not skip stage director skills.** Before executing any pipeline stage, read its director skill. The skill contains the quality bar, the workflow, and the review criteria. - Do not use deleted legacy names such as `tts_cloud`, `tts_engine`, or `video_gen`. - Do not hardcode provider names, API key names, or setup URLs. Read them from the registry's `install_instructions` and `dependencies` fields. +- Do not tell a restricted shared-installation user to add credentials manually, create a `.env`, or export a key. Surface the unavailable provider as an administrator setup request and continue only with centrally provisioned capabilities. +- Before offering a direct vendor credential, check the registry for an available wrapper through an already configured provider (for example, a partner model hosted by fal.ai). - Do not begin asset generation before user approval on the production plan. - Do not hide degraded paths. Record substitutions and blocked options explicitly. - Do not present a single unavailable tool in isolation. Always show the full capability picture: "X of Y providers configured for this capability." diff --git a/pipeline_defs/animation.yaml b/pipeline_defs/animation.yaml index 1002eabf..3f99608d 100644 --- a/pipeline_defs/animation.yaml +++ b/pipeline_defs/animation.yaml @@ -184,6 +184,7 @@ stages: - diagram_gen - code_snippet - music_gen + - fal_elevenlabs_music tools_available: - tts_selector - image_selector @@ -192,6 +193,7 @@ stages: - diagram_gen - code_snippet - music_gen + - fal_elevenlabs_music checkpoint_required: true human_approval_default: true review_focus: diff --git a/skills/pipelines/animation/asset-director.md b/skills/pipelines/animation/asset-director.md index 59ab5c18..d656d34e 100644 --- a/skills/pipelines/animation/asset-director.md +++ b/skills/pipelines/animation/asset-director.md @@ -31,7 +31,7 @@ Quick routing for common animation-pipeline needs: |-------|----------|---------| | Schema | `schemas/artifacts/asset_manifest.schema.json` | Artifact validation | | Prior artifacts | `state.artifacts["scene_plan"]["scene_plan"]`, `state.artifacts["script"]["script"]`, `state.artifacts["proposal"]["proposal_packet"]` | Tool path and beat map | -| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen` — selectors auto-discover all available providers from the registry | Asset production options | +| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen`, `fal_elevenlabs_music` — selectors auto-discover all available providers from the registry | Asset production options | | Playbook | Active style playbook | Visual consistency | ## Process diff --git a/tests/contracts/test_phase3_contracts.py b/tests/contracts/test_phase3_contracts.py index d582becd..5fd94058 100644 --- a/tests/contracts/test_phase3_contracts.py +++ b/tests/contracts/test_phase3_contracts.py @@ -694,6 +694,7 @@ class TestCapabilityMetadata: "dashscope", "doubao", "elevenlabs", + "fal.ai", "google_tts", "kling_official", "openai", diff --git a/tests/tools/test_fal_elevenlabs_music.py b/tests/tools/test_fal_elevenlabs_music.py new file mode 100644 index 00000000..103e87a4 --- /dev/null +++ b/tests/tools/test_fal_elevenlabs_music.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from tools.audio.fal_elevenlabs_music import FalElevenLabsMusic +from tools.base_tool import ToolStatus +from tools.tool_registry import ToolRegistry + + +def _response(*, json_data=None, content=b""): + response = MagicMock() + response.json.return_value = json_data + response.content = content + response.raise_for_status.return_value = None + return response + + +def test_contract_and_rounded_cost(monkeypatch): + tool = FalElevenLabsMusic() + + monkeypatch.setenv("FAL_KEY", "test-key") + assert tool.get_status() == ToolStatus.AVAILABLE + assert tool.estimate_cost({"duration_seconds": 20}) == 0.80 + assert tool.estimate_cost({"duration_seconds": 61}) == 1.60 + assert tool.get_info()["capability"] == "music_generation" + assert tool.get_info()["provider"] == "fal.ai" + + +def test_registry_discovers_provider(monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + registry = ToolRegistry() + registry.discover() + + tool = registry.get("fal_elevenlabs_music") + assert tool is not None + assert tool.get_status() == ToolStatus.AVAILABLE + + +def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + output_path = tmp_path / "music.mp3" + tool = FalElevenLabsMusic() + tool._POLL_INTERVAL_SECONDS = 0 + + post_response = _response( + json_data={ + "status_url": "https://queue.example/status", + "response_url": "https://queue.example/result", + } + ) + status_response = _response(json_data={"status": "COMPLETED"}) + result_response = _response( + json_data={"audio": {"url": "https://media.example/music.mp3"}} + ) + audio_response = _response(content=b"fake-mp3") + + with ( + patch("requests.post", return_value=post_response) as mock_post, + patch( + "requests.get", + side_effect=[status_response, result_response, audio_response], + ) as mock_get, + ): + result = tool.execute( + { + "prompt": "gentle felt piano", + "duration_seconds": 20, + "force_instrumental": True, + "output_path": str(output_path), + } + ) + + assert result.success is True + assert result.cost_usd == 0.80 + assert result.model == "fal-ai/elevenlabs/music" + assert output_path.read_bytes() == b"fake-mp3" + assert mock_post.call_count == 1 + assert mock_post.call_args.kwargs["json"]["music_length_ms"] == 20000 + assert mock_post.call_args.kwargs["json"]["force_instrumental"] is True + assert mock_get.call_count == 3 + + +def test_execute_rejects_missing_duration_without_request(monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + with patch("requests.post") as mock_post: + result = FalElevenLabsMusic().execute({"prompt": "gentle piano"}) + + assert result.success is False + assert result.error == "duration_seconds is required" + mock_post.assert_not_called() + + +def test_error_redacts_fal_key(monkeypatch): + secret = "test-production-shaped-fal-key" + monkeypatch.setenv("FAL_KEY", secret) + with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")): + result = FalElevenLabsMusic().execute( + {"prompt": "gentle piano", "duration_seconds": 20} + ) + + assert result.success is False + assert secret not in result.error + assert "[REDACTED]" in result.error diff --git a/tests/tools/test_fal_elevenlabs_tts.py b/tests/tools/test_fal_elevenlabs_tts.py new file mode 100644 index 00000000..81b2987c --- /dev/null +++ b/tests/tools/test_fal_elevenlabs_tts.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +from tools.audio.fal_elevenlabs_tts import FalElevenLabsTTS +from tools.base_tool import ToolStatus +from tools.tool_registry import ToolRegistry + + +def _response(*, json_data=None, content=b""): + response = MagicMock() + response.json.return_value = json_data + response.content = content + response.raise_for_status.return_value = None + return response + + +def test_contract_models_and_cost(monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + tool = FalElevenLabsTTS() + + assert tool.get_status() == ToolStatus.AVAILABLE + assert tool.get_info()["capability"] == "tts" + assert tool.get_info()["provider"] == "fal.ai" + assert tool.estimate_cost({"text": "a" * 1000, "model_id": "eleven-v3"}) == 0.1 + assert tool.estimate_cost({"text": "a" * 1000, "model_id": "multilingual-v2"}) == 0.1 + assert tool.estimate_cost({"text": "a" * 1000, "model_id": "turbo-v2.5"}) == 0.05 + + +def test_registry_discovers_fal_tts(monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + registry = ToolRegistry() + registry.discover() + + tool = registry.get("fal_elevenlabs_tts") + assert tool is not None + assert tool.get_status() == ToolStatus.AVAILABLE + + +def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + output_path = tmp_path / "speech.mp3" + tool = FalElevenLabsTTS() + tool._POLL_INTERVAL_SECONDS = 0 + + post_response = _response( + json_data={ + "status_url": "https://queue.example/status", + "response_url": "https://queue.example/result", + } + ) + status_response = _response(json_data={"status": "COMPLETED"}) + result_response = _response( + json_data={"audio": {"url": "https://media.example/speech.mp3"}} + ) + audio_response = _response(content=b"fake-mp3") + + with ( + patch("requests.post", return_value=post_response) as mock_post, + patch( + "requests.get", + side_effect=[status_response, result_response, audio_response], + ) as mock_get, + ): + result = tool.execute( + { + "text": "A calm, measured test.", + "voice_id": "Rachel", + "model_id": "eleven-v3", + "stability": 0.65, + "language_code": "en", + "output_path": str(output_path), + } + ) + + assert result.success is True + assert result.model == "fal-ai/elevenlabs/tts/eleven-v3" + assert output_path.read_bytes() == b"fake-mp3" + assert mock_post.call_count == 1 + assert mock_post.call_args.args[0].endswith("/fal-ai/elevenlabs/tts/eleven-v3") + assert mock_post.call_args.kwargs["json"]["voice"] == "Rachel" + assert mock_post.call_args.kwargs["json"]["stability"] == 0.65 + assert mock_get.call_count == 3 + + +def test_invalid_model_does_not_submit(monkeypatch): + monkeypatch.setenv("FAL_KEY", "test-key") + with patch("requests.post") as mock_post: + result = FalElevenLabsTTS().execute( + {"text": "hello", "model_id": "not-a-model"} + ) + + assert result.success is False + assert "model_id must be one of" in result.error + mock_post.assert_not_called() + + +def test_error_redacts_fal_key(monkeypatch): + secret = "test-production-shaped-fal-key" + monkeypatch.setenv("FAL_KEY", secret) + with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")): + result = FalElevenLabsTTS().execute({"text": "hello"}) + + assert result.success is False + assert secret not in result.error + assert "[REDACTED]" in result.error diff --git a/tests/tools/test_google_tts_scoped_key.py b/tests/tools/test_google_tts_scoped_key.py index fbc15af2..36ea45ac 100644 --- a/tests/tools/test_google_tts_scoped_key.py +++ b/tests/tools/test_google_tts_scoped_key.py @@ -34,3 +34,26 @@ def test_tts_key_uses_header_and_is_redacted_from_errors(monkeypatch, tmp_path): assert result.success is False assert secret not in result.error assert "[REDACTED]" in result.error + + +def test_production_adapter_can_force_google_tts_to_ipv4(monkeypatch, tmp_path): + import socket + + import requests + import urllib3.util.connection + + monkeypatch.setenv("GOOGLE_TTS_API_KEY", "test-key") + monkeypatch.setenv("GOOGLE_TTS_FORCE_IPV4", "1") + original = urllib3.util.connection.allowed_gai_family + + def inspect_request(url, **kwargs): + assert urllib3.util.connection.allowed_gai_family() == socket.AF_INET + raise requests.HTTPError("safe expected failure") + + monkeypatch.setattr(requests, "post", inspect_request) + result = GoogleTTS().execute( + {"text": "safe test sentence", "output_path": str(tmp_path / "speech.mp3")} + ) + + assert result.success is False + assert urllib3.util.connection.allowed_gai_family is original diff --git a/tools/audio/elevenlabs_tts.py b/tools/audio/elevenlabs_tts.py index 85c34e63..9fcc5740 100644 --- a/tools/audio/elevenlabs_tts.py +++ b/tools/audio/elevenlabs_tts.py @@ -36,7 +36,9 @@ class ElevenLabsTTS(BaseTool): install_instructions = ( "Set the ELEVENLABS_API_KEY environment variable:\n" " export ELEVENLABS_API_KEY=your_key_here\n" - "Get a key at https://elevenlabs.io" + "Get a key at https://elevenlabs.io\n" + "If fal_elevenlabs_tts is available, use it instead to access ElevenLabs " + "speech through fal.ai without a separate ElevenLabs key." ) fallback = "openai_tts" fallback_tools = ["openai_tts", "piper_tts"] diff --git a/tools/audio/fal_elevenlabs_music.py b/tools/audio/fal_elevenlabs_music.py new file mode 100644 index 00000000..521f72cd --- /dev/null +++ b/tools/audio/fal_elevenlabs_music.py @@ -0,0 +1,231 @@ +"""Generate music with ElevenLabs Music through fal.ai. + +This provider uses OpenMontage's shared ``FAL_KEY`` credential and the fal.ai +queue API, then downloads the generated MP3 to a project-local path. +""" + +from __future__ import annotations + +import math +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class FalElevenLabsMusic(BaseTool): + """Generate a precisely timed music track through fal.ai.""" + + name = "fal_elevenlabs_music" + version = "0.1.0" + tier = ToolTier.GENERATE + capability = "music_generation" + provider = "fal.ai" + stability = ToolStability.BETA + execution_mode = ExecutionMode.ASYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = ["env:FAL_KEY"] + install_instructions = ( + "Set FAL_KEY to a fal.ai API key. " + "Get one at https://fal.ai/dashboard/keys" + ) + fallback_tools = ["music_gen", "google_music"] + agent_skills = ["music", "elevenlabs"] + + capabilities = [ + "generate_background_music", + "generate_instrumental", + ] + supports = { + "instrumental": True, + "style_control": True, + "exact_duration": True, + } + best_for = [ + "precisely timed instrumental background music", + "high-quality ElevenLabs Music through a shared fal.ai account", + "short-form social video soundtracks", + ] + not_good_for = [ + "offline generation", + "free music sourcing", + "sub-3-second sound effects", + ] + + input_schema = { + "type": "object", + "required": ["prompt", "duration_seconds"], + "properties": { + "prompt": { + "type": "string", + "description": "Music description including mood, style, and instruments", + }, + "duration_seconds": { + "type": "number", + "minimum": 3, + "maximum": 600, + "description": "Exact target length in seconds", + }, + "force_instrumental": { + "type": "boolean", + "default": True, + "description": "Generate music without vocals", + }, + "output_path": { + "type": "string", + "default": "fal_music_output.mp3", + }, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, + ram_mb=256, + vram_mb=0, + disk_mb=50, + network_required=True, + ) + retry_policy = RetryPolicy( + max_retries=0, + retryable_errors=["rate_limit", "timeout"], + ) + idempotency_key_fields = ["prompt", "duration_seconds", "force_instrumental"] + side_effects = [ + "writes an MP3 file to output_path", + "submits one paid fal.ai generation request", + ] + user_visible_verification = [ + "Listen to the generated music for mood, mix, and duration", + ] + + _MODEL = "fal-ai/elevenlabs/music" + _QUEUE_URL = f"https://queue.fal.run/{_MODEL}" + _POLL_INTERVAL_SECONDS = 5 + _MAX_WAIT_SECONDS = 600 + + def _get_api_key(self) -> str | None: + return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY") + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + """fal bills $0.80 per output minute, rounded up to a full minute.""" + duration = inputs.get("duration_seconds") + if duration is None: + raise ValueError("duration_seconds is required for cost estimation") + return round(math.ceil(float(duration) / 60.0) * 0.80, 2) + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + api_key = self._get_api_key() + if not api_key: + return ToolResult( + success=False, + error="No fal.ai API key found. " + self.install_instructions, + ) + + duration = inputs.get("duration_seconds") + if duration is None: + return ToolResult(success=False, error="duration_seconds is required") + duration = float(duration) + if not 3 <= duration <= 600: + return ToolResult( + success=False, + error="duration_seconds must be between 3 and 600", + ) + + import requests + + started = time.time() + headers = { + "Authorization": f"Key {api_key}", + "Content-Type": "application/json", + } + payload = { + "prompt": inputs["prompt"], + "music_length_ms": round(duration * 1000), + "force_instrumental": bool(inputs.get("force_instrumental", True)), + } + + try: + submit_response = requests.post( + self._QUEUE_URL, + headers=headers, + json=payload, + timeout=30, + ) + submit_response.raise_for_status() + queue_data = submit_response.json() + status_url = queue_data["status_url"] + response_url = queue_data["response_url"] + + deadline = time.monotonic() + self._MAX_WAIT_SECONDS + while True: + if time.monotonic() >= deadline: + return ToolResult( + success=False, + error="fal.ai music generation timed out while waiting in the queue", + duration_seconds=round(time.time() - started, 2), + ) + time.sleep(self._POLL_INTERVAL_SECONDS) + status_response = requests.get(status_url, headers=headers, timeout=20) + status_response.raise_for_status() + status = status_response.json().get("status", "UNKNOWN") + if status == "COMPLETED": + break + if status in {"FAILED", "CANCELLED"}: + return ToolResult( + success=False, + error=f"fal.ai music generation {status.lower()}", + duration_seconds=round(time.time() - started, 2), + ) + + result_response = requests.get(response_url, headers=headers, timeout=30) + result_response.raise_for_status() + result_data = result_response.json() + audio_url = result_data["audio"]["url"] + + audio_response = requests.get(audio_url, timeout=120) + audio_response.raise_for_status() + output_path = Path(inputs.get("output_path", "fal_music_output.mp3")) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(audio_response.content) + except Exception as exc: + safe_error = str(exc).replace(api_key, "[REDACTED]") + return ToolResult( + success=False, + error=f"fal.ai ElevenLabs Music generation failed: {safe_error}", + duration_seconds=round(time.time() - started, 2), + ) + + return ToolResult( + success=True, + data={ + "provider": "fal.ai", + "model": self._MODEL, + "prompt": inputs["prompt"], + "duration_seconds": duration, + "force_instrumental": payload["force_instrumental"], + "output": str(output_path), + "format": "mp3", + }, + artifacts=[str(output_path)], + cost_usd=self.estimate_cost(inputs), + duration_seconds=round(time.time() - started, 2), + model=self._MODEL, + ) diff --git a/tools/audio/fal_elevenlabs_tts.py b/tools/audio/fal_elevenlabs_tts.py new file mode 100644 index 00000000..3ce4fe63 --- /dev/null +++ b/tools/audio/fal_elevenlabs_tts.py @@ -0,0 +1,359 @@ +"""Generate ElevenLabs speech through fal.ai using the shared FAL credential.""" + +from __future__ import annotations + +import os +import time +from pathlib import Path +from typing import Any + +from tools.base_tool import ( + BaseTool, + Determinism, + ExecutionMode, + ResourceProfile, + RetryPolicy, + ToolResult, + ToolRuntime, + ToolStability, + ToolStatus, + ToolTier, +) + + +class FalElevenLabsTTS(BaseTool): + """Generate expressive narration with ElevenLabs models hosted by fal.ai.""" + + name = "fal_elevenlabs_tts" + version = "0.1.0" + tier = ToolTier.VOICE + capability = "tts" + provider = "fal.ai" + stability = ToolStability.BETA + execution_mode = ExecutionMode.ASYNC + determinism = Determinism.STOCHASTIC + runtime = ToolRuntime.API + + dependencies = ["env:FAL_KEY"] + install_instructions = ( + "Set FAL_KEY to a fal.ai API key. No separate ElevenLabs key is needed. " + "Get a fal.ai key at https://fal.ai/dashboard/keys" + ) + fallback_tools = ["google_tts", "piper_tts", "elevenlabs_tts"] + agent_skills = ["elevenlabs"] + + capabilities = [ + "text_to_speech", + "voice_selection", + "expressive_delivery", + "multilingual", + "word_timestamps", + ] + supports = { + "voice_cloning": False, + "multilingual": True, + "offline": False, + "native_audio": True, + "inline_audio_tags": True, + "word_timestamps": True, + } + best_for = [ + "expressive ElevenLabs narration through an existing fal.ai connection", + "emotionally directed voiceover with Eleven v3 audio tags", + "multilingual narration without a separate ElevenLabs credential", + ] + not_good_for = [ + "offline generation", + "voice cloning or private custom ElevenLabs voices", + ] + + _MODELS = { + "eleven-v3": "fal-ai/elevenlabs/tts/eleven-v3", + "multilingual-v2": "fal-ai/elevenlabs/tts/multilingual-v2", + "turbo-v2.5": "fal-ai/elevenlabs/tts/turbo-v2.5", + } + _MODEL_ALIASES = { + "eleven_v3": "eleven-v3", + "eleven_multilingual_v2": "multilingual-v2", + "multilingual_v2": "multilingual-v2", + "eleven_turbo_v2_5": "turbo-v2.5", + "turbo_v2_5": "turbo-v2.5", + **{value: key for key, value in _MODELS.items()}, + } + _PRICE_PER_CHARACTER = { + "eleven-v3": 0.0001, + "multilingual-v2": 0.0001, + "turbo-v2.5": 0.00005, + } + _POLL_INTERVAL_SECONDS = 2 + _MAX_WAIT_SECONDS = 300 + + input_schema = { + "type": "object", + "required": ["text"], + "properties": { + "text": { + "type": "string", + "description": "Text to speak. Eleven v3 supports inline tags such as [whispers].", + }, + "voice": { + "type": "string", + "default": "Rachel", + "description": "fal.ai ElevenLabs voice name or ID", + }, + "voice_id": { + "type": "string", + "description": "Alias for voice, for compatibility with tts_selector", + }, + "model_id": { + "type": "string", + "default": "eleven-v3", + "description": "eleven-v3, multilingual-v2, or turbo-v2.5", + }, + "stability": { + "type": "number", + "default": 0.5, + "minimum": 0, + "maximum": 1, + }, + "similarity_boost": { + "type": "number", + "default": 0.75, + "minimum": 0, + "maximum": 1, + }, + "style": { + "type": "number", + "minimum": 0, + "maximum": 1, + }, + "speed": { + "type": "number", + "default": 1.0, + "minimum": 0.7, + "maximum": 1.2, + }, + "language_code": { + "type": "string", + "description": "Optional ISO 639-1 language code", + }, + "timestamps": { + "type": "boolean", + "default": False, + }, + "apply_text_normalization": { + "type": "string", + "default": "auto", + "enum": ["auto", "on", "off"], + }, + "output_format": { + "type": "string", + "default": "mp3_44100_128", + "enum": [ + "mp3_22050_32", + "mp3_44100_64", + "mp3_44100_96", + "mp3_44100_128", + "mp3_44100_192", + "pcm_16000", + "pcm_24000", + "pcm_44100", + "pcm_48000", + "opus_48000_64", + "opus_48000_96", + "opus_48000_128", + "opus_48000_192", + ], + }, + "seed": {"type": "integer"}, + "output_path": {"type": "string"}, + }, + } + + resource_profile = ResourceProfile( + cpu_cores=1, + ram_mb=256, + vram_mb=0, + disk_mb=50, + network_required=True, + ) + retry_policy = RetryPolicy( + max_retries=0, + retryable_errors=["rate_limit", "timeout"], + ) + idempotency_key_fields = [ + "text", + "voice", + "voice_id", + "model_id", + "stability", + "similarity_boost", + "style", + "speed", + "language_code", + "seed", + ] + side_effects = [ + "writes an audio file to output_path", + "submits one paid fal.ai ElevenLabs speech request", + ] + user_visible_verification = [ + "Listen to the generated voice sample before approving full narration", + ] + + def _get_api_key(self) -> str | None: + return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY") + + def get_status(self) -> ToolStatus: + return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE + + def _resolve_model(self, requested: str | None) -> tuple[str, str]: + model_name = requested or "eleven-v3" + model_name = self._MODEL_ALIASES.get(model_name, model_name) + if model_name not in self._MODELS: + choices = ", ".join(self._MODELS) + raise ValueError(f"model_id must be one of: {choices}") + return model_name, self._MODELS[model_name] + + def estimate_cost(self, inputs: dict[str, Any]) -> float: + model_name, _ = self._resolve_model(inputs.get("model_id")) + return round( + len(inputs.get("text", "")) * self._PRICE_PER_CHARACTER[model_name], + 4, + ) + + @staticmethod + def _output_extension(output_format: str) -> str: + return { + "mp3": "mp3", + "pcm": "pcm", + "opus": "opus", + }.get(output_format.split("_", 1)[0], "audio") + + def execute(self, inputs: dict[str, Any]) -> ToolResult: + api_key = self._get_api_key() + if not api_key: + return ToolResult( + success=False, + error="No fal.ai API key found. " + self.install_instructions, + ) + + text = str(inputs.get("text", "")).strip() + if not text: + return ToolResult(success=False, error="text is required") + + try: + model_name, model_id = self._resolve_model(inputs.get("model_id")) + except ValueError as exc: + return ToolResult(success=False, error=str(exc)) + + stability = float(inputs.get("stability", 0.5)) + similarity_boost = float(inputs.get("similarity_boost", 0.75)) + speed = float(inputs.get("speed", 1.0)) + if not 0 <= stability <= 1 or not 0 <= similarity_boost <= 1: + return ToolResult( + success=False, + error="stability and similarity_boost must be between 0 and 1", + ) + if not 0.7 <= speed <= 1.2: + return ToolResult(success=False, error="speed must be between 0.7 and 1.2") + + output_format = inputs.get("output_format", "mp3_44100_128") + voice = inputs.get("voice") or inputs.get("voice_id") or "Rachel" + payload: dict[str, Any] = { + "text": text, + "voice": voice, + "stability": stability, + "similarity_boost": similarity_boost, + "speed": speed, + "timestamps": bool(inputs.get("timestamps", False)), + "apply_text_normalization": inputs.get("apply_text_normalization", "auto"), + "output_format": output_format, + } + for optional in ("language_code", "seed", "style"): + if inputs.get(optional) is not None: + payload[optional] = inputs[optional] + + import requests + + started = time.time() + headers = { + "Authorization": f"Key {api_key}", + "Content-Type": "application/json", + } + queue_url = f"https://queue.fal.run/{model_id}" + + try: + submit_response = requests.post( + queue_url, + headers=headers, + json=payload, + timeout=30, + ) + submit_response.raise_for_status() + queue_data = submit_response.json() + status_url = queue_data["status_url"] + response_url = queue_data["response_url"] + + deadline = time.monotonic() + self._MAX_WAIT_SECONDS + while True: + if time.monotonic() >= deadline: + return ToolResult( + success=False, + error="fal.ai ElevenLabs speech timed out while waiting in the queue", + duration_seconds=round(time.time() - started, 2), + ) + time.sleep(self._POLL_INTERVAL_SECONDS) + status_response = requests.get(status_url, headers=headers, timeout=20) + status_response.raise_for_status() + status = status_response.json().get("status", "UNKNOWN") + if status == "COMPLETED": + break + if status in {"FAILED", "CANCELLED"}: + return ToolResult( + success=False, + error=f"fal.ai ElevenLabs speech {status.lower()}", + duration_seconds=round(time.time() - started, 2), + ) + + result_response = requests.get(response_url, headers=headers, timeout=30) + result_response.raise_for_status() + result_data = result_response.json() + audio_url = result_data["audio"]["url"] + + audio_response = requests.get(audio_url, timeout=120) + audio_response.raise_for_status() + default_output = f"fal_elevenlabs_tts.{self._output_extension(output_format)}" + output_path = Path(inputs.get("output_path", default_output)) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_bytes(audio_response.content) + except Exception as exc: + safe_error = str(exc).replace(api_key, "[REDACTED]") + return ToolResult( + success=False, + error=f"fal.ai ElevenLabs speech failed: {safe_error}", + duration_seconds=round(time.time() - started, 2), + ) + + data = { + "provider": self.provider, + "model": model_id, + "voice": voice, + "text_length": len(text), + "stability": stability, + "similarity_boost": similarity_boost, + "speed": speed, + "output": str(output_path), + "format": output_format, + } + if payload["timestamps"] and "timestamps" in result_data: + data["timestamps"] = result_data["timestamps"] + + return ToolResult( + success=True, + data=data, + artifacts=[str(output_path)], + cost_usd=self.estimate_cost({**inputs, "model_id": model_name, "text": text}), + duration_seconds=round(time.time() - started, 2), + model=model_id, + ) diff --git a/tools/audio/google_tts.py b/tools/audio/google_tts.py index f47dd4d6..984f73f8 100644 --- a/tools/audio/google_tts.py +++ b/tools/audio/google_tts.py @@ -8,7 +8,10 @@ from __future__ import annotations import base64 import os +import socket +import threading import time +from contextlib import contextmanager from pathlib import Path from typing import Any @@ -30,6 +33,32 @@ from tools.google_credentials import ( ) +_IPV4_REQUEST_LOCK = threading.Lock() + + +@contextmanager +def _google_tts_network_family(): + """Use IPv4 when the deployment key is restricted to the server's IPv4.""" + force_ipv4 = os.environ.get("GOOGLE_TTS_FORCE_IPV4", "").lower() in { + "1", + "true", + "yes", + } + if not force_ipv4: + yield + return + + import urllib3.util.connection + + with _IPV4_REQUEST_LOCK: + original = urllib3.util.connection.allowed_gai_family + urllib3.util.connection.allowed_gai_family = lambda: socket.AF_INET + try: + yield + finally: + urllib3.util.connection.allowed_gai_family = original + + class GoogleTTS(BaseTool): name = "google_tts" version = "0.1.0" @@ -279,12 +308,13 @@ class GoogleTTS(BaseTool): elif api_key: headers["x-goog-api-key"] = api_key - response = requests.post( - url, - headers=headers, - json=payload, - timeout=120, - ) + with _google_tts_network_family(): + response = requests.post( + url, + headers=headers, + json=payload, + timeout=120, + ) response.raise_for_status() audio_content = base64.b64decode(response.json()["audioContent"]) diff --git a/tools/audio/tts_selector.py b/tools/audio/tts_selector.py index 5c20cd57..a1b075b1 100644 --- a/tools/audio/tts_selector.py +++ b/tools/audio/tts_selector.py @@ -45,6 +45,10 @@ class TTSSelector(BaseTool): "type": "string", "description": "Provider-specific voice ID. Passed through to the selected TTS provider.", }, + "voice": { + "type": "string", + "description": "Provider-specific voice name or ID. fal.ai ElevenLabs accepts names such as Rachel.", + }, "voice_language": { "type": "string", "enum": ["zh", "en"], @@ -58,7 +62,7 @@ class TTSSelector(BaseTool): }, "model_id": { "type": "string", - "description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.", + "description": "TTS model to use (e.g. eleven-v3 or eleven_multilingual_v2). Passed through to provider.", }, "stability": { "type": "number", "minimum": 0, "maximum": 1, @@ -100,6 +104,24 @@ class TTSSelector(BaseTool): "default": "text", "description": "Use 'ssml' only when the selected provider supports tags such as .", }, + "language_code": { + "type": "string", + "description": "Provider-specific language code, such as en-US for Google or en for fal.ai ElevenLabs.", + }, + "timestamps": { + "type": "boolean", + "default": False, + "description": "Request word timestamps when the selected provider supports them.", + }, + "apply_text_normalization": { + "type": "string", + "enum": ["auto", "on", "off"], + "description": "Text normalization mode for providers that support it.", + }, + "seed": { + "type": "integer", + "description": "Optional generation seed for providers that support reproducible speech.", + }, "voice_performance": { "type": "object", "description": "Structured voice-performance plan or section delivery cues from the script artifact.",