add fal ElevenLabs speech and secure audio routing

This commit is contained in:
Codex
2026-08-08 13:02:07 +00:00
parent 6b4c73c6df
commit 21d51ab9c8
13 changed files with 906 additions and 10 deletions

View File

@@ -5,7 +5,22 @@ description: Generate AI voiceovers, sound effects, and music using ElevenLabs A
# ElevenLabs Audio Generation
Requires `ELEVENLABS_API_KEY` in `.env`.
## OpenMontage provider routing
Inspect the OpenMontage registry before choosing an authentication path.
- Prefer `fal_elevenlabs_tts` when it is available. It provides Eleven v3,
Multilingual v2, and Turbo v2.5 through the centrally managed fal.ai
connection; no separate ElevenLabs credential is needed.
- Use `elevenlabs_tts` only when that direct provider is already reported as
available by the registry.
- In a shared installation, never tell the user to create a `.env`, export a
key, or paste a credential. Report missing direct-provider access as an
administrator setup request.
The direct API examples below require a centrally configured
`ELEVENLABS_API_KEY`; they are not the default path when the fal.ai provider is
available.
## Text-to-Speech

View File

@@ -707,6 +707,8 @@ The `.agents/skills/` directory is large. When you're not coming in through a to
- **Do not skip stage director skills.** Before executing any pipeline stage, read its director skill. The skill contains the quality bar, the workflow, and the review criteria.
- Do not use deleted legacy names such as `tts_cloud`, `tts_engine`, or `video_gen`.
- Do not hardcode provider names, API key names, or setup URLs. Read them from the registry's `install_instructions` and `dependencies` fields.
- Do not tell a restricted shared-installation user to add credentials manually, create a `.env`, or export a key. Surface the unavailable provider as an administrator setup request and continue only with centrally provisioned capabilities.
- Before offering a direct vendor credential, check the registry for an available wrapper through an already configured provider (for example, a partner model hosted by fal.ai).
- Do not begin asset generation before user approval on the production plan.
- Do not hide degraded paths. Record substitutions and blocked options explicitly.
- Do not present a single unavailable tool in isolation. Always show the full capability picture: "X of Y providers configured for this capability."

View File

@@ -184,6 +184,7 @@ stages:
- diagram_gen
- code_snippet
- music_gen
- fal_elevenlabs_music
tools_available:
- tts_selector
- image_selector
@@ -192,6 +193,7 @@ stages:
- diagram_gen
- code_snippet
- music_gen
- fal_elevenlabs_music
checkpoint_required: true
human_approval_default: true
review_focus:

View File

@@ -31,7 +31,7 @@ Quick routing for common animation-pipeline needs:
|-------|----------|---------|
| Schema | `schemas/artifacts/asset_manifest.schema.json` | Artifact validation |
| Prior artifacts | `state.artifacts["scene_plan"]["scene_plan"]`, `state.artifacts["script"]["script"]`, `state.artifacts["proposal"]["proposal_packet"]` | Tool path and beat map |
| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen` — selectors auto-discover all available providers from the registry | Asset production options |
| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen`, `fal_elevenlabs_music` — selectors auto-discover all available providers from the registry | Asset production options |
| Playbook | Active style playbook | Visual consistency |
## Process

View File

@@ -694,6 +694,7 @@ class TestCapabilityMetadata:
"dashscope",
"doubao",
"elevenlabs",
"fal.ai",
"google_tts",
"kling_official",
"openai",

View File

@@ -0,0 +1,103 @@
from __future__ import annotations
from unittest.mock import MagicMock, patch
from tools.audio.fal_elevenlabs_music import FalElevenLabsMusic
from tools.base_tool import ToolStatus
from tools.tool_registry import ToolRegistry
def _response(*, json_data=None, content=b""):
response = MagicMock()
response.json.return_value = json_data
response.content = content
response.raise_for_status.return_value = None
return response
def test_contract_and_rounded_cost(monkeypatch):
tool = FalElevenLabsMusic()
monkeypatch.setenv("FAL_KEY", "test-key")
assert tool.get_status() == ToolStatus.AVAILABLE
assert tool.estimate_cost({"duration_seconds": 20}) == 0.80
assert tool.estimate_cost({"duration_seconds": 61}) == 1.60
assert tool.get_info()["capability"] == "music_generation"
assert tool.get_info()["provider"] == "fal.ai"
def test_registry_discovers_provider(monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
registry = ToolRegistry()
registry.discover()
tool = registry.get("fal_elevenlabs_music")
assert tool is not None
assert tool.get_status() == ToolStatus.AVAILABLE
def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
output_path = tmp_path / "music.mp3"
tool = FalElevenLabsMusic()
tool._POLL_INTERVAL_SECONDS = 0
post_response = _response(
json_data={
"status_url": "https://queue.example/status",
"response_url": "https://queue.example/result",
}
)
status_response = _response(json_data={"status": "COMPLETED"})
result_response = _response(
json_data={"audio": {"url": "https://media.example/music.mp3"}}
)
audio_response = _response(content=b"fake-mp3")
with (
patch("requests.post", return_value=post_response) as mock_post,
patch(
"requests.get",
side_effect=[status_response, result_response, audio_response],
) as mock_get,
):
result = tool.execute(
{
"prompt": "gentle felt piano",
"duration_seconds": 20,
"force_instrumental": True,
"output_path": str(output_path),
}
)
assert result.success is True
assert result.cost_usd == 0.80
assert result.model == "fal-ai/elevenlabs/music"
assert output_path.read_bytes() == b"fake-mp3"
assert mock_post.call_count == 1
assert mock_post.call_args.kwargs["json"]["music_length_ms"] == 20000
assert mock_post.call_args.kwargs["json"]["force_instrumental"] is True
assert mock_get.call_count == 3
def test_execute_rejects_missing_duration_without_request(monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
with patch("requests.post") as mock_post:
result = FalElevenLabsMusic().execute({"prompt": "gentle piano"})
assert result.success is False
assert result.error == "duration_seconds is required"
mock_post.assert_not_called()
def test_error_redacts_fal_key(monkeypatch):
secret = "test-production-shaped-fal-key"
monkeypatch.setenv("FAL_KEY", secret)
with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")):
result = FalElevenLabsMusic().execute(
{"prompt": "gentle piano", "duration_seconds": 20}
)
assert result.success is False
assert secret not in result.error
assert "[REDACTED]" in result.error

View File

@@ -0,0 +1,106 @@
from __future__ import annotations
from unittest.mock import MagicMock, patch
from tools.audio.fal_elevenlabs_tts import FalElevenLabsTTS
from tools.base_tool import ToolStatus
from tools.tool_registry import ToolRegistry
def _response(*, json_data=None, content=b""):
response = MagicMock()
response.json.return_value = json_data
response.content = content
response.raise_for_status.return_value = None
return response
def test_contract_models_and_cost(monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
tool = FalElevenLabsTTS()
assert tool.get_status() == ToolStatus.AVAILABLE
assert tool.get_info()["capability"] == "tts"
assert tool.get_info()["provider"] == "fal.ai"
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "eleven-v3"}) == 0.1
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "multilingual-v2"}) == 0.1
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "turbo-v2.5"}) == 0.05
def test_registry_discovers_fal_tts(monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
registry = ToolRegistry()
registry.discover()
tool = registry.get("fal_elevenlabs_tts")
assert tool is not None
assert tool.get_status() == ToolStatus.AVAILABLE
def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
output_path = tmp_path / "speech.mp3"
tool = FalElevenLabsTTS()
tool._POLL_INTERVAL_SECONDS = 0
post_response = _response(
json_data={
"status_url": "https://queue.example/status",
"response_url": "https://queue.example/result",
}
)
status_response = _response(json_data={"status": "COMPLETED"})
result_response = _response(
json_data={"audio": {"url": "https://media.example/speech.mp3"}}
)
audio_response = _response(content=b"fake-mp3")
with (
patch("requests.post", return_value=post_response) as mock_post,
patch(
"requests.get",
side_effect=[status_response, result_response, audio_response],
) as mock_get,
):
result = tool.execute(
{
"text": "A calm, measured test.",
"voice_id": "Rachel",
"model_id": "eleven-v3",
"stability": 0.65,
"language_code": "en",
"output_path": str(output_path),
}
)
assert result.success is True
assert result.model == "fal-ai/elevenlabs/tts/eleven-v3"
assert output_path.read_bytes() == b"fake-mp3"
assert mock_post.call_count == 1
assert mock_post.call_args.args[0].endswith("/fal-ai/elevenlabs/tts/eleven-v3")
assert mock_post.call_args.kwargs["json"]["voice"] == "Rachel"
assert mock_post.call_args.kwargs["json"]["stability"] == 0.65
assert mock_get.call_count == 3
def test_invalid_model_does_not_submit(monkeypatch):
monkeypatch.setenv("FAL_KEY", "test-key")
with patch("requests.post") as mock_post:
result = FalElevenLabsTTS().execute(
{"text": "hello", "model_id": "not-a-model"}
)
assert result.success is False
assert "model_id must be one of" in result.error
mock_post.assert_not_called()
def test_error_redacts_fal_key(monkeypatch):
secret = "test-production-shaped-fal-key"
monkeypatch.setenv("FAL_KEY", secret)
with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")):
result = FalElevenLabsTTS().execute({"text": "hello"})
assert result.success is False
assert secret not in result.error
assert "[REDACTED]" in result.error

View File

@@ -34,3 +34,26 @@ def test_tts_key_uses_header_and_is_redacted_from_errors(monkeypatch, tmp_path):
assert result.success is False
assert secret not in result.error
assert "[REDACTED]" in result.error
def test_production_adapter_can_force_google_tts_to_ipv4(monkeypatch, tmp_path):
import socket
import requests
import urllib3.util.connection
monkeypatch.setenv("GOOGLE_TTS_API_KEY", "test-key")
monkeypatch.setenv("GOOGLE_TTS_FORCE_IPV4", "1")
original = urllib3.util.connection.allowed_gai_family
def inspect_request(url, **kwargs):
assert urllib3.util.connection.allowed_gai_family() == socket.AF_INET
raise requests.HTTPError("safe expected failure")
monkeypatch.setattr(requests, "post", inspect_request)
result = GoogleTTS().execute(
{"text": "safe test sentence", "output_path": str(tmp_path / "speech.mp3")}
)
assert result.success is False
assert urllib3.util.connection.allowed_gai_family is original

View File

@@ -36,7 +36,9 @@ class ElevenLabsTTS(BaseTool):
install_instructions = (
"Set the ELEVENLABS_API_KEY environment variable:\n"
" export ELEVENLABS_API_KEY=your_key_here\n"
"Get a key at https://elevenlabs.io"
"Get a key at https://elevenlabs.io\n"
"If fal_elevenlabs_tts is available, use it instead to access ElevenLabs "
"speech through fal.ai without a separate ElevenLabs key."
)
fallback = "openai_tts"
fallback_tools = ["openai_tts", "piper_tts"]

View File

@@ -0,0 +1,231 @@
"""Generate music with ElevenLabs Music through fal.ai.
This provider uses OpenMontage's shared ``FAL_KEY`` credential and the fal.ai
queue API, then downloads the generated MP3 to a project-local path.
"""
from __future__ import annotations
import math
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class FalElevenLabsMusic(BaseTool):
"""Generate a precisely timed music track through fal.ai."""
name = "fal_elevenlabs_music"
version = "0.1.0"
tier = ToolTier.GENERATE
capability = "music_generation"
provider = "fal.ai"
stability = ToolStability.BETA
execution_mode = ExecutionMode.ASYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:FAL_KEY"]
install_instructions = (
"Set FAL_KEY to a fal.ai API key. "
"Get one at https://fal.ai/dashboard/keys"
)
fallback_tools = ["music_gen", "google_music"]
agent_skills = ["music", "elevenlabs"]
capabilities = [
"generate_background_music",
"generate_instrumental",
]
supports = {
"instrumental": True,
"style_control": True,
"exact_duration": True,
}
best_for = [
"precisely timed instrumental background music",
"high-quality ElevenLabs Music through a shared fal.ai account",
"short-form social video soundtracks",
]
not_good_for = [
"offline generation",
"free music sourcing",
"sub-3-second sound effects",
]
input_schema = {
"type": "object",
"required": ["prompt", "duration_seconds"],
"properties": {
"prompt": {
"type": "string",
"description": "Music description including mood, style, and instruments",
},
"duration_seconds": {
"type": "number",
"minimum": 3,
"maximum": 600,
"description": "Exact target length in seconds",
},
"force_instrumental": {
"type": "boolean",
"default": True,
"description": "Generate music without vocals",
},
"output_path": {
"type": "string",
"default": "fal_music_output.mp3",
},
},
}
resource_profile = ResourceProfile(
cpu_cores=1,
ram_mb=256,
vram_mb=0,
disk_mb=50,
network_required=True,
)
retry_policy = RetryPolicy(
max_retries=0,
retryable_errors=["rate_limit", "timeout"],
)
idempotency_key_fields = ["prompt", "duration_seconds", "force_instrumental"]
side_effects = [
"writes an MP3 file to output_path",
"submits one paid fal.ai generation request",
]
user_visible_verification = [
"Listen to the generated music for mood, mix, and duration",
]
_MODEL = "fal-ai/elevenlabs/music"
_QUEUE_URL = f"https://queue.fal.run/{_MODEL}"
_POLL_INTERVAL_SECONDS = 5
_MAX_WAIT_SECONDS = 600
def _get_api_key(self) -> str | None:
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
def get_status(self) -> ToolStatus:
return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE
def estimate_cost(self, inputs: dict[str, Any]) -> float:
"""fal bills $0.80 per output minute, rounded up to a full minute."""
duration = inputs.get("duration_seconds")
if duration is None:
raise ValueError("duration_seconds is required for cost estimation")
return round(math.ceil(float(duration) / 60.0) * 0.80, 2)
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="No fal.ai API key found. " + self.install_instructions,
)
duration = inputs.get("duration_seconds")
if duration is None:
return ToolResult(success=False, error="duration_seconds is required")
duration = float(duration)
if not 3 <= duration <= 600:
return ToolResult(
success=False,
error="duration_seconds must be between 3 and 600",
)
import requests
started = time.time()
headers = {
"Authorization": f"Key {api_key}",
"Content-Type": "application/json",
}
payload = {
"prompt": inputs["prompt"],
"music_length_ms": round(duration * 1000),
"force_instrumental": bool(inputs.get("force_instrumental", True)),
}
try:
submit_response = requests.post(
self._QUEUE_URL,
headers=headers,
json=payload,
timeout=30,
)
submit_response.raise_for_status()
queue_data = submit_response.json()
status_url = queue_data["status_url"]
response_url = queue_data["response_url"]
deadline = time.monotonic() + self._MAX_WAIT_SECONDS
while True:
if time.monotonic() >= deadline:
return ToolResult(
success=False,
error="fal.ai music generation timed out while waiting in the queue",
duration_seconds=round(time.time() - started, 2),
)
time.sleep(self._POLL_INTERVAL_SECONDS)
status_response = requests.get(status_url, headers=headers, timeout=20)
status_response.raise_for_status()
status = status_response.json().get("status", "UNKNOWN")
if status == "COMPLETED":
break
if status in {"FAILED", "CANCELLED"}:
return ToolResult(
success=False,
error=f"fal.ai music generation {status.lower()}",
duration_seconds=round(time.time() - started, 2),
)
result_response = requests.get(response_url, headers=headers, timeout=30)
result_response.raise_for_status()
result_data = result_response.json()
audio_url = result_data["audio"]["url"]
audio_response = requests.get(audio_url, timeout=120)
audio_response.raise_for_status()
output_path = Path(inputs.get("output_path", "fal_music_output.mp3"))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(audio_response.content)
except Exception as exc:
safe_error = str(exc).replace(api_key, "[REDACTED]")
return ToolResult(
success=False,
error=f"fal.ai ElevenLabs Music generation failed: {safe_error}",
duration_seconds=round(time.time() - started, 2),
)
return ToolResult(
success=True,
data={
"provider": "fal.ai",
"model": self._MODEL,
"prompt": inputs["prompt"],
"duration_seconds": duration,
"force_instrumental": payload["force_instrumental"],
"output": str(output_path),
"format": "mp3",
},
artifacts=[str(output_path)],
cost_usd=self.estimate_cost(inputs),
duration_seconds=round(time.time() - started, 2),
model=self._MODEL,
)

View File

@@ -0,0 +1,359 @@
"""Generate ElevenLabs speech through fal.ai using the shared FAL credential."""
from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
from tools.base_tool import (
BaseTool,
Determinism,
ExecutionMode,
ResourceProfile,
RetryPolicy,
ToolResult,
ToolRuntime,
ToolStability,
ToolStatus,
ToolTier,
)
class FalElevenLabsTTS(BaseTool):
"""Generate expressive narration with ElevenLabs models hosted by fal.ai."""
name = "fal_elevenlabs_tts"
version = "0.1.0"
tier = ToolTier.VOICE
capability = "tts"
provider = "fal.ai"
stability = ToolStability.BETA
execution_mode = ExecutionMode.ASYNC
determinism = Determinism.STOCHASTIC
runtime = ToolRuntime.API
dependencies = ["env:FAL_KEY"]
install_instructions = (
"Set FAL_KEY to a fal.ai API key. No separate ElevenLabs key is needed. "
"Get a fal.ai key at https://fal.ai/dashboard/keys"
)
fallback_tools = ["google_tts", "piper_tts", "elevenlabs_tts"]
agent_skills = ["elevenlabs"]
capabilities = [
"text_to_speech",
"voice_selection",
"expressive_delivery",
"multilingual",
"word_timestamps",
]
supports = {
"voice_cloning": False,
"multilingual": True,
"offline": False,
"native_audio": True,
"inline_audio_tags": True,
"word_timestamps": True,
}
best_for = [
"expressive ElevenLabs narration through an existing fal.ai connection",
"emotionally directed voiceover with Eleven v3 audio tags",
"multilingual narration without a separate ElevenLabs credential",
]
not_good_for = [
"offline generation",
"voice cloning or private custom ElevenLabs voices",
]
_MODELS = {
"eleven-v3": "fal-ai/elevenlabs/tts/eleven-v3",
"multilingual-v2": "fal-ai/elevenlabs/tts/multilingual-v2",
"turbo-v2.5": "fal-ai/elevenlabs/tts/turbo-v2.5",
}
_MODEL_ALIASES = {
"eleven_v3": "eleven-v3",
"eleven_multilingual_v2": "multilingual-v2",
"multilingual_v2": "multilingual-v2",
"eleven_turbo_v2_5": "turbo-v2.5",
"turbo_v2_5": "turbo-v2.5",
**{value: key for key, value in _MODELS.items()},
}
_PRICE_PER_CHARACTER = {
"eleven-v3": 0.0001,
"multilingual-v2": 0.0001,
"turbo-v2.5": 0.00005,
}
_POLL_INTERVAL_SECONDS = 2
_MAX_WAIT_SECONDS = 300
input_schema = {
"type": "object",
"required": ["text"],
"properties": {
"text": {
"type": "string",
"description": "Text to speak. Eleven v3 supports inline tags such as [whispers].",
},
"voice": {
"type": "string",
"default": "Rachel",
"description": "fal.ai ElevenLabs voice name or ID",
},
"voice_id": {
"type": "string",
"description": "Alias for voice, for compatibility with tts_selector",
},
"model_id": {
"type": "string",
"default": "eleven-v3",
"description": "eleven-v3, multilingual-v2, or turbo-v2.5",
},
"stability": {
"type": "number",
"default": 0.5,
"minimum": 0,
"maximum": 1,
},
"similarity_boost": {
"type": "number",
"default": 0.75,
"minimum": 0,
"maximum": 1,
},
"style": {
"type": "number",
"minimum": 0,
"maximum": 1,
},
"speed": {
"type": "number",
"default": 1.0,
"minimum": 0.7,
"maximum": 1.2,
},
"language_code": {
"type": "string",
"description": "Optional ISO 639-1 language code",
},
"timestamps": {
"type": "boolean",
"default": False,
},
"apply_text_normalization": {
"type": "string",
"default": "auto",
"enum": ["auto", "on", "off"],
},
"output_format": {
"type": "string",
"default": "mp3_44100_128",
"enum": [
"mp3_22050_32",
"mp3_44100_64",
"mp3_44100_96",
"mp3_44100_128",
"mp3_44100_192",
"pcm_16000",
"pcm_24000",
"pcm_44100",
"pcm_48000",
"opus_48000_64",
"opus_48000_96",
"opus_48000_128",
"opus_48000_192",
],
},
"seed": {"type": "integer"},
"output_path": {"type": "string"},
},
}
resource_profile = ResourceProfile(
cpu_cores=1,
ram_mb=256,
vram_mb=0,
disk_mb=50,
network_required=True,
)
retry_policy = RetryPolicy(
max_retries=0,
retryable_errors=["rate_limit", "timeout"],
)
idempotency_key_fields = [
"text",
"voice",
"voice_id",
"model_id",
"stability",
"similarity_boost",
"style",
"speed",
"language_code",
"seed",
]
side_effects = [
"writes an audio file to output_path",
"submits one paid fal.ai ElevenLabs speech request",
]
user_visible_verification = [
"Listen to the generated voice sample before approving full narration",
]
def _get_api_key(self) -> str | None:
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
def get_status(self) -> ToolStatus:
return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE
def _resolve_model(self, requested: str | None) -> tuple[str, str]:
model_name = requested or "eleven-v3"
model_name = self._MODEL_ALIASES.get(model_name, model_name)
if model_name not in self._MODELS:
choices = ", ".join(self._MODELS)
raise ValueError(f"model_id must be one of: {choices}")
return model_name, self._MODELS[model_name]
def estimate_cost(self, inputs: dict[str, Any]) -> float:
model_name, _ = self._resolve_model(inputs.get("model_id"))
return round(
len(inputs.get("text", "")) * self._PRICE_PER_CHARACTER[model_name],
4,
)
@staticmethod
def _output_extension(output_format: str) -> str:
return {
"mp3": "mp3",
"pcm": "pcm",
"opus": "opus",
}.get(output_format.split("_", 1)[0], "audio")
def execute(self, inputs: dict[str, Any]) -> ToolResult:
api_key = self._get_api_key()
if not api_key:
return ToolResult(
success=False,
error="No fal.ai API key found. " + self.install_instructions,
)
text = str(inputs.get("text", "")).strip()
if not text:
return ToolResult(success=False, error="text is required")
try:
model_name, model_id = self._resolve_model(inputs.get("model_id"))
except ValueError as exc:
return ToolResult(success=False, error=str(exc))
stability = float(inputs.get("stability", 0.5))
similarity_boost = float(inputs.get("similarity_boost", 0.75))
speed = float(inputs.get("speed", 1.0))
if not 0 <= stability <= 1 or not 0 <= similarity_boost <= 1:
return ToolResult(
success=False,
error="stability and similarity_boost must be between 0 and 1",
)
if not 0.7 <= speed <= 1.2:
return ToolResult(success=False, error="speed must be between 0.7 and 1.2")
output_format = inputs.get("output_format", "mp3_44100_128")
voice = inputs.get("voice") or inputs.get("voice_id") or "Rachel"
payload: dict[str, Any] = {
"text": text,
"voice": voice,
"stability": stability,
"similarity_boost": similarity_boost,
"speed": speed,
"timestamps": bool(inputs.get("timestamps", False)),
"apply_text_normalization": inputs.get("apply_text_normalization", "auto"),
"output_format": output_format,
}
for optional in ("language_code", "seed", "style"):
if inputs.get(optional) is not None:
payload[optional] = inputs[optional]
import requests
started = time.time()
headers = {
"Authorization": f"Key {api_key}",
"Content-Type": "application/json",
}
queue_url = f"https://queue.fal.run/{model_id}"
try:
submit_response = requests.post(
queue_url,
headers=headers,
json=payload,
timeout=30,
)
submit_response.raise_for_status()
queue_data = submit_response.json()
status_url = queue_data["status_url"]
response_url = queue_data["response_url"]
deadline = time.monotonic() + self._MAX_WAIT_SECONDS
while True:
if time.monotonic() >= deadline:
return ToolResult(
success=False,
error="fal.ai ElevenLabs speech timed out while waiting in the queue",
duration_seconds=round(time.time() - started, 2),
)
time.sleep(self._POLL_INTERVAL_SECONDS)
status_response = requests.get(status_url, headers=headers, timeout=20)
status_response.raise_for_status()
status = status_response.json().get("status", "UNKNOWN")
if status == "COMPLETED":
break
if status in {"FAILED", "CANCELLED"}:
return ToolResult(
success=False,
error=f"fal.ai ElevenLabs speech {status.lower()}",
duration_seconds=round(time.time() - started, 2),
)
result_response = requests.get(response_url, headers=headers, timeout=30)
result_response.raise_for_status()
result_data = result_response.json()
audio_url = result_data["audio"]["url"]
audio_response = requests.get(audio_url, timeout=120)
audio_response.raise_for_status()
default_output = f"fal_elevenlabs_tts.{self._output_extension(output_format)}"
output_path = Path(inputs.get("output_path", default_output))
output_path.parent.mkdir(parents=True, exist_ok=True)
output_path.write_bytes(audio_response.content)
except Exception as exc:
safe_error = str(exc).replace(api_key, "[REDACTED]")
return ToolResult(
success=False,
error=f"fal.ai ElevenLabs speech failed: {safe_error}",
duration_seconds=round(time.time() - started, 2),
)
data = {
"provider": self.provider,
"model": model_id,
"voice": voice,
"text_length": len(text),
"stability": stability,
"similarity_boost": similarity_boost,
"speed": speed,
"output": str(output_path),
"format": output_format,
}
if payload["timestamps"] and "timestamps" in result_data:
data["timestamps"] = result_data["timestamps"]
return ToolResult(
success=True,
data=data,
artifacts=[str(output_path)],
cost_usd=self.estimate_cost({**inputs, "model_id": model_name, "text": text}),
duration_seconds=round(time.time() - started, 2),
model=model_id,
)

View File

@@ -8,7 +8,10 @@ from __future__ import annotations
import base64
import os
import socket
import threading
import time
from contextlib import contextmanager
from pathlib import Path
from typing import Any
@@ -30,6 +33,32 @@ from tools.google_credentials import (
)
_IPV4_REQUEST_LOCK = threading.Lock()
@contextmanager
def _google_tts_network_family():
"""Use IPv4 when the deployment key is restricted to the server's IPv4."""
force_ipv4 = os.environ.get("GOOGLE_TTS_FORCE_IPV4", "").lower() in {
"1",
"true",
"yes",
}
if not force_ipv4:
yield
return
import urllib3.util.connection
with _IPV4_REQUEST_LOCK:
original = urllib3.util.connection.allowed_gai_family
urllib3.util.connection.allowed_gai_family = lambda: socket.AF_INET
try:
yield
finally:
urllib3.util.connection.allowed_gai_family = original
class GoogleTTS(BaseTool):
name = "google_tts"
version = "0.1.0"
@@ -279,12 +308,13 @@ class GoogleTTS(BaseTool):
elif api_key:
headers["x-goog-api-key"] = api_key
response = requests.post(
url,
headers=headers,
json=payload,
timeout=120,
)
with _google_tts_network_family():
response = requests.post(
url,
headers=headers,
json=payload,
timeout=120,
)
response.raise_for_status()
audio_content = base64.b64decode(response.json()["audioContent"])

View File

@@ -45,6 +45,10 @@ class TTSSelector(BaseTool):
"type": "string",
"description": "Provider-specific voice ID. Passed through to the selected TTS provider.",
},
"voice": {
"type": "string",
"description": "Provider-specific voice name or ID. fal.ai ElevenLabs accepts names such as Rachel.",
},
"voice_language": {
"type": "string",
"enum": ["zh", "en"],
@@ -58,7 +62,7 @@ class TTSSelector(BaseTool):
},
"model_id": {
"type": "string",
"description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.",
"description": "TTS model to use (e.g. eleven-v3 or eleven_multilingual_v2). Passed through to provider.",
},
"stability": {
"type": "number", "minimum": 0, "maximum": 1,
@@ -100,6 +104,24 @@ class TTSSelector(BaseTool):
"default": "text",
"description": "Use 'ssml' only when the selected provider supports tags such as <break>.",
},
"language_code": {
"type": "string",
"description": "Provider-specific language code, such as en-US for Google or en for fal.ai ElevenLabs.",
},
"timestamps": {
"type": "boolean",
"default": False,
"description": "Request word timestamps when the selected provider supports them.",
},
"apply_text_normalization": {
"type": "string",
"enum": ["auto", "on", "off"],
"description": "Text normalization mode for providers that support it.",
},
"seed": {
"type": "integer",
"description": "Optional generation seed for providers that support reproducible speech.",
},
"voice_performance": {
"type": "object",
"description": "Structured voice-performance plan or section delivery cues from the script artifact.",