mirror of
https://github.com/calesthio/OpenMontage.git
synced 2026-08-23 16:46:47 +08:00
add fal ElevenLabs speech and secure audio routing
This commit is contained in:
@@ -5,7 +5,22 @@ description: Generate AI voiceovers, sound effects, and music using ElevenLabs A
|
||||
|
||||
# ElevenLabs Audio Generation
|
||||
|
||||
Requires `ELEVENLABS_API_KEY` in `.env`.
|
||||
## OpenMontage provider routing
|
||||
|
||||
Inspect the OpenMontage registry before choosing an authentication path.
|
||||
|
||||
- Prefer `fal_elevenlabs_tts` when it is available. It provides Eleven v3,
|
||||
Multilingual v2, and Turbo v2.5 through the centrally managed fal.ai
|
||||
connection; no separate ElevenLabs credential is needed.
|
||||
- Use `elevenlabs_tts` only when that direct provider is already reported as
|
||||
available by the registry.
|
||||
- In a shared installation, never tell the user to create a `.env`, export a
|
||||
key, or paste a credential. Report missing direct-provider access as an
|
||||
administrator setup request.
|
||||
|
||||
The direct API examples below require a centrally configured
|
||||
`ELEVENLABS_API_KEY`; they are not the default path when the fal.ai provider is
|
||||
available.
|
||||
|
||||
## Text-to-Speech
|
||||
|
||||
|
||||
@@ -707,6 +707,8 @@ The `.agents/skills/` directory is large. When you're not coming in through a to
|
||||
- **Do not skip stage director skills.** Before executing any pipeline stage, read its director skill. The skill contains the quality bar, the workflow, and the review criteria.
|
||||
- Do not use deleted legacy names such as `tts_cloud`, `tts_engine`, or `video_gen`.
|
||||
- Do not hardcode provider names, API key names, or setup URLs. Read them from the registry's `install_instructions` and `dependencies` fields.
|
||||
- Do not tell a restricted shared-installation user to add credentials manually, create a `.env`, or export a key. Surface the unavailable provider as an administrator setup request and continue only with centrally provisioned capabilities.
|
||||
- Before offering a direct vendor credential, check the registry for an available wrapper through an already configured provider (for example, a partner model hosted by fal.ai).
|
||||
- Do not begin asset generation before user approval on the production plan.
|
||||
- Do not hide degraded paths. Record substitutions and blocked options explicitly.
|
||||
- Do not present a single unavailable tool in isolation. Always show the full capability picture: "X of Y providers configured for this capability."
|
||||
|
||||
@@ -184,6 +184,7 @@ stages:
|
||||
- diagram_gen
|
||||
- code_snippet
|
||||
- music_gen
|
||||
- fal_elevenlabs_music
|
||||
tools_available:
|
||||
- tts_selector
|
||||
- image_selector
|
||||
@@ -192,6 +193,7 @@ stages:
|
||||
- diagram_gen
|
||||
- code_snippet
|
||||
- music_gen
|
||||
- fal_elevenlabs_music
|
||||
checkpoint_required: true
|
||||
human_approval_default: true
|
||||
review_focus:
|
||||
|
||||
@@ -31,7 +31,7 @@ Quick routing for common animation-pipeline needs:
|
||||
|-------|----------|---------|
|
||||
| Schema | `schemas/artifacts/asset_manifest.schema.json` | Artifact validation |
|
||||
| Prior artifacts | `state.artifacts["scene_plan"]["scene_plan"]`, `state.artifacts["script"]["script"]`, `state.artifacts["proposal"]["proposal_packet"]` | Tool path and beat map |
|
||||
| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen` — selectors auto-discover all available providers from the registry | Asset production options |
|
||||
| Tools | `tts_selector`, `image_selector`, `video_selector`, `math_animate`, `diagram_gen`, `code_snippet`, `music_gen`, `fal_elevenlabs_music` — selectors auto-discover all available providers from the registry | Asset production options |
|
||||
| Playbook | Active style playbook | Visual consistency |
|
||||
|
||||
## Process
|
||||
|
||||
@@ -694,6 +694,7 @@ class TestCapabilityMetadata:
|
||||
"dashscope",
|
||||
"doubao",
|
||||
"elevenlabs",
|
||||
"fal.ai",
|
||||
"google_tts",
|
||||
"kling_official",
|
||||
"openai",
|
||||
|
||||
103
tests/tools/test_fal_elevenlabs_music.py
Normal file
103
tests/tools/test_fal_elevenlabs_music.py
Normal file
@@ -0,0 +1,103 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from tools.audio.fal_elevenlabs_music import FalElevenLabsMusic
|
||||
from tools.base_tool import ToolStatus
|
||||
from tools.tool_registry import ToolRegistry
|
||||
|
||||
|
||||
def _response(*, json_data=None, content=b""):
|
||||
response = MagicMock()
|
||||
response.json.return_value = json_data
|
||||
response.content = content
|
||||
response.raise_for_status.return_value = None
|
||||
return response
|
||||
|
||||
|
||||
def test_contract_and_rounded_cost(monkeypatch):
|
||||
tool = FalElevenLabsMusic()
|
||||
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
assert tool.get_status() == ToolStatus.AVAILABLE
|
||||
assert tool.estimate_cost({"duration_seconds": 20}) == 0.80
|
||||
assert tool.estimate_cost({"duration_seconds": 61}) == 1.60
|
||||
assert tool.get_info()["capability"] == "music_generation"
|
||||
assert tool.get_info()["provider"] == "fal.ai"
|
||||
|
||||
|
||||
def test_registry_discovers_provider(monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
registry = ToolRegistry()
|
||||
registry.discover()
|
||||
|
||||
tool = registry.get("fal_elevenlabs_music")
|
||||
assert tool is not None
|
||||
assert tool.get_status() == ToolStatus.AVAILABLE
|
||||
|
||||
|
||||
def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
output_path = tmp_path / "music.mp3"
|
||||
tool = FalElevenLabsMusic()
|
||||
tool._POLL_INTERVAL_SECONDS = 0
|
||||
|
||||
post_response = _response(
|
||||
json_data={
|
||||
"status_url": "https://queue.example/status",
|
||||
"response_url": "https://queue.example/result",
|
||||
}
|
||||
)
|
||||
status_response = _response(json_data={"status": "COMPLETED"})
|
||||
result_response = _response(
|
||||
json_data={"audio": {"url": "https://media.example/music.mp3"}}
|
||||
)
|
||||
audio_response = _response(content=b"fake-mp3")
|
||||
|
||||
with (
|
||||
patch("requests.post", return_value=post_response) as mock_post,
|
||||
patch(
|
||||
"requests.get",
|
||||
side_effect=[status_response, result_response, audio_response],
|
||||
) as mock_get,
|
||||
):
|
||||
result = tool.execute(
|
||||
{
|
||||
"prompt": "gentle felt piano",
|
||||
"duration_seconds": 20,
|
||||
"force_instrumental": True,
|
||||
"output_path": str(output_path),
|
||||
}
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.cost_usd == 0.80
|
||||
assert result.model == "fal-ai/elevenlabs/music"
|
||||
assert output_path.read_bytes() == b"fake-mp3"
|
||||
assert mock_post.call_count == 1
|
||||
assert mock_post.call_args.kwargs["json"]["music_length_ms"] == 20000
|
||||
assert mock_post.call_args.kwargs["json"]["force_instrumental"] is True
|
||||
assert mock_get.call_count == 3
|
||||
|
||||
|
||||
def test_execute_rejects_missing_duration_without_request(monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
with patch("requests.post") as mock_post:
|
||||
result = FalElevenLabsMusic().execute({"prompt": "gentle piano"})
|
||||
|
||||
assert result.success is False
|
||||
assert result.error == "duration_seconds is required"
|
||||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
def test_error_redacts_fal_key(monkeypatch):
|
||||
secret = "test-production-shaped-fal-key"
|
||||
monkeypatch.setenv("FAL_KEY", secret)
|
||||
with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")):
|
||||
result = FalElevenLabsMusic().execute(
|
||||
{"prompt": "gentle piano", "duration_seconds": 20}
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert secret not in result.error
|
||||
assert "[REDACTED]" in result.error
|
||||
106
tests/tools/test_fal_elevenlabs_tts.py
Normal file
106
tests/tools/test_fal_elevenlabs_tts.py
Normal file
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from tools.audio.fal_elevenlabs_tts import FalElevenLabsTTS
|
||||
from tools.base_tool import ToolStatus
|
||||
from tools.tool_registry import ToolRegistry
|
||||
|
||||
|
||||
def _response(*, json_data=None, content=b""):
|
||||
response = MagicMock()
|
||||
response.json.return_value = json_data
|
||||
response.content = content
|
||||
response.raise_for_status.return_value = None
|
||||
return response
|
||||
|
||||
|
||||
def test_contract_models_and_cost(monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
tool = FalElevenLabsTTS()
|
||||
|
||||
assert tool.get_status() == ToolStatus.AVAILABLE
|
||||
assert tool.get_info()["capability"] == "tts"
|
||||
assert tool.get_info()["provider"] == "fal.ai"
|
||||
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "eleven-v3"}) == 0.1
|
||||
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "multilingual-v2"}) == 0.1
|
||||
assert tool.estimate_cost({"text": "a" * 1000, "model_id": "turbo-v2.5"}) == 0.05
|
||||
|
||||
|
||||
def test_registry_discovers_fal_tts(monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
registry = ToolRegistry()
|
||||
registry.discover()
|
||||
|
||||
tool = registry.get("fal_elevenlabs_tts")
|
||||
assert tool is not None
|
||||
assert tool.get_status() == ToolStatus.AVAILABLE
|
||||
|
||||
|
||||
def test_execute_submits_once_and_downloads_audio(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
output_path = tmp_path / "speech.mp3"
|
||||
tool = FalElevenLabsTTS()
|
||||
tool._POLL_INTERVAL_SECONDS = 0
|
||||
|
||||
post_response = _response(
|
||||
json_data={
|
||||
"status_url": "https://queue.example/status",
|
||||
"response_url": "https://queue.example/result",
|
||||
}
|
||||
)
|
||||
status_response = _response(json_data={"status": "COMPLETED"})
|
||||
result_response = _response(
|
||||
json_data={"audio": {"url": "https://media.example/speech.mp3"}}
|
||||
)
|
||||
audio_response = _response(content=b"fake-mp3")
|
||||
|
||||
with (
|
||||
patch("requests.post", return_value=post_response) as mock_post,
|
||||
patch(
|
||||
"requests.get",
|
||||
side_effect=[status_response, result_response, audio_response],
|
||||
) as mock_get,
|
||||
):
|
||||
result = tool.execute(
|
||||
{
|
||||
"text": "A calm, measured test.",
|
||||
"voice_id": "Rachel",
|
||||
"model_id": "eleven-v3",
|
||||
"stability": 0.65,
|
||||
"language_code": "en",
|
||||
"output_path": str(output_path),
|
||||
}
|
||||
)
|
||||
|
||||
assert result.success is True
|
||||
assert result.model == "fal-ai/elevenlabs/tts/eleven-v3"
|
||||
assert output_path.read_bytes() == b"fake-mp3"
|
||||
assert mock_post.call_count == 1
|
||||
assert mock_post.call_args.args[0].endswith("/fal-ai/elevenlabs/tts/eleven-v3")
|
||||
assert mock_post.call_args.kwargs["json"]["voice"] == "Rachel"
|
||||
assert mock_post.call_args.kwargs["json"]["stability"] == 0.65
|
||||
assert mock_get.call_count == 3
|
||||
|
||||
|
||||
def test_invalid_model_does_not_submit(monkeypatch):
|
||||
monkeypatch.setenv("FAL_KEY", "test-key")
|
||||
with patch("requests.post") as mock_post:
|
||||
result = FalElevenLabsTTS().execute(
|
||||
{"text": "hello", "model_id": "not-a-model"}
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert "model_id must be one of" in result.error
|
||||
mock_post.assert_not_called()
|
||||
|
||||
|
||||
def test_error_redacts_fal_key(monkeypatch):
|
||||
secret = "test-production-shaped-fal-key"
|
||||
monkeypatch.setenv("FAL_KEY", secret)
|
||||
with patch("requests.post", side_effect=RuntimeError(f"request used {secret}")):
|
||||
result = FalElevenLabsTTS().execute({"text": "hello"})
|
||||
|
||||
assert result.success is False
|
||||
assert secret not in result.error
|
||||
assert "[REDACTED]" in result.error
|
||||
@@ -34,3 +34,26 @@ def test_tts_key_uses_header_and_is_redacted_from_errors(monkeypatch, tmp_path):
|
||||
assert result.success is False
|
||||
assert secret not in result.error
|
||||
assert "[REDACTED]" in result.error
|
||||
|
||||
|
||||
def test_production_adapter_can_force_google_tts_to_ipv4(monkeypatch, tmp_path):
|
||||
import socket
|
||||
|
||||
import requests
|
||||
import urllib3.util.connection
|
||||
|
||||
monkeypatch.setenv("GOOGLE_TTS_API_KEY", "test-key")
|
||||
monkeypatch.setenv("GOOGLE_TTS_FORCE_IPV4", "1")
|
||||
original = urllib3.util.connection.allowed_gai_family
|
||||
|
||||
def inspect_request(url, **kwargs):
|
||||
assert urllib3.util.connection.allowed_gai_family() == socket.AF_INET
|
||||
raise requests.HTTPError("safe expected failure")
|
||||
|
||||
monkeypatch.setattr(requests, "post", inspect_request)
|
||||
result = GoogleTTS().execute(
|
||||
{"text": "safe test sentence", "output_path": str(tmp_path / "speech.mp3")}
|
||||
)
|
||||
|
||||
assert result.success is False
|
||||
assert urllib3.util.connection.allowed_gai_family is original
|
||||
|
||||
@@ -36,7 +36,9 @@ class ElevenLabsTTS(BaseTool):
|
||||
install_instructions = (
|
||||
"Set the ELEVENLABS_API_KEY environment variable:\n"
|
||||
" export ELEVENLABS_API_KEY=your_key_here\n"
|
||||
"Get a key at https://elevenlabs.io"
|
||||
"Get a key at https://elevenlabs.io\n"
|
||||
"If fal_elevenlabs_tts is available, use it instead to access ElevenLabs "
|
||||
"speech through fal.ai without a separate ElevenLabs key."
|
||||
)
|
||||
fallback = "openai_tts"
|
||||
fallback_tools = ["openai_tts", "piper_tts"]
|
||||
|
||||
231
tools/audio/fal_elevenlabs_music.py
Normal file
231
tools/audio/fal_elevenlabs_music.py
Normal file
@@ -0,0 +1,231 @@
|
||||
"""Generate music with ElevenLabs Music through fal.ai.
|
||||
|
||||
This provider uses OpenMontage's shared ``FAL_KEY`` credential and the fal.ai
|
||||
queue API, then downloads the generated MP3 to a project-local path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolStatus,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class FalElevenLabsMusic(BaseTool):
|
||||
"""Generate a precisely timed music track through fal.ai."""
|
||||
|
||||
name = "fal_elevenlabs_music"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.GENERATE
|
||||
capability = "music_generation"
|
||||
provider = "fal.ai"
|
||||
stability = ToolStability.BETA
|
||||
execution_mode = ExecutionMode.ASYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:FAL_KEY"]
|
||||
install_instructions = (
|
||||
"Set FAL_KEY to a fal.ai API key. "
|
||||
"Get one at https://fal.ai/dashboard/keys"
|
||||
)
|
||||
fallback_tools = ["music_gen", "google_music"]
|
||||
agent_skills = ["music", "elevenlabs"]
|
||||
|
||||
capabilities = [
|
||||
"generate_background_music",
|
||||
"generate_instrumental",
|
||||
]
|
||||
supports = {
|
||||
"instrumental": True,
|
||||
"style_control": True,
|
||||
"exact_duration": True,
|
||||
}
|
||||
best_for = [
|
||||
"precisely timed instrumental background music",
|
||||
"high-quality ElevenLabs Music through a shared fal.ai account",
|
||||
"short-form social video soundtracks",
|
||||
]
|
||||
not_good_for = [
|
||||
"offline generation",
|
||||
"free music sourcing",
|
||||
"sub-3-second sound effects",
|
||||
]
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["prompt", "duration_seconds"],
|
||||
"properties": {
|
||||
"prompt": {
|
||||
"type": "string",
|
||||
"description": "Music description including mood, style, and instruments",
|
||||
},
|
||||
"duration_seconds": {
|
||||
"type": "number",
|
||||
"minimum": 3,
|
||||
"maximum": 600,
|
||||
"description": "Exact target length in seconds",
|
||||
},
|
||||
"force_instrumental": {
|
||||
"type": "boolean",
|
||||
"default": True,
|
||||
"description": "Generate music without vocals",
|
||||
},
|
||||
"output_path": {
|
||||
"type": "string",
|
||||
"default": "fal_music_output.mp3",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1,
|
||||
ram_mb=256,
|
||||
vram_mb=0,
|
||||
disk_mb=50,
|
||||
network_required=True,
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=0,
|
||||
retryable_errors=["rate_limit", "timeout"],
|
||||
)
|
||||
idempotency_key_fields = ["prompt", "duration_seconds", "force_instrumental"]
|
||||
side_effects = [
|
||||
"writes an MP3 file to output_path",
|
||||
"submits one paid fal.ai generation request",
|
||||
]
|
||||
user_visible_verification = [
|
||||
"Listen to the generated music for mood, mix, and duration",
|
||||
]
|
||||
|
||||
_MODEL = "fal-ai/elevenlabs/music"
|
||||
_QUEUE_URL = f"https://queue.fal.run/{_MODEL}"
|
||||
_POLL_INTERVAL_SECONDS = 5
|
||||
_MAX_WAIT_SECONDS = 600
|
||||
|
||||
def _get_api_key(self) -> str | None:
|
||||
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
|
||||
|
||||
def get_status(self) -> ToolStatus:
|
||||
return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
"""fal bills $0.80 per output minute, rounded up to a full minute."""
|
||||
duration = inputs.get("duration_seconds")
|
||||
if duration is None:
|
||||
raise ValueError("duration_seconds is required for cost estimation")
|
||||
return round(math.ceil(float(duration) / 60.0) * 0.80, 2)
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
api_key = self._get_api_key()
|
||||
if not api_key:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="No fal.ai API key found. " + self.install_instructions,
|
||||
)
|
||||
|
||||
duration = inputs.get("duration_seconds")
|
||||
if duration is None:
|
||||
return ToolResult(success=False, error="duration_seconds is required")
|
||||
duration = float(duration)
|
||||
if not 3 <= duration <= 600:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="duration_seconds must be between 3 and 600",
|
||||
)
|
||||
|
||||
import requests
|
||||
|
||||
started = time.time()
|
||||
headers = {
|
||||
"Authorization": f"Key {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
payload = {
|
||||
"prompt": inputs["prompt"],
|
||||
"music_length_ms": round(duration * 1000),
|
||||
"force_instrumental": bool(inputs.get("force_instrumental", True)),
|
||||
}
|
||||
|
||||
try:
|
||||
submit_response = requests.post(
|
||||
self._QUEUE_URL,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
submit_response.raise_for_status()
|
||||
queue_data = submit_response.json()
|
||||
status_url = queue_data["status_url"]
|
||||
response_url = queue_data["response_url"]
|
||||
|
||||
deadline = time.monotonic() + self._MAX_WAIT_SECONDS
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="fal.ai music generation timed out while waiting in the queue",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
time.sleep(self._POLL_INTERVAL_SECONDS)
|
||||
status_response = requests.get(status_url, headers=headers, timeout=20)
|
||||
status_response.raise_for_status()
|
||||
status = status_response.json().get("status", "UNKNOWN")
|
||||
if status == "COMPLETED":
|
||||
break
|
||||
if status in {"FAILED", "CANCELLED"}:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"fal.ai music generation {status.lower()}",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
|
||||
result_response = requests.get(response_url, headers=headers, timeout=30)
|
||||
result_response.raise_for_status()
|
||||
result_data = result_response.json()
|
||||
audio_url = result_data["audio"]["url"]
|
||||
|
||||
audio_response = requests.get(audio_url, timeout=120)
|
||||
audio_response.raise_for_status()
|
||||
output_path = Path(inputs.get("output_path", "fal_music_output.mp3"))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(audio_response.content)
|
||||
except Exception as exc:
|
||||
safe_error = str(exc).replace(api_key, "[REDACTED]")
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"fal.ai ElevenLabs Music generation failed: {safe_error}",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data={
|
||||
"provider": "fal.ai",
|
||||
"model": self._MODEL,
|
||||
"prompt": inputs["prompt"],
|
||||
"duration_seconds": duration,
|
||||
"force_instrumental": payload["force_instrumental"],
|
||||
"output": str(output_path),
|
||||
"format": "mp3",
|
||||
},
|
||||
artifacts=[str(output_path)],
|
||||
cost_usd=self.estimate_cost(inputs),
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
model=self._MODEL,
|
||||
)
|
||||
359
tools/audio/fal_elevenlabs_tts.py
Normal file
359
tools/audio/fal_elevenlabs_tts.py
Normal file
@@ -0,0 +1,359 @@
|
||||
"""Generate ElevenLabs speech through fal.ai using the shared FAL credential."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from tools.base_tool import (
|
||||
BaseTool,
|
||||
Determinism,
|
||||
ExecutionMode,
|
||||
ResourceProfile,
|
||||
RetryPolicy,
|
||||
ToolResult,
|
||||
ToolRuntime,
|
||||
ToolStability,
|
||||
ToolStatus,
|
||||
ToolTier,
|
||||
)
|
||||
|
||||
|
||||
class FalElevenLabsTTS(BaseTool):
|
||||
"""Generate expressive narration with ElevenLabs models hosted by fal.ai."""
|
||||
|
||||
name = "fal_elevenlabs_tts"
|
||||
version = "0.1.0"
|
||||
tier = ToolTier.VOICE
|
||||
capability = "tts"
|
||||
provider = "fal.ai"
|
||||
stability = ToolStability.BETA
|
||||
execution_mode = ExecutionMode.ASYNC
|
||||
determinism = Determinism.STOCHASTIC
|
||||
runtime = ToolRuntime.API
|
||||
|
||||
dependencies = ["env:FAL_KEY"]
|
||||
install_instructions = (
|
||||
"Set FAL_KEY to a fal.ai API key. No separate ElevenLabs key is needed. "
|
||||
"Get a fal.ai key at https://fal.ai/dashboard/keys"
|
||||
)
|
||||
fallback_tools = ["google_tts", "piper_tts", "elevenlabs_tts"]
|
||||
agent_skills = ["elevenlabs"]
|
||||
|
||||
capabilities = [
|
||||
"text_to_speech",
|
||||
"voice_selection",
|
||||
"expressive_delivery",
|
||||
"multilingual",
|
||||
"word_timestamps",
|
||||
]
|
||||
supports = {
|
||||
"voice_cloning": False,
|
||||
"multilingual": True,
|
||||
"offline": False,
|
||||
"native_audio": True,
|
||||
"inline_audio_tags": True,
|
||||
"word_timestamps": True,
|
||||
}
|
||||
best_for = [
|
||||
"expressive ElevenLabs narration through an existing fal.ai connection",
|
||||
"emotionally directed voiceover with Eleven v3 audio tags",
|
||||
"multilingual narration without a separate ElevenLabs credential",
|
||||
]
|
||||
not_good_for = [
|
||||
"offline generation",
|
||||
"voice cloning or private custom ElevenLabs voices",
|
||||
]
|
||||
|
||||
_MODELS = {
|
||||
"eleven-v3": "fal-ai/elevenlabs/tts/eleven-v3",
|
||||
"multilingual-v2": "fal-ai/elevenlabs/tts/multilingual-v2",
|
||||
"turbo-v2.5": "fal-ai/elevenlabs/tts/turbo-v2.5",
|
||||
}
|
||||
_MODEL_ALIASES = {
|
||||
"eleven_v3": "eleven-v3",
|
||||
"eleven_multilingual_v2": "multilingual-v2",
|
||||
"multilingual_v2": "multilingual-v2",
|
||||
"eleven_turbo_v2_5": "turbo-v2.5",
|
||||
"turbo_v2_5": "turbo-v2.5",
|
||||
**{value: key for key, value in _MODELS.items()},
|
||||
}
|
||||
_PRICE_PER_CHARACTER = {
|
||||
"eleven-v3": 0.0001,
|
||||
"multilingual-v2": 0.0001,
|
||||
"turbo-v2.5": 0.00005,
|
||||
}
|
||||
_POLL_INTERVAL_SECONDS = 2
|
||||
_MAX_WAIT_SECONDS = 300
|
||||
|
||||
input_schema = {
|
||||
"type": "object",
|
||||
"required": ["text"],
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string",
|
||||
"description": "Text to speak. Eleven v3 supports inline tags such as [whispers].",
|
||||
},
|
||||
"voice": {
|
||||
"type": "string",
|
||||
"default": "Rachel",
|
||||
"description": "fal.ai ElevenLabs voice name or ID",
|
||||
},
|
||||
"voice_id": {
|
||||
"type": "string",
|
||||
"description": "Alias for voice, for compatibility with tts_selector",
|
||||
},
|
||||
"model_id": {
|
||||
"type": "string",
|
||||
"default": "eleven-v3",
|
||||
"description": "eleven-v3, multilingual-v2, or turbo-v2.5",
|
||||
},
|
||||
"stability": {
|
||||
"type": "number",
|
||||
"default": 0.5,
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
},
|
||||
"similarity_boost": {
|
||||
"type": "number",
|
||||
"default": 0.75,
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
},
|
||||
"style": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1,
|
||||
},
|
||||
"speed": {
|
||||
"type": "number",
|
||||
"default": 1.0,
|
||||
"minimum": 0.7,
|
||||
"maximum": 1.2,
|
||||
},
|
||||
"language_code": {
|
||||
"type": "string",
|
||||
"description": "Optional ISO 639-1 language code",
|
||||
},
|
||||
"timestamps": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
},
|
||||
"apply_text_normalization": {
|
||||
"type": "string",
|
||||
"default": "auto",
|
||||
"enum": ["auto", "on", "off"],
|
||||
},
|
||||
"output_format": {
|
||||
"type": "string",
|
||||
"default": "mp3_44100_128",
|
||||
"enum": [
|
||||
"mp3_22050_32",
|
||||
"mp3_44100_64",
|
||||
"mp3_44100_96",
|
||||
"mp3_44100_128",
|
||||
"mp3_44100_192",
|
||||
"pcm_16000",
|
||||
"pcm_24000",
|
||||
"pcm_44100",
|
||||
"pcm_48000",
|
||||
"opus_48000_64",
|
||||
"opus_48000_96",
|
||||
"opus_48000_128",
|
||||
"opus_48000_192",
|
||||
],
|
||||
},
|
||||
"seed": {"type": "integer"},
|
||||
"output_path": {"type": "string"},
|
||||
},
|
||||
}
|
||||
|
||||
resource_profile = ResourceProfile(
|
||||
cpu_cores=1,
|
||||
ram_mb=256,
|
||||
vram_mb=0,
|
||||
disk_mb=50,
|
||||
network_required=True,
|
||||
)
|
||||
retry_policy = RetryPolicy(
|
||||
max_retries=0,
|
||||
retryable_errors=["rate_limit", "timeout"],
|
||||
)
|
||||
idempotency_key_fields = [
|
||||
"text",
|
||||
"voice",
|
||||
"voice_id",
|
||||
"model_id",
|
||||
"stability",
|
||||
"similarity_boost",
|
||||
"style",
|
||||
"speed",
|
||||
"language_code",
|
||||
"seed",
|
||||
]
|
||||
side_effects = [
|
||||
"writes an audio file to output_path",
|
||||
"submits one paid fal.ai ElevenLabs speech request",
|
||||
]
|
||||
user_visible_verification = [
|
||||
"Listen to the generated voice sample before approving full narration",
|
||||
]
|
||||
|
||||
def _get_api_key(self) -> str | None:
|
||||
return os.environ.get("FAL_KEY") or os.environ.get("FAL_AI_API_KEY")
|
||||
|
||||
def get_status(self) -> ToolStatus:
|
||||
return ToolStatus.AVAILABLE if self._get_api_key() else ToolStatus.UNAVAILABLE
|
||||
|
||||
def _resolve_model(self, requested: str | None) -> tuple[str, str]:
|
||||
model_name = requested or "eleven-v3"
|
||||
model_name = self._MODEL_ALIASES.get(model_name, model_name)
|
||||
if model_name not in self._MODELS:
|
||||
choices = ", ".join(self._MODELS)
|
||||
raise ValueError(f"model_id must be one of: {choices}")
|
||||
return model_name, self._MODELS[model_name]
|
||||
|
||||
def estimate_cost(self, inputs: dict[str, Any]) -> float:
|
||||
model_name, _ = self._resolve_model(inputs.get("model_id"))
|
||||
return round(
|
||||
len(inputs.get("text", "")) * self._PRICE_PER_CHARACTER[model_name],
|
||||
4,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _output_extension(output_format: str) -> str:
|
||||
return {
|
||||
"mp3": "mp3",
|
||||
"pcm": "pcm",
|
||||
"opus": "opus",
|
||||
}.get(output_format.split("_", 1)[0], "audio")
|
||||
|
||||
def execute(self, inputs: dict[str, Any]) -> ToolResult:
|
||||
api_key = self._get_api_key()
|
||||
if not api_key:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="No fal.ai API key found. " + self.install_instructions,
|
||||
)
|
||||
|
||||
text = str(inputs.get("text", "")).strip()
|
||||
if not text:
|
||||
return ToolResult(success=False, error="text is required")
|
||||
|
||||
try:
|
||||
model_name, model_id = self._resolve_model(inputs.get("model_id"))
|
||||
except ValueError as exc:
|
||||
return ToolResult(success=False, error=str(exc))
|
||||
|
||||
stability = float(inputs.get("stability", 0.5))
|
||||
similarity_boost = float(inputs.get("similarity_boost", 0.75))
|
||||
speed = float(inputs.get("speed", 1.0))
|
||||
if not 0 <= stability <= 1 or not 0 <= similarity_boost <= 1:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="stability and similarity_boost must be between 0 and 1",
|
||||
)
|
||||
if not 0.7 <= speed <= 1.2:
|
||||
return ToolResult(success=False, error="speed must be between 0.7 and 1.2")
|
||||
|
||||
output_format = inputs.get("output_format", "mp3_44100_128")
|
||||
voice = inputs.get("voice") or inputs.get("voice_id") or "Rachel"
|
||||
payload: dict[str, Any] = {
|
||||
"text": text,
|
||||
"voice": voice,
|
||||
"stability": stability,
|
||||
"similarity_boost": similarity_boost,
|
||||
"speed": speed,
|
||||
"timestamps": bool(inputs.get("timestamps", False)),
|
||||
"apply_text_normalization": inputs.get("apply_text_normalization", "auto"),
|
||||
"output_format": output_format,
|
||||
}
|
||||
for optional in ("language_code", "seed", "style"):
|
||||
if inputs.get(optional) is not None:
|
||||
payload[optional] = inputs[optional]
|
||||
|
||||
import requests
|
||||
|
||||
started = time.time()
|
||||
headers = {
|
||||
"Authorization": f"Key {api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
queue_url = f"https://queue.fal.run/{model_id}"
|
||||
|
||||
try:
|
||||
submit_response = requests.post(
|
||||
queue_url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=30,
|
||||
)
|
||||
submit_response.raise_for_status()
|
||||
queue_data = submit_response.json()
|
||||
status_url = queue_data["status_url"]
|
||||
response_url = queue_data["response_url"]
|
||||
|
||||
deadline = time.monotonic() + self._MAX_WAIT_SECONDS
|
||||
while True:
|
||||
if time.monotonic() >= deadline:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error="fal.ai ElevenLabs speech timed out while waiting in the queue",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
time.sleep(self._POLL_INTERVAL_SECONDS)
|
||||
status_response = requests.get(status_url, headers=headers, timeout=20)
|
||||
status_response.raise_for_status()
|
||||
status = status_response.json().get("status", "UNKNOWN")
|
||||
if status == "COMPLETED":
|
||||
break
|
||||
if status in {"FAILED", "CANCELLED"}:
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"fal.ai ElevenLabs speech {status.lower()}",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
|
||||
result_response = requests.get(response_url, headers=headers, timeout=30)
|
||||
result_response.raise_for_status()
|
||||
result_data = result_response.json()
|
||||
audio_url = result_data["audio"]["url"]
|
||||
|
||||
audio_response = requests.get(audio_url, timeout=120)
|
||||
audio_response.raise_for_status()
|
||||
default_output = f"fal_elevenlabs_tts.{self._output_extension(output_format)}"
|
||||
output_path = Path(inputs.get("output_path", default_output))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_bytes(audio_response.content)
|
||||
except Exception as exc:
|
||||
safe_error = str(exc).replace(api_key, "[REDACTED]")
|
||||
return ToolResult(
|
||||
success=False,
|
||||
error=f"fal.ai ElevenLabs speech failed: {safe_error}",
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
)
|
||||
|
||||
data = {
|
||||
"provider": self.provider,
|
||||
"model": model_id,
|
||||
"voice": voice,
|
||||
"text_length": len(text),
|
||||
"stability": stability,
|
||||
"similarity_boost": similarity_boost,
|
||||
"speed": speed,
|
||||
"output": str(output_path),
|
||||
"format": output_format,
|
||||
}
|
||||
if payload["timestamps"] and "timestamps" in result_data:
|
||||
data["timestamps"] = result_data["timestamps"]
|
||||
|
||||
return ToolResult(
|
||||
success=True,
|
||||
data=data,
|
||||
artifacts=[str(output_path)],
|
||||
cost_usd=self.estimate_cost({**inputs, "model_id": model_name, "text": text}),
|
||||
duration_seconds=round(time.time() - started, 2),
|
||||
model=model_id,
|
||||
)
|
||||
@@ -8,7 +8,10 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -30,6 +33,32 @@ from tools.google_credentials import (
|
||||
)
|
||||
|
||||
|
||||
_IPV4_REQUEST_LOCK = threading.Lock()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _google_tts_network_family():
|
||||
"""Use IPv4 when the deployment key is restricted to the server's IPv4."""
|
||||
force_ipv4 = os.environ.get("GOOGLE_TTS_FORCE_IPV4", "").lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
}
|
||||
if not force_ipv4:
|
||||
yield
|
||||
return
|
||||
|
||||
import urllib3.util.connection
|
||||
|
||||
with _IPV4_REQUEST_LOCK:
|
||||
original = urllib3.util.connection.allowed_gai_family
|
||||
urllib3.util.connection.allowed_gai_family = lambda: socket.AF_INET
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
urllib3.util.connection.allowed_gai_family = original
|
||||
|
||||
|
||||
class GoogleTTS(BaseTool):
|
||||
name = "google_tts"
|
||||
version = "0.1.0"
|
||||
@@ -279,12 +308,13 @@ class GoogleTTS(BaseTool):
|
||||
elif api_key:
|
||||
headers["x-goog-api-key"] = api_key
|
||||
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=120,
|
||||
)
|
||||
with _google_tts_network_family():
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
json=payload,
|
||||
timeout=120,
|
||||
)
|
||||
response.raise_for_status()
|
||||
|
||||
audio_content = base64.b64decode(response.json()["audioContent"])
|
||||
|
||||
@@ -45,6 +45,10 @@ class TTSSelector(BaseTool):
|
||||
"type": "string",
|
||||
"description": "Provider-specific voice ID. Passed through to the selected TTS provider.",
|
||||
},
|
||||
"voice": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific voice name or ID. fal.ai ElevenLabs accepts names such as Rachel.",
|
||||
},
|
||||
"voice_language": {
|
||||
"type": "string",
|
||||
"enum": ["zh", "en"],
|
||||
@@ -58,7 +62,7 @@ class TTSSelector(BaseTool):
|
||||
},
|
||||
"model_id": {
|
||||
"type": "string",
|
||||
"description": "TTS model to use (e.g. eleven_multilingual_v2). Passed through to provider.",
|
||||
"description": "TTS model to use (e.g. eleven-v3 or eleven_multilingual_v2). Passed through to provider.",
|
||||
},
|
||||
"stability": {
|
||||
"type": "number", "minimum": 0, "maximum": 1,
|
||||
@@ -100,6 +104,24 @@ class TTSSelector(BaseTool):
|
||||
"default": "text",
|
||||
"description": "Use 'ssml' only when the selected provider supports tags such as <break>.",
|
||||
},
|
||||
"language_code": {
|
||||
"type": "string",
|
||||
"description": "Provider-specific language code, such as en-US for Google or en for fal.ai ElevenLabs.",
|
||||
},
|
||||
"timestamps": {
|
||||
"type": "boolean",
|
||||
"default": False,
|
||||
"description": "Request word timestamps when the selected provider supports them.",
|
||||
},
|
||||
"apply_text_normalization": {
|
||||
"type": "string",
|
||||
"enum": ["auto", "on", "off"],
|
||||
"description": "Text normalization mode for providers that support it.",
|
||||
},
|
||||
"seed": {
|
||||
"type": "integer",
|
||||
"description": "Optional generation seed for providers that support reproducible speech.",
|
||||
},
|
||||
"voice_performance": {
|
||||
"type": "object",
|
||||
"description": "Structured voice-performance plan or section delivery cues from the script artifact.",
|
||||
|
||||
Reference in New Issue
Block a user