Module livekit.plugins.fishaudio
Fish Audio plugin for LiveKit Agents
See https://docs.fish.audio for more information.
Environment variables used:
- FISH_API_KEY for authentication (required)
Classes
class TTS (*,
api_key: NotGivenOr[str] = NOT_GIVEN,
model: TTSModels | str = 's2.1-pro',
voice_id: NotGivenOr[str] = '933563129e564b19a115bedd57b7406a',
output_format: OutputFormat = 'wav',
sample_rate: NotGivenOr[int] = NOT_GIVEN,
base_url: NotGivenOr[str] = NOT_GIVEN,
latency_mode: LatencyMode = 'balanced',
chunk_length: int = 100,
speed: NotGivenOr[float] = NOT_GIVEN,
volume: NotGivenOr[float] = NOT_GIVEN,
temperature: float = 0.7,
top_p: float = 0.7,
mp3_bitrate: MP3Bitrate = 64,
opus_bitrate: OpusBitrate = 64000,
normalize: bool = True,
normalize_loudness: NotGivenOr[bool] = NOT_GIVEN,
max_new_tokens: NotGivenOr[int] = NOT_GIVEN,
min_chunk_length: NotGivenOr[int] = NOT_GIVEN,
condition_on_previous_chunks: NotGivenOr[bool] = NOT_GIVEN,
early_stop_threshold: NotGivenOr[float] = NOT_GIVEN,
tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN,
http_session: aiohttp.ClientSession | None = None)-
Expand source code
class TTS(tts.TTS): def __init__( self, *, api_key: NotGivenOr[str] = NOT_GIVEN, model: TTSModels | str = DEFAULT_MODEL, voice_id: NotGivenOr[str] = DEFAULT_VOICE_ID, output_format: OutputFormat = "wav", sample_rate: NotGivenOr[int] = NOT_GIVEN, base_url: NotGivenOr[str] = NOT_GIVEN, latency_mode: LatencyMode = "balanced", chunk_length: int = 100, speed: NotGivenOr[float] = NOT_GIVEN, volume: NotGivenOr[float] = NOT_GIVEN, temperature: float = 0.7, top_p: float = 0.7, mp3_bitrate: MP3Bitrate = 64, opus_bitrate: OpusBitrate = 64000, normalize: bool = True, normalize_loudness: NotGivenOr[bool] = NOT_GIVEN, max_new_tokens: NotGivenOr[int] = NOT_GIVEN, min_chunk_length: NotGivenOr[int] = NOT_GIVEN, condition_on_previous_chunks: NotGivenOr[bool] = NOT_GIVEN, early_stop_threshold: NotGivenOr[float] = NOT_GIVEN, tokenizer: NotGivenOr[tokenize.SentenceTokenizer] = NOT_GIVEN, http_session: aiohttp.ClientSession | None = None, ) -> None: """ Create a new instance of Fish Audio TTS. See https://docs.fish.audio/api-reference/endpoint/websocket/tts-live for more details on the Fish Audio Live TTS WebSocket API. Args: api_key (NotGivenOr[str]): Fish Audio API key. Reads ``FISH_API_KEY`` if unset. model (TTSModels | str): TTS model to use. Defaults to ``"s2.1-pro"``. voice_id (NotGivenOr[str]): Voice model ID. Fish Audio's API refers to this as ``reference_id``; it's the same value either way. output_format (OutputFormat): Audio output format. Defaults to ``"wav"``. sample_rate (int): Audio sample rate in Hz. base_url (NotGivenOr[str]): Custom base URL. Defaults to ``https://api.fish.audio``. latency_mode (LatencyMode): Streaming latency mode. ``"normal"``, ``"balanced"``, or ``"low"``. Defaults to ``"balanced"``. chunk_length (int): Upper bound on text Fish buffers before auto-synthesizing (100–300). With sentence-level flushing this is only hit by sentences longer than ``chunk_length``; otherwise audio is produced when each sentence is flushed. Defaults to 100. speed (NotGivenOr[float]): Speaking rate multiplier (Fish ``prosody.speed``). ``1.0`` is normal; below 1.0 is slower, above is faster. Unset uses the voice's natural pace. volume (NotGivenOr[float]): Loudness adjustment in decibels (Fish ``prosody.volume``). ``0`` is the voice's natural level. Unset leaves it unchanged. temperature (float): Sampling temperature (0–1). Higher values produce more varied, expressive speech; lower values are more stable. Defaults to 0.7. top_p (float): Nucleus sampling probability mass (0–1). Defaults to 0.7. mp3_bitrate (MP3Bitrate): MP3 bitrate in kbps: 64, 128, or 192. Only used when ``output_format`` is ``"mp3"``. Defaults to 64. opus_bitrate (OpusBitrate): Opus bitrate in bps: -1000 (auto), 24000, 32000, 48000, or 64000. Only used when ``output_format`` is ``"opus"``. Defaults to 64000. normalize (bool): Whether Fish normalizes the input text (numbers, dates, abbreviations) before synthesis. Defaults to True. normalize_loudness (NotGivenOr[bool]): Whether Fish normalizes the output loudness for more consistent volume (Fish ``prosody.normalize_loudness``). S2-Pro family only: on the ``s1`` model the option is dropped with a warning. Unset uses Fish's server default (True). max_new_tokens (NotGivenOr[int]): Maximum audio tokens Fish generates per text chunk. Unset uses Fish's server default (1024). min_chunk_length (NotGivenOr[int]): Minimum characters before Fish splits text into a new chunk (0-100). Unset uses Fish's server default (50). condition_on_previous_chunks (NotGivenOr[bool]): Whether Fish uses previous audio as context for voice consistency across chunks. Unset uses Fish's server default (True). early_stop_threshold (NotGivenOr[float]): Early stopping threshold for batch processing (0-1). Unset uses Fish's server default (1.0). tokenizer (tokenize.SentenceTokenizer): Sentence tokenizer used to detect sentence boundaries. Defaults to ``tokenize.blingfire.SentenceTokenizer()``. http_session (aiohttp.ClientSession | None): Optional aiohttp session. """ if is_given(sample_rate): if output_format == "opus" and sample_rate != 48000: raise ValueError( "Fish Audio only supports 48000 Hz for opus output; " f"got sample_rate={sample_rate}" ) resolved_sample_rate = sample_rate else: resolved_sample_rate = _DEFAULT_SAMPLE_RATE[output_format] super().__init__( capabilities=tts.TTSCapabilities(streaming=True), sample_rate=resolved_sample_rate, num_channels=NUM_CHANNELS, ) fish_api_key = api_key if is_given(api_key) else os.getenv("FISH_API_KEY") if not fish_api_key: raise ValueError( "Fish Audio API key is required, either as argument or set " "FISH_API_KEY environment variable" ) if not 100 <= chunk_length <= 300: raise ValueError("chunk_length must be between 100 and 300") if not 0 <= temperature <= 1: raise ValueError("temperature must be between 0 and 1") if not 0 <= top_p <= 1: raise ValueError("top_p must be between 0 and 1") if is_given(max_new_tokens) and max_new_tokens < 0: raise ValueError("max_new_tokens must be non-negative") if is_given(min_chunk_length) and not 0 <= min_chunk_length <= 100: raise ValueError("min_chunk_length must be between 0 and 100") if is_given(early_stop_threshold) and not 0 <= early_stop_threshold <= 1: raise ValueError("early_stop_threshold must be between 0 and 1") if is_given(normalize_loudness) and model == "s1": logger.warning("normalize_loudness is not supported by the s1 model, dropping") normalize_loudness = NOT_GIVEN self._opts = _TTSOptions( model=model, output_format=output_format, sample_rate=resolved_sample_rate, voice_id=voice_id, base_url=base_url if is_given(base_url) else DEFAULT_BASE_URL, api_key=fish_api_key, latency_mode=latency_mode, chunk_length=chunk_length, speed=speed, volume=volume, temperature=temperature, top_p=top_p, mp3_bitrate=mp3_bitrate, opus_bitrate=opus_bitrate, normalize=normalize, normalize_loudness=normalize_loudness, max_new_tokens=max_new_tokens, min_chunk_length=min_chunk_length, condition_on_previous_chunks=condition_on_previous_chunks, early_stop_threshold=early_stop_threshold, ) self._session = http_session self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse]( connect_cb=self._connect_ws, close_cb=self._close_ws, max_session_duration=300, mark_refreshed_on_get=True, ) # min_sentence_len=1 emits each sentence as soon as the next one starts, # rather than batching short sentences together — minimizes TTFB on the # first sentence and keeps Fish synthesizing continuously. self._sentence_tokenizer = ( tokenizer if is_given(tokenizer) else tokenize.blingfire.SentenceTokenizer(min_sentence_len=1) ) self._streams = weakref.WeakSet[SynthesizeStream]() @property def model(self) -> TTSModels | str: return self._opts.model @property def provider(self) -> str: return "FishAudio" @property def output_format(self) -> OutputFormat: return self._opts.output_format @property def voice_id(self) -> NotGivenOr[str]: return self._opts.voice_id @property def latency_mode(self) -> LatencyMode: return self._opts.latency_mode def _ensure_session(self) -> aiohttp.ClientSession: if not self._session: self._session = utils.http_context.http_session() return self._session async def _connect_ws(self, timeout: float) -> aiohttp.ClientWebSocketResponse: session = self._ensure_session() return await asyncio.wait_for( session.ws_connect( self._opts.get_ws_url("/v1/tts/live"), headers={ "Authorization": f"Bearer {self._opts.api_key}", "User-Agent": USER_AGENT, "model": self._opts.model, }, heartbeat=30.0, ), timeout, ) async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse) -> None: await ws.close() def prewarm(self) -> None: self._pool.prewarm() def update_options( self, *, model: NotGivenOr[TTSModels | str] = NOT_GIVEN, voice_id: NotGivenOr[str] = NOT_GIVEN, latency_mode: NotGivenOr[LatencyMode] = NOT_GIVEN, chunk_length: NotGivenOr[int] = NOT_GIVEN, speed: NotGivenOr[float] = NOT_GIVEN, volume: NotGivenOr[float] = NOT_GIVEN, temperature: NotGivenOr[float] = NOT_GIVEN, top_p: NotGivenOr[float] = NOT_GIVEN, mp3_bitrate: NotGivenOr[MP3Bitrate] = NOT_GIVEN, opus_bitrate: NotGivenOr[OpusBitrate] = NOT_GIVEN, normalize: NotGivenOr[bool] = NOT_GIVEN, normalize_loudness: NotGivenOr[bool] = NOT_GIVEN, max_new_tokens: NotGivenOr[int] = NOT_GIVEN, min_chunk_length: NotGivenOr[int] = NOT_GIVEN, condition_on_previous_chunks: NotGivenOr[bool] = NOT_GIVEN, early_stop_threshold: NotGivenOr[float] = NOT_GIVEN, ) -> None: if is_given(model) and model != self._opts.model: self._opts.model = model # The model is sent as a connection header at ws-handshake time, not in the # per-request body, so a pooled socket keeps the old model. Drop pooled # connections so the next stream reconnects with the new model. Other # options ride in the per-request body and need no reconnect. self._pool.invalidate() if model == "s1" and is_given(self._opts.normalize_loudness): logger.warning("normalize_loudness is not supported by the s1 model, dropping") self._opts.normalize_loudness = NOT_GIVEN if is_given(voice_id): self._opts.voice_id = voice_id if is_given(latency_mode): self._opts.latency_mode = latency_mode if is_given(chunk_length): if not 100 <= chunk_length <= 300: raise ValueError("chunk_length must be between 100 and 300") self._opts.chunk_length = chunk_length if is_given(speed): self._opts.speed = speed if is_given(volume): self._opts.volume = volume if is_given(temperature): if not 0 <= temperature <= 1: raise ValueError("temperature must be between 0 and 1") self._opts.temperature = temperature if is_given(top_p): if not 0 <= top_p <= 1: raise ValueError("top_p must be between 0 and 1") self._opts.top_p = top_p if is_given(mp3_bitrate): self._opts.mp3_bitrate = mp3_bitrate if is_given(opus_bitrate): self._opts.opus_bitrate = opus_bitrate if is_given(normalize): self._opts.normalize = normalize if is_given(normalize_loudness): if self._opts.model == "s1": logger.warning("normalize_loudness is not supported by the s1 model, dropping") else: self._opts.normalize_loudness = normalize_loudness if is_given(max_new_tokens): if max_new_tokens < 0: raise ValueError("max_new_tokens must be non-negative") self._opts.max_new_tokens = max_new_tokens if is_given(min_chunk_length): if not 0 <= min_chunk_length <= 100: raise ValueError("min_chunk_length must be between 0 and 100") self._opts.min_chunk_length = min_chunk_length if is_given(condition_on_previous_chunks): self._opts.condition_on_previous_chunks = condition_on_previous_chunks if is_given(early_stop_threshold): if not 0 <= early_stop_threshold <= 1: raise ValueError("early_stop_threshold must be between 0 and 1") self._opts.early_stop_threshold = early_stop_threshold def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def stream( self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> SynthesizeStream: stream = SynthesizeStream(tts=self, conn_options=conn_options) self._streams.add(stream) return stream async def aclose(self) -> None: for stream in list(self._streams): await stream.aclose() self._streams.clear() await self._pool.aclose()Helper class that provides a standard way to create an ABC using inheritance.
Create a new instance of Fish Audio TTS.
See https://docs.fish.audio/api-reference/endpoint/websocket/tts-live for more details on the Fish Audio Live TTS WebSocket API.
Args
api_key:NotGivenOr[str]- Fish Audio API key. Reads
FISH_API_KEYif unset. model:TTSModels | str- TTS model to use. Defaults to
"s2.1-pro". voice_id:NotGivenOr[str]- Voice model ID. Fish Audio's API refers to this
as
reference_id; it's the same value either way. output_format:OutputFormat- Audio output format. Defaults to
"wav". sample_rate:int- Audio sample rate in Hz.
base_url:NotGivenOr[str]- Custom base URL. Defaults to
https://api.fish.audio. latency_mode:LatencyMode- Streaming latency mode.
"normal","balanced", or"low". Defaults to"balanced". chunk_length:int- Upper bound on text Fish buffers before auto-synthesizing
(100–300). With sentence-level flushing this is only hit by sentences longer
than
chunk_length; otherwise audio is produced when each sentence is flushed. Defaults to 100. speed:NotGivenOr[float]- Speaking rate multiplier (Fish
prosody.speed).1.0is normal; below 1.0 is slower, above is faster. Unset uses the voice's natural pace. volume:NotGivenOr[float]- Loudness adjustment in decibels (Fish
prosody.volume).0is the voice's natural level. Unset leaves it unchanged. temperature:float- Sampling temperature (0–1). Higher values produce more varied, expressive speech; lower values are more stable. Defaults to 0.7.
top_p:float- Nucleus sampling probability mass (0–1). Defaults to 0.7.
mp3_bitrate:MP3Bitrate- MP3 bitrate in kbps: 64, 128, or 192. Only used
when
output_formatis"mp3". Defaults to 64. opus_bitrate:OpusBitrate- Opus bitrate in bps: -1000 (auto), 24000, 32000,
48000, or 64000. Only used when
output_formatis"opus". Defaults to 64000. normalize:bool- Whether Fish normalizes the input text (numbers, dates, abbreviations) before synthesis. Defaults to True.
normalize_loudness:NotGivenOr[bool]- Whether Fish normalizes the output
loudness for more consistent volume (Fish
prosody.normalize_loudness). S2-Pro family only: on thes1model the option is dropped with a warning. Unset uses Fish's server default (True). max_new_tokens:NotGivenOr[int]- Maximum audio tokens Fish generates per text chunk. Unset uses Fish's server default (1024).
min_chunk_length:NotGivenOr[int]- Minimum characters before Fish splits text into a new chunk (0-100). Unset uses Fish's server default (50).
condition_on_previous_chunks:NotGivenOr[bool]- Whether Fish uses previous audio as context for voice consistency across chunks. Unset uses Fish's server default (True).
early_stop_threshold:NotGivenOr[float]- Early stopping threshold for batch processing (0-1). Unset uses Fish's server default (1.0).
tokenizer:tokenize.SentenceTokenizer- Sentence tokenizer used to detect
sentence boundaries. Defaults to
tokenize.blingfire.SentenceTokenizer(). http_session:aiohttp.ClientSession | None- Optional aiohttp session.
Ancestors
- livekit.agents.tts.tts.TTS
- abc.ABC
- EventEmitter
- typing.Generic
Instance variables
prop latency_mode : LatencyMode-
Expand source code
@property def latency_mode(self) -> LatencyMode: return self._opts.latency_mode prop model : TTSModels | str-
Expand source code
@property def model(self) -> TTSModels | str: return self._opts.modelGet the model name/identifier for this TTS instance.
Returns
The model name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their model information.
prop output_format : OutputFormat-
Expand source code
@property def output_format(self) -> OutputFormat: return self._opts.output_format prop provider : str-
Expand source code
@property def provider(self) -> str: return "FishAudio"Get the provider name/identifier for this TTS instance.
Returns
The provider name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their provider information.
prop voice_id : NotGivenOr[str]-
Expand source code
@property def voice_id(self) -> NotGivenOr[str]: return self._opts.voice_id
Methods
async def aclose(self) ‑> None-
Expand source code
async def aclose(self) -> None: for stream in list(self._streams): await stream.aclose() self._streams.clear() await self._pool.aclose() def prewarm(self) ‑> None-
Expand source code
def prewarm(self) -> None: self._pool.prewarm()Pre-warm connection to the TTS service
def stream(self,
*,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0)) ‑> livekit.plugins.fishaudio.tts.SynthesizeStream-
Expand source code
def stream( self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> SynthesizeStream: stream = SynthesizeStream(tts=self, conn_options=conn_options) self._streams.add(stream) return stream def synthesize(self,
text: str,
*,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0)) ‑> livekit.plugins.fishaudio.tts.ChunkedStream-
Expand source code
def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def update_options(self,
*,
model: NotGivenOr[TTSModels | str] = NOT_GIVEN,
voice_id: NotGivenOr[str] = NOT_GIVEN,
latency_mode: NotGivenOr[LatencyMode] = NOT_GIVEN,
chunk_length: NotGivenOr[int] = NOT_GIVEN,
speed: NotGivenOr[float] = NOT_GIVEN,
volume: NotGivenOr[float] = NOT_GIVEN,
temperature: NotGivenOr[float] = NOT_GIVEN,
top_p: NotGivenOr[float] = NOT_GIVEN,
mp3_bitrate: NotGivenOr[MP3Bitrate] = NOT_GIVEN,
opus_bitrate: NotGivenOr[OpusBitrate] = NOT_GIVEN,
normalize: NotGivenOr[bool] = NOT_GIVEN,
normalize_loudness: NotGivenOr[bool] = NOT_GIVEN,
max_new_tokens: NotGivenOr[int] = NOT_GIVEN,
min_chunk_length: NotGivenOr[int] = NOT_GIVEN,
condition_on_previous_chunks: NotGivenOr[bool] = NOT_GIVEN,
early_stop_threshold: NotGivenOr[float] = NOT_GIVEN) ‑> None-
Expand source code
def update_options( self, *, model: NotGivenOr[TTSModels | str] = NOT_GIVEN, voice_id: NotGivenOr[str] = NOT_GIVEN, latency_mode: NotGivenOr[LatencyMode] = NOT_GIVEN, chunk_length: NotGivenOr[int] = NOT_GIVEN, speed: NotGivenOr[float] = NOT_GIVEN, volume: NotGivenOr[float] = NOT_GIVEN, temperature: NotGivenOr[float] = NOT_GIVEN, top_p: NotGivenOr[float] = NOT_GIVEN, mp3_bitrate: NotGivenOr[MP3Bitrate] = NOT_GIVEN, opus_bitrate: NotGivenOr[OpusBitrate] = NOT_GIVEN, normalize: NotGivenOr[bool] = NOT_GIVEN, normalize_loudness: NotGivenOr[bool] = NOT_GIVEN, max_new_tokens: NotGivenOr[int] = NOT_GIVEN, min_chunk_length: NotGivenOr[int] = NOT_GIVEN, condition_on_previous_chunks: NotGivenOr[bool] = NOT_GIVEN, early_stop_threshold: NotGivenOr[float] = NOT_GIVEN, ) -> None: if is_given(model) and model != self._opts.model: self._opts.model = model # The model is sent as a connection header at ws-handshake time, not in the # per-request body, so a pooled socket keeps the old model. Drop pooled # connections so the next stream reconnects with the new model. Other # options ride in the per-request body and need no reconnect. self._pool.invalidate() if model == "s1" and is_given(self._opts.normalize_loudness): logger.warning("normalize_loudness is not supported by the s1 model, dropping") self._opts.normalize_loudness = NOT_GIVEN if is_given(voice_id): self._opts.voice_id = voice_id if is_given(latency_mode): self._opts.latency_mode = latency_mode if is_given(chunk_length): if not 100 <= chunk_length <= 300: raise ValueError("chunk_length must be between 100 and 300") self._opts.chunk_length = chunk_length if is_given(speed): self._opts.speed = speed if is_given(volume): self._opts.volume = volume if is_given(temperature): if not 0 <= temperature <= 1: raise ValueError("temperature must be between 0 and 1") self._opts.temperature = temperature if is_given(top_p): if not 0 <= top_p <= 1: raise ValueError("top_p must be between 0 and 1") self._opts.top_p = top_p if is_given(mp3_bitrate): self._opts.mp3_bitrate = mp3_bitrate if is_given(opus_bitrate): self._opts.opus_bitrate = opus_bitrate if is_given(normalize): self._opts.normalize = normalize if is_given(normalize_loudness): if self._opts.model == "s1": logger.warning("normalize_loudness is not supported by the s1 model, dropping") else: self._opts.normalize_loudness = normalize_loudness if is_given(max_new_tokens): if max_new_tokens < 0: raise ValueError("max_new_tokens must be non-negative") self._opts.max_new_tokens = max_new_tokens if is_given(min_chunk_length): if not 0 <= min_chunk_length <= 100: raise ValueError("min_chunk_length must be between 0 and 100") self._opts.min_chunk_length = min_chunk_length if is_given(condition_on_previous_chunks): self._opts.condition_on_previous_chunks = condition_on_previous_chunks if is_given(early_stop_threshold): if not 0 <= early_stop_threshold <= 1: raise ValueError("early_stop_threshold must be between 0 and 1") self._opts.early_stop_threshold = early_stop_threshold
Inherited members