Module livekit.plugins.kalpa
Kalpa plugin for LiveKit Agents
Support for text-to-speech with Kalpa.
See https://docs.kalpalabs.ai/text-to-speech for more information.
Classes
class ChunkedStream (*,
tts: TTS,
input_text: str,
conn_options: APIConnectOptions)-
Expand source code
class ChunkedStream(tts.ChunkedStream): """Synthesizes the whole text in one `POST /v1/tts/{voice_id}/stream`, which returns raw PCM as it is generated.""" def __init__(self, *, tts: TTS, input_text: str, conn_options: APIConnectOptions) -> None: super().__init__(tts=tts, input_text=input_text, conn_options=conn_options) self._tts: TTS = tts self._opts = replace(tts._opts) async def _run(self, output_emitter: tts.AudioEmitter) -> None: output_emitter.initialize( request_id=utils.shortuuid(), sample_rate=SAMPLE_RATE, num_channels=NUM_CHANNELS, mime_type="audio/pcm", ) if not self._input_text.strip(): return # the API rejects blank text, and there is nothing to say try: async with self._tts._ensure_session().post( f"{self._opts.base_url}/v1/tts/{self._opts.voice}/stream", headers={"Authorization": f"Bearer {self._opts.api_key}"}, json={ "text": self._input_text, "model": self._opts.model, "sample_rate": SAMPLE_RATE, }, timeout=aiohttp.ClientTimeout( sock_connect=self._conn_options.timeout, sock_read=self._conn_options.timeout ), ) as resp: if resp.status != 200: try: error = (await resp.json(content_type=None))["error"] except Exception: error = {} raise _status_error(error, status=resp.status) output_emitter._note_provider_request_id(resp.headers.get("X-Request-ID", "")) async for data, _ in resp.content.iter_chunks(): output_emitter.push(data) output_emitter.flush() except asyncio.TimeoutError: raise APITimeoutError() from None except APIError: raise except Exception as e: raise APIConnectionError() from eSynthesizes the whole text in one
POST /v1/tts/{voice_id}/stream, which returns raw PCM as it is generated.Ancestors
- livekit.agents.tts.tts.ChunkedStream
- abc.ABC
class SynthesizeStream (*,
tts: TTS,
conn_options: APIConnectOptions)-
Expand source code
class SynthesizeStream(tts.SynthesizeStream): """Streams one reply's text over a pooled `WSS /v1/tts/{voice_id}/stream` connection and receives its audio while the text is still arriving.""" def __init__(self, *, tts: TTS, conn_options: APIConnectOptions) -> None: super().__init__(tts=tts, conn_options=conn_options) self._tts: TTS = tts async def _run(self, output_emitter: tts.AudioEmitter) -> None: request_id = utils.shortuuid() output_emitter.initialize( request_id=request_id, sample_rate=SAMPLE_RATE, num_channels=NUM_CHANNELS, mime_type="audio/pcm", stream=True, ) output_emitter.start_segment(segment_id=request_id) try: await self._run_ws(output_emitter) except asyncio.TimeoutError: raise APITimeoutError() from None except aiohttp.WSServerHandshakeError as e: raise APIStatusError(e.message, status_code=e.status) from None except APIError: raise except Exception as e: raise APIConnectionError() from e output_emitter.end_segment() async def _run_ws(self, output_emitter: tts.AudioEmitter) -> None: pool = self._tts._pool ws: aiohttp.ClientWebSocketResponse | None = None sent = False # the server holds text or a response from this reply done = False # the reply's response finished rejected = False # the server refused one of the reply's messages but kept the connection # the server must answer within the timeout of the flush, and of every event after it; # before the flush, audio may legitimately pause while the reply's text is still arriving deadline: float | None = None async def _send_task(ws: aiohttp.ClientWebSocketResponse) -> None: nonlocal sent, deadline async for data in self._input_ch: if isinstance(data, self._FlushSentinel): if sent: break continue # a flush before any text ends nothing if data.strip(): self._mark_started() sent = True await ws.send_str(json.dumps({"type": "sendText", "text": data})) sent = True await ws.send_str(json.dumps({"type": "sendText", "flush": True})) deadline = time.monotonic() + self._conn_options.timeout async def _recv_task(ws: aiohttp.ClientWebSocketResponse) -> None: nonlocal done, rejected, deadline error: dict[str, Any] | None = None while True: timeout = self._conn_options.timeout if deadline is not None: timeout = max(deadline - time.monotonic(), 0.001) try: event = await _receive_event(ws, timeout) except asyncio.TimeoutError: if deadline is not None and time.monotonic() >= deadline: raise continue if deadline is not None: deadline = time.monotonic() + self._conn_options.timeout if event["type"] == "error": # non-fatal: the connection stays open if not event.get("response_id"): rejected = True # a refused sendText: the reply's text is incomplete raise _status_error(event["error"]) error = event["error"] # the response failed; its responseDone follows elif event["type"] == "responseCreated": output_emitter._note_provider_request_id(event["response_id"]) elif event["type"] == "responseAudio": output_emitter.push(base64.b64decode(event["pcm_b64"])) elif event["type"] == "responseDone": done = True if event["status"] != "completed": if error: raise _status_error(error) raise APIError(f"Kalpa response ended with status {event['status']}") return try: ws = await pool.get(timeout=self._conn_options.timeout) self._acquire_time = pool.last_acquire_time self._connection_reused = pool.last_connection_reused tasks = [ asyncio.create_task(_send_task(ws)), asyncio.create_task(_recv_task(ws)), ] try: await asyncio.gather(*tasks) finally: await utils.aio.gracefully_cancel(*tasks) except BaseException as e: if ws is not None: # an interrupted reply, or a non-fatal error, leaves the socket usable once the # server holds nothing more of this reply if isinstance(e, asyncio.CancelledError) or done or rejected: if sent and not done: self._tts._recycle(ws) else: pool.put(ws) else: self._tts._drop(ws) raise pool.put(ws)Streams one reply's text over a pooled
WSS /v1/tts/{voice_id}/streamconnection and receives its audio while the text is still arriving.Ancestors
- livekit.agents.tts.tts.SynthesizeStream
- abc.ABC
class TTS (*,
voice: str = '18c26f49-9f9c-400c-bec9-7467f28e1db5',
model: TTSModels | str = 'kalpa-tts-1',
api_key: str | None = None,
base_url: str = 'https://api.kalpalabs.ai',
http_session: aiohttp.ClientSession | None = None)-
Expand source code
class TTS(tts.TTS): def __init__( self, *, voice: str = DEFAULT_VOICE, model: TTSModels | str = "kalpa-tts-1", api_key: str | None = None, base_url: str = BASE_URL, http_session: aiohttp.ClientSession | None = None, ) -> None: """ Create a new instance of Kalpa TTS. Args: voice (str): Voice id, from `GET /v1/voices`. Defaults to Arjun. model (TTSModels | str): TTS model. Defaults to "kalpa-tts-1". api_key (str): Kalpa API key. Defaults to the KALPA_API_KEY environment variable. base_url (str): API base URL. Defaults to "https://api.kalpalabs.ai". http_session (aiohttp.ClientSession): Optional aiohttp session to use for requests. """ super().__init__( capabilities=tts.TTSCapabilities(streaming=True), sample_rate=SAMPLE_RATE, num_channels=NUM_CHANNELS, ) api_key = api_key or os.environ.get("KALPA_API_KEY") if not api_key: raise ValueError("Kalpa API key required. Set KALPA_API_KEY or provide api_key.") self._opts = _TTSOptions( model=model, voice=voice, api_key=api_key, base_url=base_url.rstrip("/") ) self._session = http_session self._streams = weakref.WeakSet[SynthesizeStream]() self._tasks: set[asyncio.Task[Any]] = set() self._pool = utils.ConnectionPool[aiohttp.ClientWebSocketResponse]( connect_cb=self._connect_ws, close_cb=self._close_ws, max_session_duration=MAX_IDLE_SECONDS, mark_refreshed_on_get=True, ) @property def model(self) -> str: return self._opts.model @property def provider(self) -> str: return "Kalpa" async def _connect_ws(self, timeout: float) -> aiohttp.ClientWebSocketResponse: url = f"{self._opts.base_url.replace('http', 'ws', 1)}/v1/tts/{self._opts.voice}/stream" ws = await asyncio.wait_for(self._ensure_session().ws_connect(url), timeout) try: await ws.send_str( json.dumps( { "type": "initializeConnection", "api_key": self._opts.api_key, "model": self._opts.model, "sample_rate": SAMPLE_RATE, # audio starts at the first complete sentences, before the reply's flush "generation_config": {}, } ) ) event = await _receive_event(ws, timeout) if event["type"] != "sessionCreated": raise APIError(f"unexpected Kalpa message before sessionCreated: {event['type']}") except BaseException: await self._close_ws(ws) raise return ws async def _close_ws(self, ws: aiohttp.ClientWebSocketResponse) -> None: # a half-open peer never answers the closing handshake; cancelling drops the connection with contextlib.suppress(asyncio.TimeoutError): await asyncio.wait_for(ws.close(), CLOSE_TIMEOUT) def _recycle(self, ws: aiohttp.ClientWebSocketResponse) -> None: """Cancel an interrupted response in the background and return its socket to the pool once the server acknowledges the cancel; without a timely acknowledgement, replace the socket.""" async def _cancel() -> None: try: await asyncio.wait_for(_cancel_response(ws), CANCEL_ACK_TIMEOUT) except Exception: self._drop(ws) self._pool.prewarm() else: self._pool.put(ws) self._spawn(_cancel()) def _drop(self, ws: aiohttp.ClientWebSocketResponse) -> None: """Take a socket out of the pool and close it now, so the server stops any work for it.""" self._pool.remove(ws) self._spawn(self._close_ws(ws)) def _spawn(self, coro: Coroutine[Any, Any, Any]) -> None: task = asyncio.create_task(coro) self._tasks.add(task) task.add_done_callback(self._tasks.discard) def _ensure_session(self) -> aiohttp.ClientSession: if not self._session: self._session = utils.http_context.http_session() return self._session def update_options( self, *, voice: NotGivenOr[str] = NOT_GIVEN, model: NotGivenOr[TTSModels | str] = NOT_GIVEN, ) -> None: """ Update the voice or model. Replies that already hold a connection finish with the old ones; every other reply, including one still connecting, uses the new ones. Args: voice (str): Voice id, from `GET /v1/voices`. model (TTSModels | str): TTS model. """ if is_given(voice): self._opts.voice = voice if is_given(model): self._opts.model = model if is_given(voice) or is_given(model): self._pool.invalidate() def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def stream( self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> SynthesizeStream: stream = SynthesizeStream(tts=self, conn_options=conn_options) self._streams.add(stream) return stream def prewarm(self) -> None: self._pool.prewarm() async def aclose(self) -> None: for stream in list(self._streams): await stream.aclose() self._streams.clear() await utils.aio.cancel_and_wait(*self._tasks) await self._pool.aclose()Helper class that provides a standard way to create an ABC using inheritance.
Create a new instance of Kalpa TTS.
Args
voice:str- Voice id, from
GET /v1/voices. Defaults to Arjun. model:TTSModels | str- TTS model. Defaults to "kalpa-tts-1".
api_key:str- Kalpa API key. Defaults to the KALPA_API_KEY environment variable.
base_url:str- API base URL. Defaults to "https://api.kalpalabs.ai".
http_session:aiohttp.ClientSession- Optional aiohttp session to use for requests.
Ancestors
- livekit.agents.tts.tts.TTS
- abc.ABC
- EventEmitter
- typing.Generic
Instance variables
prop model : str-
Expand source code
@property def model(self) -> str: return self._opts.modelGet the model name/identifier for this TTS instance.
Returns
The model name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their model information.
prop provider : str-
Expand source code
@property def provider(self) -> str: return "Kalpa"Get the provider name/identifier for this TTS instance.
Returns
The provider name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their provider information.
Methods
async def aclose(self) ‑> None-
Expand source code
async def aclose(self) -> None: for stream in list(self._streams): await stream.aclose() self._streams.clear() await utils.aio.cancel_and_wait(*self._tasks) await self._pool.aclose() def prewarm(self) ‑> None-
Expand source code
def prewarm(self) -> None: self._pool.prewarm()Pre-warm connection to the TTS service
def stream(self,
*,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0, liveness_timeout=6.0)) ‑> livekit.plugins.kalpa.tts.SynthesizeStream-
Expand source code
def stream( self, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> SynthesizeStream: stream = SynthesizeStream(tts=self, conn_options=conn_options) self._streams.add(stream) return stream def synthesize(self,
text: str,
*,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0, liveness_timeout=6.0)) ‑> livekit.plugins.kalpa.tts.ChunkedStream-
Expand source code
def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def update_options(self,
*,
voice: NotGivenOr[str] = NOT_GIVEN,
model: NotGivenOr[TTSModels | str] = NOT_GIVEN) ‑> None-
Expand source code
def update_options( self, *, voice: NotGivenOr[str] = NOT_GIVEN, model: NotGivenOr[TTSModels | str] = NOT_GIVEN, ) -> None: """ Update the voice or model. Replies that already hold a connection finish with the old ones; every other reply, including one still connecting, uses the new ones. Args: voice (str): Voice id, from `GET /v1/voices`. model (TTSModels | str): TTS model. """ if is_given(voice): self._opts.voice = voice if is_given(model): self._opts.model = model if is_given(voice) or is_given(model): self._pool.invalidate()Update the voice or model. Replies that already hold a connection finish with the old ones; every other reply, including one still connecting, uses the new ones.
Args
voice:str- Voice id, from
GET /v1/voices. model:TTSModels | str- TTS model.
Inherited members