Module livekit.plugins.google.beta
Classes
class GeminiTranslationModel (*,
target_language: str,
model: GeminiTranslationModels | str = 'gemini-3.5-live-translate-preview',
echo_target_language: bool = False,
input_audio_transcription: NotGivenOr[types.AudioTranscriptionConfig | None] = NOT_GIVEN,
output_audio_transcription: NotGivenOr[types.AudioTranscriptionConfig | None] = NOT_GIVEN,
api_key: str | None = None,
vertexai: bool | None = None,
project: str | None = None,
location: str | None = None,
credentials: google.auth.credentials.Credentials | None = None,
http_options: types.HttpOptions | None = None,
transcript_segment_timeout: float = 1.0,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0, liveness_timeout=6.0))-
Expand source code
class GeminiTranslationModel(TranslationModel): """Speech-to-speech translation with Gemini 3.5 Live Translate. The source language is detected by the model, so only the language to translate into is set. Run it with :class:`~livekit.agents.beta.translation.TranslationSession`. Example: ```python session = TranslationSession( model=google.beta.GeminiTranslationModel(target_language="es") ) await session.start(room=ctx.room) ``` """ def __init__( self, *, target_language: str, model: GeminiTranslationModels | str = DEFAULT_TRANSLATION_MODEL, echo_target_language: bool = False, input_audio_transcription: NotGivenOr[types.AudioTranscriptionConfig | None] = NOT_GIVEN, output_audio_transcription: NotGivenOr[types.AudioTranscriptionConfig | None] = NOT_GIVEN, api_key: str | None = None, vertexai: bool | None = None, project: str | None = None, location: str | None = None, credentials: google.auth.credentials.Credentials | None = None, http_options: types.HttpOptions | None = None, transcript_segment_timeout: float = DEFAULT_TRANSCRIPT_SEGMENT_TIMEOUT, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> None: """ Args: target_language: The language to translate into, as a BCP-47 code such as ``"es"``. model: The translation model. echo_target_language: Whether speech already in the target language is repeated. When off, the model stays silent for it. input_audio_transcription: Transcription of the speech heard, in the language it was spoken. On by default; ``None`` turns it off. output_audio_transcription: Transcription of the translation spoken. On by default; ``None`` turns it off. api_key: Gemini API key; read from ``GOOGLE_API_KEY`` when omitted. vertexai: Whether to use Vertex AI; read from ``GOOGLE_GENAI_USE_VERTEXAI`` when omitted. project: Google Cloud project for Vertex AI; read from ``GOOGLE_CLOUD_PROJECT`` or the default credentials when omitted. location: Vertex AI location; defaults to ``global``, the only one serving the model. credentials: Google Cloud credentials for Vertex AI. http_options: HTTP options for the genai client. transcript_segment_timeout: Seconds without a transcript piece that end a transcript the server did not mark finished. conn_options: Retry and timeout settings for the connection. Raises: ValueError: If no API key or Vertex AI project can be found. """ if not is_given(input_audio_transcription): input_audio_transcription = types.AudioTranscriptionConfig() if not is_given(output_audio_transcription): output_audio_transcription = types.AudioTranscriptionConfig() super().__init__( capabilities=TranslationCapabilities( input_transcription=input_audio_transcription is not None, output_transcription=output_audio_transcription is not None, turn_reporting=False, output_segmentation=False, retargetable=True, ), target_language=target_language, source_language=None, ) use_vertexai = ( vertexai if vertexai is not None else os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in ["true", "1"] ) gemini_api_key = api_key or os.environ.get("GOOGLE_API_KEY") gcp_project = project or os.environ.get("GOOGLE_CLOUD_PROJECT") gcp_location: str | None = location or DEFAULT_VERTEX_LOCATION if use_vertexai: if not gcp_project: _, gcp_project = default_async( # type: ignore scopes=["https://www.googleapis.com/auth/cloud-platform"] ) if not gcp_project: raise ValueError( "Project is required for VertexAI via project kwarg or " "GOOGLE_CLOUD_PROJECT environment variable" ) gemini_api_key = None # VertexAI does not require an API key else: gcp_project = None gcp_location = None if credentials is not None: logger.warning( "'credentials' is only applicable to VertexAI and will be ignored for the " "Gemini API" ) credentials = None if not gemini_api_key: raise ValueError( "API key is required for Google API either via api_key or GOOGLE_API_KEY " "environment variable" ) self._opts = _TranslationOptions( model=model, target_language=target_language, echo_target_language=echo_target_language, input_audio_transcription=input_audio_transcription, output_audio_transcription=output_audio_transcription, api_key=gemini_api_key, vertexai=use_vertexai, project=gcp_project, location=gcp_location, credentials=credentials, http_options=http_options, transcript_segment_timeout=transcript_segment_timeout, conn_options=conn_options, ) @property def model(self) -> str: return self._opts.model @property def provider(self) -> str: return "Vertex AI" if self._opts.vertexai else "Gemini" def update_options(self, *, target_language: str) -> None: super().update_options(target_language=target_language) self._opts.target_language = target_language def session(self) -> GeminiTranslationSession: return GeminiTranslationSession(self) async def aclose(self) -> None: passSpeech-to-speech translation with Gemini 3.5 Live Translate.
The source language is detected by the model, so only the language to translate into is set. Run it with :class:
~livekit.agents.beta.translation.TranslationSession.Example
session = TranslationSession( model=google.beta.GeminiTranslationModel(target_language="es") ) await session.start(room=ctx.room)Args
target_language- The language to translate into, as a BCP-47 code such as
"es". model- The translation model.
echo_target_language- Whether speech already in the target language is repeated. When off, the model stays silent for it.
input_audio_transcription- Transcription of the speech heard, in the language it
was spoken. On by default;
Noneturns it off. output_audio_transcription- Transcription of the translation spoken. On by
default;
Noneturns it off. api_key- Gemini API key; read from
GOOGLE_API_KEYwhen omitted. vertexai- Whether to use Vertex AI; read from
GOOGLE_GENAI_USE_VERTEXAIwhen omitted. project- Google Cloud project for Vertex AI; read from
GOOGLE_CLOUD_PROJECTor the default credentials when omitted. location- Vertex AI location; defaults to
global, the only one serving the model. credentials- Google Cloud credentials for Vertex AI.
http_options- HTTP options for the genai client.
transcript_segment_timeout- Seconds without a transcript piece that end a transcript the server did not mark finished.
conn_options- Retry and timeout settings for the connection.
Raises
ValueError- If no API key or Vertex AI project can be found.
Ancestors
- TranslationModel
- abc.ABC
Instance variables
prop model : str-
Expand source code
@property def model(self) -> str: return self._opts.model prop provider : str-
Expand source code
@property def provider(self) -> str: return "Vertex AI" if self._opts.vertexai else "Gemini"
Methods
async def aclose(self) ‑> None-
Expand source code
async def aclose(self) -> None: pass
Inherited members
class GeminiTranslationSession (translation_model: GeminiTranslationModel)-
Expand source code
class GeminiTranslationSession(TranslationModelSession[None]): """One run of Gemini Live translation, reconnecting when its setup has to change. An ``input_transcript`` carries the transcript so far, as an STT interim does, and the final one the whole of it. An ``output_transcript`` carries the text added since the last, as the translation's transcript is a delta stream, and the one that ends it carries none, only ``is_final``. """ def __init__(self, translation_model: GeminiTranslationModel) -> None: super().__init__(translation_model) self._model: GeminiTranslationModel = translation_model self._opts = replace(translation_model._opts) self._loop = asyncio.get_event_loop() self._msg_ch = utils.aio.Chan[types.Blob]() self._audio_ch = utils.aio.Chan[TranslatedAudioFrame | OutputSpeechStopped]() self._input_resampler: rtc.AudioResampler | None = None # 100ms chunks, as the model recommends self._bstream = audio_utils.AudioByteStream( INPUT_AUDIO_SAMPLE_RATE, INPUT_AUDIO_CHANNELS, samples_per_channel=INPUT_AUDIO_SAMPLE_RATE // 10, ) # whether audio went out since the model last marked the end of a translation self._speaking = False # set to replace the connection, e.g. once the target language changed self._restart = asyncio.Event() self._num_retries = 0 # the language the current connection was set up with; audio still arriving from an # old connection after a retarget is in that one, not the new target self._output_language = self._opts.target_language # audio whose send failed, sent first on the next connection so no speech is lost self._unsent_audio: types.Blob | None = None self._input_segmenter = _TranscriptSegmenter( self._opts.transcript_segment_timeout, on_text=lambda _, text, language: self.emit( "input_transcript", InputTranscript(text=text, is_final=False, language=language) ), on_final=lambda text, language: self.emit( "input_transcript", InputTranscript(text=text, is_final=True, language=language) ), loop=self._loop, ) self._output_segmenter = _TranscriptSegmenter( self._opts.transcript_segment_timeout, on_text=lambda delta, _, language: self.emit( "output_transcript", OutputTranscript( text=delta, is_final=False, language=language or self._output_language ), ), on_final=lambda _, language: self.emit( "output_transcript", OutputTranscript( text="", is_final=True, language=language or self._output_language ), ), loop=self._loop, ) http_options = self._opts.http_options or types.HttpOptions( timeout=int(self._opts.conn_options.timeout * 1000) ) if not http_options.headers: http_options.headers = {} http_options.headers["x-goog-api-client"] = f"livekit-agents/{__version__}" self._client = GenAIClient( api_key=self._opts.api_key, vertexai=self._opts.vertexai, project=self._opts.project, location=self._opts.location, credentials=self._opts.credentials, http_options=http_options, ) self._main_atask = asyncio.create_task( self._main_task(), name="GeminiTranslationSession._main_task" ) @property def audio_stream(self) -> AsyncIterable[TranslatedAudioFrame | OutputSpeechStopped]: return self._audio_ch def push_audio(self, frame: rtc.AudioFrame) -> None: for f in self._resample_audio(frame): self._send_audio(self._bstream.write(f.data.tobytes())) def _send_audio(self, frames: list[rtc.AudioFrame]) -> None: for nf in frames: with contextlib.suppress(utils.aio.channel.ChanClosed): self._msg_ch.send_nowait( types.Blob( data=nf.data.tobytes(), mime_type=f"audio/pcm;rate={INPUT_AUDIO_SAMPLE_RATE}", ) ) def _flush_input_audio(self) -> None: """Send the tail of the input still held short of a full chunk.""" if self._input_resampler is not None: for f in self._input_resampler.flush(): self._send_audio(self._bstream.write(f.data.tobytes())) self._input_resampler = None self._send_audio(self._bstream.flush()) def clear_audio(self) -> None: # what the server sent is already here; drop what the consumer has not read yet with contextlib.suppress(utils.aio.channel.ChanEmpty, utils.aio.channel.ChanClosed): while True: self._audio_ch.recv_nowait() def update_options(self, *, target_language: str) -> None: if target_language == self._opts.target_language: return self._opts.target_language = target_language # the setup is fixed for a connection's life, so the new language needs a new one self._restart.set() async def aclose(self) -> None: if not self._closing: self._closing = True self._flush_input_audio() # the send task drains what is queued, then ends the connection self._msg_ch.close() # a failure that ended the session was already emitted as an error with contextlib.suppress(Exception): await self._main_atask try: await self._client.aio.aclose() except Exception: logger.warning("failed to close the genai client", exc_info=True) def _build_connect_config(self) -> types.LiveConnectConfig: return types.LiveConnectConfig( response_modalities=[types.Modality.AUDIO], input_audio_transcription=self._opts.input_audio_transcription, output_audio_transcription=self._opts.output_audio_transcription, translation_config=types.TranslationConfig( target_language_code=self._opts.target_language, echo_target_language=self._opts.echo_target_language, ), ) def _connect( self, config: types.LiveConnectConfig ) -> AbstractAsyncContextManager[AsyncSession]: return self._client.aio.live.connect(model=self._opts.model, config=config) def _resample_audio(self, frame: rtc.AudioFrame) -> Iterator[rtc.AudioFrame]: if self._input_resampler: if frame.sample_rate != self._input_resampler._input_rate: # input audio changed to a different sample rate yield from self._input_resampler.flush() self._input_resampler = None if self._input_resampler is None and ( frame.sample_rate != INPUT_AUDIO_SAMPLE_RATE or frame.num_channels != INPUT_AUDIO_CHANNELS ): self._input_resampler = rtc.AudioResampler( input_rate=frame.sample_rate, output_rate=INPUT_AUDIO_SAMPLE_RATE, num_channels=INPUT_AUDIO_CHANNELS, ) if self._input_resampler: yield from self._input_resampler.push(frame) else: yield frame @utils.log_exceptions(logger=logger) async def _main_task(self) -> None: max_retries = self._opts.conn_options.max_retry try: while not self._msg_ch.closed: self._restart.clear() connected = False try: language = self._opts.target_language async with self._connect(self._build_connect_config()) as session: connected = True self._output_language = language await self._run_session(session) except Exception as e: if self._msg_ch.closed: break if ( (not connected and _is_setup_error(e)) or max_retries == 0 or self._num_retries == max_retries ): self._emit_error(e, recoverable=False) raise APIConnectionError( message="Gemini Live translation connection failed" ) from e self._emit_error(e, recoverable=True) retry_interval = self._opts.conn_options._interval_for_retry(self._num_retries) logger.warning( f"Gemini Live translation connection failed, retrying in {retry_interval}s", exc_info=e, extra={"attempt": self._num_retries, "max_retries": max_retries}, ) await asyncio.sleep(retry_interval) self._num_retries += 1 finally: # what the old connection was saying will not be finished on the next one self._input_segmenter.flush() self._output_segmenter.flush() self._end_output_speech() finally: self._audio_ch.close() async def _run_session(self, session: AsyncSession) -> None: tasks = [ asyncio.create_task(self._send_task(session), name="gemini-translation-send"), asyncio.create_task(self._recv_task(session), name="gemini-translation-recv"), asyncio.create_task(self._restart.wait(), name="gemini-translation-restart"), ] try: done, _ = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED) for task in done: task.result() finally: await utils.aio.cancel_and_wait(*tasks) async def _send_task(self, session: AsyncSession) -> None: if (blob := self._unsent_audio) is not None: self._unsent_audio = None await self._send_or_keep(session, blob) async for blob in self._msg_ch: await self._send_or_keep(session, blob) async def _send_or_keep(self, session: AsyncSession, blob: types.Blob) -> None: try: await session.send_realtime_input(audio=blob) except (Exception, asyncio.CancelledError): # a failed or interrupted send means the connection is going; keep the audio for # the next one self._unsent_audio = blob raise async def _recv_task(self, session: AsyncSession) -> None: while True: # receive() ends at each turn_complete; the session goes on past it async for response in session.receive(): # reset only once the server answers, so a connection that drops straight # after opening still runs out of retries self._num_retries = 0 if response.server_content: self._handle_server_content(response.server_content) if response.go_away: logger.debug( "Gemini Live translation is closing the connection, reconnecting", extra={"time_left": response.go_away.time_left}, ) return def _handle_server_content(self, server_content: types.LiveServerContent) -> None: if model_turn := server_content.model_turn: for part in model_turn.parts or []: if part.inline_data and isinstance(data := part.inline_data.data, bytes): self._handle_output_audio(data) if (transcription := server_content.input_transcription) is not None: self._input_segmenter.push( transcription.text or "", language=transcription.language_code, finished=bool(transcription.finished), ) if (transcription := server_content.output_transcription) is not None: self._output_segmenter.push( transcription.text or "", language=transcription.language_code, finished=bool(transcription.finished), ) if server_content.generation_complete or server_content.turn_complete: self._end_output_speech() def _handle_output_audio(self, data: bytes) -> None: samples_per_channel = len(data) // (2 * OUTPUT_AUDIO_CHANNELS) if samples_per_channel == 0: return frame = rtc.AudioFrame( data=data[: samples_per_channel * 2 * OUTPUT_AUDIO_CHANNELS], sample_rate=OUTPUT_AUDIO_SAMPLE_RATE, num_channels=OUTPUT_AUDIO_CHANNELS, samples_per_channel=samples_per_channel, ) with contextlib.suppress(utils.aio.channel.ChanClosed): self._audio_ch.send_nowait( TranslatedAudioFrame(frame=frame, language=self._output_language) ) self._speaking = True def _end_output_speech(self) -> None: """Mark the end of the translation the model has been speaking, if it spoke one.""" if not self._speaking: return self._speaking = False with contextlib.suppress(utils.aio.channel.ChanClosed): self._audio_ch.send_nowait(OutputSpeechStopped()) def _emit_error(self, error: Exception, recoverable: bool) -> None: self.emit( "error", TranslationModelError( timestamp=time.time(), label=self._model.label, error=error, recoverable=recoverable, ), )One run of Gemini Live translation, reconnecting when its setup has to change.
An
input_transcriptcarries the transcript so far, as an STT interim does, and the final one the whole of it. Anoutput_transcriptcarries the text added since the last, as the translation's transcript is a delta stream, and the one that ends it carries none, onlyis_final.Ancestors
- TranslationModelSession
- abc.ABC
- EventEmitter
- typing.Generic
Methods
async def aclose(self) ‑> None-
Expand source code
async def aclose(self) -> None: if not self._closing: self._closing = True self._flush_input_audio() # the send task drains what is queued, then ends the connection self._msg_ch.close() # a failure that ended the session was already emitted as an error with contextlib.suppress(Exception): await self._main_atask try: await self._client.aio.aclose() except Exception: logger.warning("failed to close the genai client", exc_info=True)
Inherited members
class GeminiSTT (*,
model: str = 'gemini-3.5-transcribe-live',
language: LanguageCode | str | None = 'en-US',
language_codes: list[str] | None = None,
custom_vocabulary: list[str] | None = None,
sample_rate: int = 16000,
api_key: NotGivenOr[str] = NOT_GIVEN,
vertexai: NotGivenOr[bool] = NOT_GIVEN,
credentials: Any | None = None,
credentials_path: str | None = None,
project: NotGivenOr[str] = NOT_GIVEN,
location: NotGivenOr[str] = NOT_GIVEN,
http_options: Any | None = None)-
Expand source code
class STT(stt.STT): def __init__( self, *, model: str = DEFAULT_MODEL, language: LanguageCode | str | None = "en-US", language_codes: list[str] | None = None, custom_vocabulary: list[str] | None = None, sample_rate: int = DEFAULT_SAMPLE_RATE, api_key: NotGivenOr[str] = NOT_GIVEN, vertexai: NotGivenOr[bool] = NOT_GIVEN, credentials: Any | None = None, credentials_path: str | None = None, project: NotGivenOr[str] = NOT_GIVEN, location: NotGivenOr[str] = NOT_GIVEN, http_options: Any | None = None, ) -> None: """Create a new instance of Gemini STT. Args: model: Live-capable Gemini model identifier. Defaults to "gemini-3.5-transcribe-live". Plain chat models are not accepted by the Live endpoint. language: Target language BCP-47 code or LanguageCode. Defaults to "en-US". language_codes: BCP-47 codes for the languages in the audio. Omit, or pass an empty list, to let the model detect the language. custom_vocabulary: Up to 1000 terms that bias recognition -- names, acronyms and jargon the model would otherwise mishear. sample_rate: Sample rate in Hz. Defaults to 16000. api_key: Optional Gemini API key. If not set, uses environment variables. vertexai: Whether to use Vertex AI backend. credentials: Service account credentials object or JSON string. credentials_path: Path to service account credentials JSON file. project: Google Cloud project ID (required for Vertex AI). location: Google Cloud region (e.g. "us-central1"). http_options: Optional HTTP options for Client. """ super().__init__( capabilities=stt.STTCapabilities( streaming=True, interim_results=True, offline_recognize=False, ) ) lang_code = LanguageCode(language) if language is not None else None self._opts = _STTOptions( model=model, language=lang_code, language_codes=language_codes, custom_vocabulary=custom_vocabulary, sample_rate=sample_rate, vertexai=vertexai if is_given(vertexai) else None, project=project if is_given(project) else None, location=location if is_given(location) else None, ) self._api_key = api_key if is_given(api_key) else None self._credentials = credentials self._credentials_path = credentials_path self._http_options = http_options @property def model(self) -> str: return self._opts.model @property def provider(self) -> str: return "google" def stream( self, *, language: NotGivenOr[str] = NOT_GIVEN, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> RecognizeStream: opts = self._opts if is_given(language): opts = _STTOptions( model=self._opts.model, language=LanguageCode(language), language_codes=self._opts.language_codes, custom_vocabulary=self._opts.custom_vocabulary, sample_rate=self._opts.sample_rate, vertexai=self._opts.vertexai, project=self._opts.project, location=self._opts.location, ) return RecognizeStream( stt=self, opts=opts, conn_options=conn_options, api_key=self._api_key, credentials=self._credentials, credentials_path=self._credentials_path, http_options=self._http_options, ) async def _recognize_impl( self, buffer: utils.AudioBuffer, *, language: NotGivenOr[str] = NOT_GIVEN, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> stt.SpeechEvent: raise NotImplementedError("Gemini STT only supports streaming recognition")Helper class that provides a standard way to create an ABC using inheritance.
Create a new instance of Gemini STT.
Args
model- Live-capable Gemini model identifier. Defaults to "gemini-3.5-transcribe-live". Plain chat models are not accepted by the Live endpoint.
language- Target language BCP-47 code or LanguageCode. Defaults to "en-US".
language_codes- BCP-47 codes for the languages in the audio. Omit, or pass an empty list, to let the model detect the language.
custom_vocabulary- Up to 1000 terms that bias recognition – names, acronyms and jargon the model would otherwise mishear.
sample_rate- Sample rate in Hz. Defaults to 16000.
api_key- Optional Gemini API key. If not set, uses environment variables.
vertexai- Whether to use Vertex AI backend.
credentials- Service account credentials object or JSON string.
credentials_path- Path to service account credentials JSON file.
project- Google Cloud project ID (required for Vertex AI).
location- Google Cloud region (e.g. "us-central1").
http_options- Optional HTTP options for Client.
Ancestors
- livekit.agents.stt.stt.STT
- abc.ABC
- EventEmitter
- typing.Generic
Instance variables
prop model : str-
Expand source code
@property def model(self) -> str: return self._opts.modelGet the model name/identifier for this STT instance.
Returns
The model name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their model information.
prop provider : str-
Expand source code
@property def provider(self) -> str: return "google"Get the provider name/identifier for this STT instance.
Returns
The provider name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their provider information.
Methods
def stream(self,
*,
language: NotGivenOr[str] = NOT_GIVEN,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0, liveness_timeout=6.0)) ‑> livekit.plugins.google.beta.gemini_stt.RecognizeStream-
Expand source code
def stream( self, *, language: NotGivenOr[str] = NOT_GIVEN, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS, ) -> RecognizeStream: opts = self._opts if is_given(language): opts = _STTOptions( model=self._opts.model, language=LanguageCode(language), language_codes=self._opts.language_codes, custom_vocabulary=self._opts.custom_vocabulary, sample_rate=self._opts.sample_rate, vertexai=self._opts.vertexai, project=self._opts.project, location=self._opts.location, ) return RecognizeStream( stt=self, opts=opts, conn_options=conn_options, api_key=self._api_key, credentials=self._credentials, credentials_path=self._credentials_path, http_options=self._http_options, )
Inherited members
class GeminiTTS (*,
model: GEMINI_TTS_MODELS | str = 'gemini-3.1-flash-tts-preview',
voice_name: GEMINI_VOICES | str = 'Kore',
api_key: NotGivenOr[str] = NOT_GIVEN,
vertexai: NotGivenOr[bool] = NOT_GIVEN,
project: NotGivenOr[str] = NOT_GIVEN,
location: NotGivenOr[str] = NOT_GIVEN,
instructions: NotGivenOr[str | None] = NOT_GIVEN,
speakers: NotGivenOr[dict[str, str]] = NOT_GIVEN,
speaker: NotGivenOr[str] = NOT_GIVEN)-
Expand source code
class TTS(tts.TTS): def __init__( self, *, model: GEMINI_TTS_MODELS | str = DEFAULT_MODEL, voice_name: GEMINI_VOICES | str = DEFAULT_VOICE, api_key: NotGivenOr[str] = NOT_GIVEN, vertexai: NotGivenOr[bool] = NOT_GIVEN, project: NotGivenOr[str] = NOT_GIVEN, location: NotGivenOr[str] = NOT_GIVEN, instructions: NotGivenOr[str | None] = NOT_GIVEN, speakers: NotGivenOr[dict[str, str]] = NOT_GIVEN, speaker: NotGivenOr[str] = NOT_GIVEN, ) -> None: """ Create a new instance of Gemini TTS. Environment Requirements: - For VertexAI: Set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable to the path of the service account key file. - For Google Gemini API: Set the `api_key` argument or the `GOOGLE_API_KEY` environment variable. Args: model (str, optional): The Gemini TTS model to use. Defaults to "gemini-3.1-flash-tts-preview". voice_name (str, optional): The voice to use for synthesis. Defaults to "Kore". api_key (str, optional): The API key for Google Gemini. If not provided, it attempts to read from the `GOOGLE_API_KEY` environment variable. vertexai (bool, optional): Whether to use VertexAI. Defaults to False. project (str, optional): The Google Cloud project to use (only for VertexAI). location (str, optional): The location to use for VertexAI API requests. Defaults to "us-central1". instructions (str, optional): Control the style, tone, accent, and pace using prompts. See https://ai.google.dev/gemini-api/docs/speech-generation#controllable speakers (dict[str, str], optional): Speaker name -> voice, for a multi-speaker voice config. Replaces `voice_name`. speaker (str, optional): Which of `speakers` this instance voices. Required with `speakers`, and switchable via `update_options`. """ # noqa: E501 super().__init__( capabilities=tts.TTSCapabilities(streaming=False), sample_rate=DEFAULT_SAMPLE_RATE, num_channels=NUM_CHANNELS, ) gcp_project: str | None = ( project if is_given(project) else os.environ.get("GOOGLE_CLOUD_PROJECT") ) gcp_location: str | None = ( location if is_given(location) else os.environ.get("GOOGLE_CLOUD_LOCATION") or "us-central1" ) use_vertexai = ( vertexai if is_given(vertexai) else os.environ.get("GOOGLE_GENAI_USE_VERTEXAI", "0").lower() in ["true", "1"] ) gemini_api_key = api_key if is_given(api_key) else os.environ.get("GOOGLE_API_KEY") if use_vertexai: if not gcp_project: from google.auth._default_async import default_async _, gcp_project = default_async( # type: ignore scopes=["https://www.googleapis.com/auth/cloud-platform"] ) gemini_api_key = None # VertexAI does not require an API key else: gcp_project = None gcp_location = None if not gemini_api_key: raise ValueError( "API key is required for Google API either via api_key or GOOGLE_API_KEY environment variable" # noqa: E501 ) speaker_map = dict(speakers) if is_given(speakers) else None current_speaker = speaker if is_given(speaker) else None if speaker_map is not None: # not "up to two": the API rejects any other count outright, with # "the number of speaker_voice_configs must equal 2" if len(speaker_map) != 2: raise ValueError(f"`speakers` must name exactly 2 speakers, got {len(speaker_map)}") # the API rejects a multi-speaker turn whose speech_metadata names no speaker, # so there is no useful default to fall back to here if current_speaker is None: raise ValueError("`speaker` is required when `speakers` is set") if current_speaker not in speaker_map: raise ValueError( f"speaker {current_speaker!r} is not one of the configured speakers: " f"{', '.join(sorted(speaker_map))}" ) if not _styles_per_part(model): raise ValueError( f"multi-speaker needs a model that takes per-part speech_metadata; " f"{model!r} does not" ) self._opts = _TTSOptions( speakers=speaker_map, speaker=current_speaker, model=model, voice_name=voice_name, vertexai=use_vertexai, project=gcp_project, location=gcp_location, # this family speaks anything left in a part's text, so the preamble would # be read aloud (measured: it roughly triples a short line). _styled_part # puts instructions in speech_metadata.style instead. instructions=instructions if is_given(instructions) else (None if _styles_per_part(model) else DEFAULT_INSTRUCTIONS), ) self._client = Client( api_key=gemini_api_key, vertexai=use_vertexai, project=gcp_project, location=gcp_location, ) class Markup(tts.TTS.Markup): # only a model that can carry a style out of band declares the dialect; on the # others a marker would have nowhere to go and would be read aloud def _provider_key(self) -> str: assert isinstance(self._tts, TTS) return "gemini" if _styles_per_part(self._tts._opts.model) else "" @property def model(self) -> str: return self._opts.model @property def provider(self) -> str: if self._client.vertexai: return "Vertex AI" else: return "Gemini" def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def update_options( self, *, voice_name: NotGivenOr[str] = NOT_GIVEN, speaker: NotGivenOr[str] = NOT_GIVEN, ) -> None: """ Update the TTS options. Args: voice_name (str, optional): The voice to use for synthesis. speaker (str, optional): Which configured speaker to voice from now on. """ if is_given(voice_name): self._opts.voice_name = voice_name if is_given(speaker): if not self._opts.speakers: raise ValueError("`speaker` needs a TTS constructed with `speakers`") if speaker not in self._opts.speakers: raise ValueError( f"speaker {speaker!r} is not one of the configured speakers: " f"{', '.join(sorted(self._opts.speakers))}" ) self._opts.speaker = speaker async def aclose(self) -> None: """Close the TTS and release its GenAI HTTP clients.""" try: await self._client.aio.aclose() except Exception: logger.warning("failed to close the genai client", exc_info=True)Helper class that provides a standard way to create an ABC using inheritance.
Create a new instance of Gemini TTS.
Environment Requirements: - For VertexAI: Set the
GOOGLE_APPLICATION_CREDENTIALSenvironment variable to the path of the service account key file. - For Google Gemini API: Set theapi_keyargument or theGOOGLE_API_KEYenvironment variable.Args
model:str, optional- The Gemini TTS model to use. Defaults to "gemini-3.1-flash-tts-preview".
voice_name:str, optional- The voice to use for synthesis. Defaults to "Kore".
api_key:str, optional- The API key for Google Gemini. If not provided, it attempts to read from the
GOOGLE_API_KEYenvironment variable. vertexai:bool, optional- Whether to use VertexAI. Defaults to False.
project:str, optional- The Google Cloud project to use (only for VertexAI).
location:str, optional- The location to use for VertexAI API requests. Defaults to "us-central1".
instructions:str, optional- Control the style, tone, accent, and pace using prompts. See https://ai.google.dev/gemini-api/docs/speech-generation#controllable
speakers:dict[str, str], optional- Speaker name -> voice, for a multi-speaker voice config. Replaces
voice_name. speaker:str, optional- Which of
speakersthis instance voices. Required withspeakers, and switchable viaupdate_options.
Ancestors
- livekit.agents.tts.tts.TTS
- abc.ABC
- EventEmitter
- typing.Generic
Class variables
var Markup-
Declares TTS markup capabilities for the expressive pipeline.
Plugins override this inner class to declare which markup dialect the TTS speaks: what the LLM is taught to write, and how those markers are normalized and lowered to the provider's native syntax before synthesis. Stripping markup back out is not here — the transcript sinks do it provider-agnostically (see
_provider_format.split_all_markup).
Instance variables
prop model : str-
Expand source code
@property def model(self) -> str: return self._opts.modelGet the model name/identifier for this TTS instance.
Returns
The model name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their model information.
prop provider : str-
Expand source code
@property def provider(self) -> str: if self._client.vertexai: return "Vertex AI" else: return "Gemini"Get the provider name/identifier for this TTS instance.
Returns
The provider name if available, "unknown" otherwise.
Note
Plugins should override this property to provide their provider information.
Methods
async def aclose(self) ‑> None-
Expand source code
async def aclose(self) -> None: """Close the TTS and release its GenAI HTTP clients.""" try: await self._client.aio.aclose() except Exception: logger.warning("failed to close the genai client", exc_info=True)Close the TTS and release its GenAI HTTP clients.
def synthesize(self,
text: str,
*,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=10.0, liveness_timeout=6.0)) ‑> livekit.plugins.google.beta.gemini_tts.ChunkedStream-
Expand source code
def synthesize( self, text: str, *, conn_options: APIConnectOptions = DEFAULT_API_CONNECT_OPTIONS ) -> ChunkedStream: return ChunkedStream(tts=self, input_text=text, conn_options=conn_options) def update_options(self,
*,
voice_name: NotGivenOr[str] = NOT_GIVEN,
speaker: NotGivenOr[str] = NOT_GIVEN) ‑> None-
Expand source code
def update_options( self, *, voice_name: NotGivenOr[str] = NOT_GIVEN, speaker: NotGivenOr[str] = NOT_GIVEN, ) -> None: """ Update the TTS options. Args: voice_name (str, optional): The voice to use for synthesis. speaker (str, optional): Which configured speaker to voice from now on. """ if is_given(voice_name): self._opts.voice_name = voice_name if is_given(speaker): if not self._opts.speakers: raise ValueError("`speaker` needs a TTS constructed with `speakers`") if speaker not in self._opts.speakers: raise ValueError( f"speaker {speaker!r} is not one of the configured speakers: " f"{', '.join(sorted(self._opts.speakers))}" ) self._opts.speaker = speakerUpdate the TTS options.
Args
voice_name:str, optional- The voice to use for synthesis.
speaker:str, optional- Which configured speaker to voice from now on.
Inherited members