Audio customization

from livekit.agents.utils.audio import audio_frames_from_file

await session.say(
    "Your phrase",
    audio=audio_frames_from_file(path, sample_rate=24000, num_channels=1),
)

import { audioFramesFromFile } from '@livekit/agents';

await session.say('Your phrase', {
  audio: audioFramesFromFile(path, { sampleRate: 24000, numChannels: 1 }),
});

from livekit import rtc
from livekit.agents import AgentSession

# Hold a reference to the TTS instance you pass to AgentSession.
tts_cache: dict[str, list[rtc.AudioFrame]] = {}

async def say_cached(session: AgentSession, tts, text: str) -> None:
    if text not in tts_cache:
        stream = tts.synthesize(text)
        frames: list[rtc.AudioFrame] = []
        async for event in stream:
            frames.append(event.frame)
        tts_cache[text] = frames

    async def audio_gen():
        for frame in tts_cache[text]:
            yield frame

    await session.say(text, audio=audio_gen())

import { voice } from '@livekit/agents';
import type { AudioFrame } from '@livekit/rtc-node';

// Hold a reference to the TTS instance you pass to AgentSession.
const ttsCache = new Map<string, AudioFrame[]>();

async function sayCached(
  session: voice.AgentSession,
  tts: { synthesize(text: string): AsyncIterableIterator<{ frame: AudioFrame }> },
  text: string,
): Promise<void> {
  let frames = ttsCache.get(text);
  if (!frames) {
    frames = [];
    for await (const event of tts.synthesize(text)) {
      frames.push(event.frame);
    }
    ttsCache.set(text, frames);
  }

  const stream = new ReadableStream<AudioFrame>({
    start(controller) {
      for (const frame of frames!) {
        controller.enqueue(frame);
      }
      controller.close();
    },
  });
  await session.say(text, { audio: stream });
}

from livekit import rtc
from livekit.agents import Agent, RunContext, function_tool

# Pre-synthesize a hold message once at startup
HOLD_FRAMES: list[rtc.AudioFrame] = []

async def preload_hold_message(tts) -> None:
    global HOLD_FRAMES
    async for event in tts.synthesize("Let me check that for you."):
        HOLD_FRAMES.append(event.frame)

class MyAgent(Agent):
    @function_tool()
    async def check_order_status(
        self,
        context: RunContext,
        order_id: str,
    ) -> str:
        """Check the status of an order.

        Args:
            order_id: The order ID to look up.
        """
        async def cached_audio():
            for frame in HOLD_FRAMES:
                yield frame

        # Play the hold message concurrently — don't await
        hold_handle = context.session.say(
            "Let me check that for you.",
            audio=cached_audio(),
            add_to_chat_ctx=False,
        )

        # Call the external API (runs while the hold message plays)
        result = await fetch_order_status(order_id)

        # If the API returned before the hold message finished, cancel it
        if not hold_handle.interrupted and not hold_handle.done():
            hold_handle.interrupt()

        return result

import { voice, llm } from '@livekit/agents';
import type { AudioFrame } from '@livekit/rtc-node';
import { z } from 'zod';

// Pre-synthesize a hold message once at startup
let holdFrames: AudioFrame[] = [];

async function preloadHoldMessage(
  tts: { synthesize(text: string): AsyncIterableIterator<{ frame: AudioFrame }> },
) {
  holdFrames = [];
  for await (const event of tts.synthesize('Let me check that for you.')) {
    holdFrames.push(event.frame);
  }
}

class MyAgent extends voice.Agent {
  constructor() {
    super({
      instructions: 'You are a helpful assistant.',
      tools: {
        checkOrderStatus: llm.tool({
          description: 'Check the status of an order.',
          parameters: z.object({
            orderId: z.string().describe('The order ID to look up.'),
          }),
          execute: async ({ orderId }, { ctx }) => {
            // Play the hold message concurrently — don't await
            const stream = new ReadableStream<AudioFrame>({
              start(controller) {
                for (const frame of holdFrames) {
                  controller.enqueue(frame);
                }
                controller.close();
              },
            });
            const holdHandle = ctx.session.say('Let me check that for you.', {
              audio: stream,
              addToChatCtx: false,
            });

            // Call the external API (runs while the hold message plays)
            const result = await fetchOrderStatus(orderId);

            // If the API returned before the hold message finished, cancel it
            if (!holdHandle.interrupted && !holdHandle.done()) {
              holdHandle.interrupt();
            }

            return result;
          },
        }),
      },
    });
  }
}