Module livekit.agents.beta

Sub-modules

livekit.agents.beta.tools
livekit.agents.beta.toolsets
livekit.agents.beta.workflows

Classes

class EndCallTool (*,
extra_description: str = '',
delete_room: bool = True,
end_instructions: str | None = 'say goodbye to the user',
ignore_on_enter: bool = False,
on_tool_called: collections.abc.Callable[[livekit.agents.llm.tool_context.Toolset.ToolCalledEvent], collections.abc.Awaitable[None]] | None = None,
on_tool_completed: collections.abc.Callable[[livekit.agents.llm.tool_context.Toolset.ToolCompletedEvent], collections.abc.Awaitable[None]] | None = None)
Expand source code
class EndCallTool(Toolset):
    def __init__(
        self,
        *,
        extra_description: str = "",
        delete_room: bool = True,
        end_instructions: str | None = "say goodbye to the user",
        ignore_on_enter: bool = False,
        on_tool_called: Callable[[Toolset.ToolCalledEvent], Awaitable[None]] | None = None,
        on_tool_completed: Callable[[Toolset.ToolCompletedEvent], Awaitable[None]] | None = None,
    ):
        """
        This tool allows the agent to end the call and disconnect from the room.

        Args:
            extra_description: Additional description to add to the end call tool.
            delete_room: Whether to delete the room when the user ends the call. deleting the room disconnects all remote users, including SIP callers.
            end_instructions: Tool output to the LLM for generating the tool response.
            ignore_on_enter: Hide the tool during ``on_enter`` so the model can't end the call while greeting.
            on_tool_called: Callback to call when the tool is called.
            on_tool_completed: Callback to call when the tool is completed.
        """
        end_call_tool = function_tool(
            self._end_call,
            name="end_call",
            description=f"{END_CALL_DESCRIPTION}\n{extra_description}",
            flags=ToolFlag.IGNORE_ON_ENTER if ignore_on_enter else ToolFlag.NONE,
        )
        super().__init__(id="end_call", tools=[end_call_tool])

        self._delete_room = delete_room
        self._extra_description = extra_description

        self._end_instructions = end_instructions
        self._on_tool_called = on_tool_called
        self._on_tool_completed = on_tool_completed

        self._shutdown_session_task: asyncio.Task[None] | None = None

    async def _end_call(self, ctx: RunContext) -> Any | None:
        logger.debug("end_call tool called")
        activity = ctx.session.current_agent._get_activity_or_raise()
        llm_v = activity.llm

        def _on_speech_done(_: SpeechHandle) -> None:
            # read auto_tool_reply_generation from the active realtime session so a fallback
            # adapter reports the model actually in use
            rt_session = activity.realtime_llm_session
            auto_tool_reply = (
                rt_session is not None and rt_session.capabilities.auto_tool_reply_generation
            )
            if not isinstance(llm_v, RealtimeModel) or not auto_tool_reply:
                # tool reply will reuse the same speech handle, so we can shutdown the session
                # directly after this speech handle is done
                ctx.session.shutdown()
            else:
                self._shutdown_session_task = asyncio.create_task(
                    self._delayed_session_shutdown(ctx)
                )

        ctx.speech_handle.add_done_callback(_on_speech_done)
        ctx.session.once("close", self._on_session_close)

        if self._on_tool_called:
            await self._on_tool_called(Toolset.ToolCalledEvent(ctx=ctx, arguments={}))

        completed_ev = Toolset.ToolCompletedEvent(ctx=ctx, output=self._end_instructions)
        if self._on_tool_completed:
            await self._on_tool_completed(completed_ev)

        return completed_ev.output

    async def _delayed_session_shutdown(self, ctx: RunContext) -> None:
        """Shutdown the session after the tool reply is played out"""
        speech_created_fut = asyncio.Future[SpeechHandle]()

        @ctx.session.once("speech_created")
        def _on_speech_created(ev: SpeechCreatedEvent) -> None:
            if not speech_created_fut.done():
                speech_created_fut.set_result(ev.speech_handle)

        try:
            speech_handle = await asyncio.wait_for(speech_created_fut, timeout=5.0)
            await speech_handle
        except asyncio.TimeoutError:
            logger.warning("tool reply timed out, shutting down session")
        finally:
            ctx.session.off("speech_created", _on_speech_created)
            ctx.session.shutdown()

    def _on_session_close(self, ev: CloseEvent) -> None:
        """Close the job process when AgentSession is closed"""
        if self._shutdown_session_task:
            # cleanup
            self._shutdown_session_task.cancel()
            self._shutdown_session_task = None

        job_ctx = get_job_context()

        if self._delete_room:

            async def _on_shutdown() -> None:
                logger.info("deleting the room because the user ended the call")
                await job_ctx.delete_room()

            job_ctx.add_shutdown_callback(_on_shutdown)

        # shutdown the job process
        job_ctx.shutdown(reason=ev.reason.value)

This tool allows the agent to end the call and disconnect from the room.

Args

extra_description
Additional description to add to the end call tool.
delete_room
Whether to delete the room when the user ends the call. deleting the room disconnects all remote users, including SIP callers.
end_instructions
Tool output to the LLM for generating the tool response.
ignore_on_enter
Hide the tool during on_enter so the model can't end the call while greeting.
on_tool_called
Callback to call when the tool is called.
on_tool_completed
Callback to call when the tool is completed.

Ancestors

  • livekit.agents.llm.tool_context.Toolset
class Instructions (common: str = '', *, audio: str | None = None, text: str | None = None)
Expand source code
class Instructions:
    """Instructions with optional modality-specific additions.

    Construction::

        # Simple — same instructions for all modalities
        Instructions("You are a helpful assistant.")

        # With modality-specific additions
        Instructions(
            "You are a helpful assistant.",
            audio="Keep responses short for voice.",
            text="Use markdown formatting.",
        )

    Rendering::

        instr.render()                              # → common text
        instr.render(modality="audio")               # → common + audio addition
        instr.render(modality="text", name="Alex")   # → common + text, with {name} filled
    """

    def __init__(
        self,
        common: str = "",
        *,
        audio: str | None = None,
        text: str | None = None,
    ) -> None:
        self.common = common
        self.audio = audio
        self.text = text

    def render(
        self,
        *,
        modality: Literal["audio", "text"] | None = None,
        data: dict[str, object] | None = None,
    ) -> str:
        """Render instructions to a plain string.

        Args:
            modality: If given, appends the modality-specific addition to the common text.
            data: Template variables to fill. Missing placeholders log a warning
                and are replaced with empty strings.
        """
        parts = [self.common]
        if modality is not None:
            addition = self.audio if modality == "audio" else self.text
            if addition:
                parts.append(addition)

        result = "\n\n".join(p for p in parts if p)

        if data:
            result = utils.misc.safe_render(result, data)

        return result

    @staticmethod
    def resolve_template(template: str, **kwargs: object) -> Instructions:
        """Fill a template string, producing an ``Instructions`` with modality variants.

        If any kwarg value is an ``Instructions`` object, its ``common``/``audio``/``text``
        parts are substituted into the matching variant of the result. This is used by
        workflow tasks to build modality-aware instructions from a single template.
        """
        any_instructions = any(isinstance(v, Instructions) for v in kwargs.values())
        if any_instructions:
            common_kw: dict[str, object] = {
                k: str(v) if isinstance(v, Instructions) else v for k, v in kwargs.items()
            }
            audio_kw: dict[str, object] = {
                # an explicit "" removes the section; only None falls back to common
                k: (v.audio if v.audio is not None else str(v))
                if isinstance(v, Instructions)
                else v
                for k, v in kwargs.items()
            }
            text_kw: dict[str, object] = {
                k: (v.text if v.text is not None else str(v)) if isinstance(v, Instructions) else v
                for k, v in kwargs.items()
            }
            return Instructions(
                common=utils.misc.safe_render(template, common_kw),
                audio=utils.misc.safe_render(template, audio_kw),
                text=utils.misc.safe_render(template, text_kw),
            )
        else:
            rendered = utils.misc.safe_render(template, kwargs)
            return Instructions(common=rendered)

    def __str__(self) -> str:
        return self.common

    def __repr__(self) -> str:
        return f"Instructions({self.common!r})"

    def __hash__(self) -> int:
        return hash((self.common, self.audio, self.text))

    def __eq__(self, other: object) -> bool:
        if isinstance(other, Instructions):
            return (
                self.common == other.common
                and self.audio == other.audio
                and self.text == other.text
            )
        if isinstance(other, str):
            return self.common == other
        return NotImplemented

Instructions with optional modality-specific additions.

Construction::

# Simple — same instructions for all modalities
Instructions("You are a helpful assistant.")

# With modality-specific additions
Instructions(
    "You are a helpful assistant.",
    audio="Keep responses short for voice.",
    text="Use markdown formatting.",
)

Rendering::

instr.render()                              # → common text
instr.render(modality="audio")               # → common + audio addition
instr.render(modality="text", name="Alex")   # → common + text, with {name} filled

Subclasses

Static methods

def resolve_template(template: str, **kwargs: object) ‑> livekit.agents.llm.chat_context.Instructions
Expand source code
@staticmethod
def resolve_template(template: str, **kwargs: object) -> Instructions:
    """Fill a template string, producing an ``Instructions`` with modality variants.

    If any kwarg value is an ``Instructions`` object, its ``common``/``audio``/``text``
    parts are substituted into the matching variant of the result. This is used by
    workflow tasks to build modality-aware instructions from a single template.
    """
    any_instructions = any(isinstance(v, Instructions) for v in kwargs.values())
    if any_instructions:
        common_kw: dict[str, object] = {
            k: str(v) if isinstance(v, Instructions) else v for k, v in kwargs.items()
        }
        audio_kw: dict[str, object] = {
            # an explicit "" removes the section; only None falls back to common
            k: (v.audio if v.audio is not None else str(v))
            if isinstance(v, Instructions)
            else v
            for k, v in kwargs.items()
        }
        text_kw: dict[str, object] = {
            k: (v.text if v.text is not None else str(v)) if isinstance(v, Instructions) else v
            for k, v in kwargs.items()
        }
        return Instructions(
            common=utils.misc.safe_render(template, common_kw),
            audio=utils.misc.safe_render(template, audio_kw),
            text=utils.misc.safe_render(template, text_kw),
        )
    else:
        rendered = utils.misc.safe_render(template, kwargs)
        return Instructions(common=rendered)

Fill a template string, producing an Instructions with modality variants.

If any kwarg value is an Instructions object, its common/audio/text parts are substituted into the matching variant of the result. This is used by workflow tasks to build modality-aware instructions from a single template.

Methods

def render(self,
*,
modality: "Literal['audio', 'text'] | None" = None,
data: dict[str, object] | None = None) ‑> str
Expand source code
def render(
    self,
    *,
    modality: Literal["audio", "text"] | None = None,
    data: dict[str, object] | None = None,
) -> str:
    """Render instructions to a plain string.

    Args:
        modality: If given, appends the modality-specific addition to the common text.
        data: Template variables to fill. Missing placeholders log a warning
            and are replaced with empty strings.
    """
    parts = [self.common]
    if modality is not None:
        addition = self.audio if modality == "audio" else self.text
        if addition:
            parts.append(addition)

    result = "\n\n".join(p for p in parts if p)

    if data:
        result = utils.misc.safe_render(result, data)

    return result

Render instructions to a plain string.

Args

modality
If given, appends the modality-specific addition to the common text.
data
Template variables to fill. Missing placeholders log a warning and are replaced with empty strings.