Module livekit.plugins.vlmrun

VLM Run plugin for LiveKit Agents

Vision-language models from the VLM Run gateway, through its OpenAI-compatible chat completions API, and typed decisions through its System One API.

Classes

class DecisionModel (*,
model: DecisionModels | str = 'qwen/qwen3.8-27b',
api_key: str | None = None,
base_url: str | None = None,
http_session: aiohttp.ClientSession | None = None)
Expand source code
class DecisionModel(llm.DecisionModel):
    """Typed decisions from the VLM Run gateway's System One API.

    Example::

        model = vlmrun.DecisionModel()
        response = await model.evaluate(
            input=chat_ctx,
            decisions={"handoff": llm.Probability("The caller requests a human.")},
        )

    Args:
        model: Gateway model id. Defaults to ``qwen/qwen3.8-27b``.
        api_key: VLM Run API key. Defaults to ``VLMRUN_API_KEY``.
        base_url: Gateway root URL. Defaults to ``VLMRUN_GATEWAY_BASE_URL``, then
            ``VLMRUN_GATEWAY_URL``, then ``https://gateway.vlm.run``.
        http_session: Optional caller-owned aiohttp session. Defaults to the job's HTTP
            session; outside a job, use ``async with utils.http_context.open()``.

    Each attempt times out after 70 s by default, above the gateway's own deadline. Pass
    ``conn_options`` to ``evaluate()`` to change it.
    """

    def __init__(
        self,
        *,
        model: DecisionModels | str = "qwen/qwen3.8-27b",
        api_key: str | None = None,
        base_url: str | None = None,
        http_session: aiohttp.ClientSession | None = None,
    ) -> None:
        super().__init__(
            capabilities=DecisionCapabilities(decision_kinds=["probability", "choice", "score"])
        )
        key = resolve_api_key(api_key)
        if not key:
            raise ValueError(
                "VLMRUN_API_KEY is required, either as argument or set VLMRUN_API_KEY environment variable"  # noqa: E501
            )
        self._model = model
        root = gateway_url(base_url)
        self._url = f"{root}/typesafe/v1/systemone"
        # Host and port only: netloc would carry user:password@ into telemetry.
        parts = urlsplit(root)
        host = parts.hostname or ""
        if ":" in host:
            host = f"[{host}]"
        self._provider = f"{host}:{parts.port}" if parts.port else host
        self._headers = client_headers(key)
        self._session = http_session

    @property
    def model(self) -> str:
        return self._model

    @property
    def provider(self) -> str:
        return self._provider

    async def evaluate(
        self,
        *,
        input: str | llm.ChatContext,
        decisions: Mapping[str, Decision],
        conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS,
    ) -> DecisionResponse:
        """Evaluate a batch of decisions.

        Args:
            input: Text to evaluate, sent as-is, or a conversation converted to a
                transcript of user and assistant text.
            decisions: Decisions keyed by name.
            conn_options: Retry and per-attempt timeout settings. The timeout defaults to
                70 s, above the gateway's own deadline.

        Each requested name appears in either ``results`` or ``errors``. Request failures
        and malformed responses raise.
        """
        return await super().evaluate(
            input=input,
            decisions=decisions,
            conn_options=conn_options_or_default(conn_options),
        )

    async def _evaluate_impl(
        self,
        *,
        input: str,
        decisions: Mapping[str, Decision],
        conn_options: APIConnectOptions,
    ) -> DecisionResponse:
        questions = {name: systemone.encode_question(d) for name, d in decisions.items()}
        session = self._session or utils.http_context.http_session()
        try:
            async with session.post(
                self._url,
                headers=self._headers,
                json={"model": self._model, "state": input, "questions": questions},
                timeout=aiohttp.ClientTimeout(total=conn_options.timeout),
                allow_redirects=False,
            ) as response:
                request_id = response.headers.get("x-request-id", "")
                if response.status != 200:
                    raise APIStatusError(
                        "VLM Run decision request failed",
                        status_code=response.status,
                        request_id=request_id or None,
                        body=await response.text(),
                        retryable=response.status in (408, 429) or 500 <= response.status < 600,
                    )
                parsed = systemone.Response.model_validate(await response.json())
                request_id = parsed.id or request_id
            results, errors = systemone.decode_answers(
                parsed.answers, invalid="invalid VLM Run decision answer"
            )
            return DecisionResponse(
                results=results,
                errors=errors,
                model=parsed.model,
                provider=self.provider,
                request_id=request_id,
                input_tokens=parsed.usage.input_tokens,
                output_tokens=parsed.usage.output_tokens,
            )
        except asyncio.TimeoutError:
            raise APITimeoutError("VLM Run decision request timed out") from None
        except aiohttp.ContentTypeError:
            raise APIError("invalid VLM Run decision response", retryable=False) from None
        except (aiohttp.ClientError, ConnectionError, OSError):
            raise APIConnectionError("VLM Run decision transport failed") from None
        except ValueError:
            raise APIError("invalid VLM Run decision response", retryable=False) from None

Typed decisions from the VLM Run gateway's System One API.

Example::

model = vlmrun.DecisionModel()
response = await model.evaluate(
    input=chat_ctx,
    decisions={"handoff": llm.Probability("The caller requests a human.")},
)

Args

model
Gateway model id. Defaults to qwen/qwen3.8-27b.
api_key
VLM Run API key. Defaults to VLMRUN_API_KEY.
base_url
Gateway root URL. Defaults to VLMRUN_GATEWAY_BASE_URL, then VLMRUN_GATEWAY_URL, then https://gateway.vlm.run.
http_session
Optional caller-owned aiohttp session. Defaults to the job's HTTP session; outside a job, use async with utils.http_context.open().

Each attempt times out after 70 s by default, above the gateway's own deadline. Pass conn_options to evaluate() to change it.

Ancestors

  • livekit.agents.llm.decisions.DecisionModel
  • abc.ABC
  • EventEmitter
  • typing.Generic

Instance variables

prop model : str
Expand source code
@property
def model(self) -> str:
    return self._model
prop provider : str
Expand source code
@property
def provider(self) -> str:
    return self._provider

Methods

async def evaluate(self,
*,
input: str | llm.ChatContext,
decisions: Mapping[str, Decision],
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=70.0, liveness_timeout=6.0)) ‑> livekit.agents.llm.decisions.DecisionResponse
Expand source code
async def evaluate(
    self,
    *,
    input: str | llm.ChatContext,
    decisions: Mapping[str, Decision],
    conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS,
) -> DecisionResponse:
    """Evaluate a batch of decisions.

    Args:
        input: Text to evaluate, sent as-is, or a conversation converted to a
            transcript of user and assistant text.
        decisions: Decisions keyed by name.
        conn_options: Retry and per-attempt timeout settings. The timeout defaults to
            70 s, above the gateway's own deadline.

    Each requested name appears in either ``results`` or ``errors``. Request failures
    and malformed responses raise.
    """
    return await super().evaluate(
        input=input,
        decisions=decisions,
        conn_options=conn_options_or_default(conn_options),
    )

Evaluate a batch of decisions.

Args

input
Text to evaluate, sent as-is, or a conversation converted to a transcript of user and assistant text.
decisions
Decisions keyed by name.
conn_options
Retry and per-attempt timeout settings. The timeout defaults to 70 s, above the gateway's own deadline.

Each requested name appears in either results or errors. Request failures and malformed responses raise.

Inherited members

class LLM (*,
model: str | ChatModels = 'qwen/qwen3.8-27b',
api_key: NotGivenOr[str] = NOT_GIVEN,
base_url: NotGivenOr[str] = NOT_GIVEN,
client: openai.AsyncClient | None = None,
temperature: NotGivenOr[float] = NOT_GIVEN,
top_p: NotGivenOr[float] = NOT_GIVEN,
parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
max_completion_tokens: NotGivenOr[int] = NOT_GIVEN,
extra_body: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
max_retries: NotGivenOr[int] = NOT_GIVEN)
Expand source code
class LLM(OpenAILLM):
    def __init__(
        self,
        *,
        model: str | ChatModels = "qwen/qwen3.8-27b",
        api_key: NotGivenOr[str] = NOT_GIVEN,
        base_url: NotGivenOr[str] = NOT_GIVEN,
        client: openai.AsyncClient | None = None,
        temperature: NotGivenOr[float] = NOT_GIVEN,
        top_p: NotGivenOr[float] = NOT_GIVEN,
        parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
        tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
        max_completion_tokens: NotGivenOr[int] = NOT_GIVEN,
        extra_body: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
        max_retries: NotGivenOr[int] = NOT_GIVEN,
    ) -> None:
        """Create a new instance of a VLM Run gateway LLM.

        The VLM Run gateway serves open-weight and hosted vision-language models through an
        OpenAI-compatible Chat Completions API. Images in the chat context (``ImageContent``,
        including an ``rtc.VideoFrame``) are sent to the model. An ``AgentSession`` forwards
        live video only to realtime models, so the agent must add camera frames to the chat
        context itself.

        Requests time out after 70 s by default, above the gateway's first-token deadline. Pass
        ``conn_options`` to ``chat()``, or ``AgentSession(conn_options=...)``, to change it.

        ``api_key`` defaults to the ``VLMRUN_API_KEY`` environment variable and is required.
        ``base_url`` defaults to ``VLMRUN_GATEWAY_BASE_URL``, then
        ``VLMRUN_GATEWAY_URL``, then ``https://gateway.vlm.run``.

        Tool calling is supported by ``qwen/qwen3.8-27b`` and the hosted models (Gemini, Kimi,
        MiniMax, Muse), not by ``qwen/qwen3.5-0.8b`` or the Gemma models.

        Only parameters you set are sent. ``temperature`` and ``top_p`` are checked against what
        the model accepts, since the gateway rejects unsupported sampling fields.

        Args:
            model: Gateway model id. Defaults to ``qwen/qwen3.8-27b``.
            api_key: VLM Run API key. Defaults to ``VLMRUN_API_KEY``.
            base_url: Gateway root URL, without the ``/openai`` suffix.
            client: A preconfigured ``openai.AsyncClient``; overrides key and URL.
            temperature: Sampling temperature.
            top_p: Nucleus sampling.
            parallel_tool_calls: Allow parallel tool calls.
            tool_choice: Tool choice strategy.
            max_completion_tokens: Maximum tokens per reply.
            extra_body: Gateway-specific fields, such as ``method`` or ``method_params``.
            max_retries: Retries made by the openai client.
        """
        _check_sampling(model, temperature=temperature, top_p=top_p)

        key = resolve_api_key(api_key if is_given(api_key) else None)
        created_client = False
        if client is None:
            if not key:
                raise ValueError(
                    "VLMRUN_API_KEY is required, either as argument or set VLMRUN_API_KEY environment variable"  # noqa: E501
                )
            client = openai.AsyncClient(
                api_key=key,
                base_url=openai_base_url(base_url if is_given(base_url) else None),
                default_headers=client_headers(None),
                max_retries=max_retries if is_given(max_retries) else 0,
                http_client=httpx.AsyncClient(
                    follow_redirects=True,
                    limits=httpx.Limits(
                        max_connections=50,
                        max_keepalive_connections=50,
                        keepalive_expiry=120,
                    ),
                ),
            )
            created_client = True

        super().__init__(
            model=model,
            client=client,
            temperature=temperature,
            top_p=top_p,
            parallel_tool_calls=parallel_tool_calls,
            tool_choice=tool_choice,
            max_completion_tokens=max_completion_tokens,
            extra_body=extra_body,
            _strict_tool_schema=False,
        )

        if created_client:
            self._owns_client = True

    def chat(
        self,
        *,
        chat_ctx: ChatContext,
        tools: list[Tool] | None = None,
        conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS,
        parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
        tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
        response_format: NotGivenOr[Any] = NOT_GIVEN,
        extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
    ) -> LLMStream:
        return super().chat(
            chat_ctx=_gateway_chat_ctx(chat_ctx),
            tools=tools,
            conn_options=conn_options_or_default(conn_options),
            parallel_tool_calls=parallel_tool_calls,
            tool_choice=tool_choice,
            response_format=response_format,
            extra_kwargs=extra_kwargs,
        )

Helper class that provides a standard way to create an ABC using inheritance.

Create a new instance of a VLM Run gateway LLM.

The VLM Run gateway serves open-weight and hosted vision-language models through an OpenAI-compatible Chat Completions API. Images in the chat context (ImageContent, including an rtc.VideoFrame) are sent to the model. An AgentSession forwards live video only to realtime models, so the agent must add camera frames to the chat context itself.

Requests time out after 70 s by default, above the gateway's first-token deadline. Pass conn_options to chat(), or AgentSession(conn_options=...), to change it.

api_key defaults to the VLMRUN_API_KEY environment variable and is required. base_url defaults to VLMRUN_GATEWAY_BASE_URL, then VLMRUN_GATEWAY_URL, then https://gateway.vlm.run.

Tool calling is supported by qwen/qwen3.8-27b and the hosted models (Gemini, Kimi, MiniMax, Muse), not by qwen/qwen3.5-0.8b or the Gemma models.

Only parameters you set are sent. temperature and top_p are checked against what the model accepts, since the gateway rejects unsupported sampling fields.

Args

model
Gateway model id. Defaults to qwen/qwen3.8-27b.
api_key
VLM Run API key. Defaults to VLMRUN_API_KEY.
base_url
Gateway root URL, without the /openai suffix.
client
A preconfigured openai.AsyncClient; overrides key and URL.
temperature
Sampling temperature.
top_p
Nucleus sampling.
parallel_tool_calls
Allow parallel tool calls.
tool_choice
Tool choice strategy.
max_completion_tokens
Maximum tokens per reply.
extra_body
Gateway-specific fields, such as method or method_params.
max_retries
Retries made by the openai client.

Ancestors

  • livekit.plugins.openai.llm.LLM
  • livekit.agents.llm.llm.LLM
  • abc.ABC
  • EventEmitter
  • typing.Generic

Methods

def chat(self,
*,
chat_ctx: ChatContext,
tools: list[Tool] | None = None,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=70.0, liveness_timeout=6.0),
parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
response_format: NotGivenOr[Any] = NOT_GIVEN,
extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN) ‑> livekit.plugins.openai.llm.LLMStream
Expand source code
def chat(
    self,
    *,
    chat_ctx: ChatContext,
    tools: list[Tool] | None = None,
    conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS,
    parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
    tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
    response_format: NotGivenOr[Any] = NOT_GIVEN,
    extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
) -> LLMStream:
    return super().chat(
        chat_ctx=_gateway_chat_ctx(chat_ctx),
        tools=tools,
        conn_options=conn_options_or_default(conn_options),
        parallel_tool_calls=parallel_tool_calls,
        tool_choice=tool_choice,
        response_format=response_format,
        extra_kwargs=extra_kwargs,
    )

Inherited members