Module livekit.plugins.vlmrun
VLM Run plugin for LiveKit Agents
Vision-language models from the VLM Run gateway, through its OpenAI-compatible chat completions API, and typed decisions through its System One API.
Classes
class DecisionModel (*,
model: DecisionModels | str = 'qwen/qwen3.8-27b',
api_key: str | None = None,
base_url: str | None = None,
http_session: aiohttp.ClientSession | None = None)-
Expand source code
class DecisionModel(llm.DecisionModel): """Typed decisions from the VLM Run gateway's System One API. Example:: model = vlmrun.DecisionModel() response = await model.evaluate( input=chat_ctx, decisions={"handoff": llm.Probability("The caller requests a human.")}, ) Args: model: Gateway model id. Defaults to ``qwen/qwen3.8-27b``. api_key: VLM Run API key. Defaults to ``VLMRUN_API_KEY``. base_url: Gateway root URL. Defaults to ``VLMRUN_GATEWAY_BASE_URL``, then ``VLMRUN_GATEWAY_URL``, then ``https://gateway.vlm.run``. http_session: Optional caller-owned aiohttp session. Defaults to the job's HTTP session; outside a job, use ``async with utils.http_context.open()``. Each attempt times out after 70 s by default, above the gateway's own deadline. Pass ``conn_options`` to ``evaluate()`` to change it. """ def __init__( self, *, model: DecisionModels | str = "qwen/qwen3.8-27b", api_key: str | None = None, base_url: str | None = None, http_session: aiohttp.ClientSession | None = None, ) -> None: super().__init__( capabilities=DecisionCapabilities(decision_kinds=["probability", "choice", "score"]) ) key = resolve_api_key(api_key) if not key: raise ValueError( "VLMRUN_API_KEY is required, either as argument or set VLMRUN_API_KEY environment variable" # noqa: E501 ) self._model = model root = gateway_url(base_url) self._url = f"{root}/typesafe/v1/systemone" # Host and port only: netloc would carry user:password@ into telemetry. parts = urlsplit(root) host = parts.hostname or "" if ":" in host: host = f"[{host}]" self._provider = f"{host}:{parts.port}" if parts.port else host self._headers = client_headers(key) self._session = http_session @property def model(self) -> str: return self._model @property def provider(self) -> str: return self._provider async def evaluate( self, *, input: str | llm.ChatContext, decisions: Mapping[str, Decision], conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS, ) -> DecisionResponse: """Evaluate a batch of decisions. Args: input: Text to evaluate, sent as-is, or a conversation converted to a transcript of user and assistant text. decisions: Decisions keyed by name. conn_options: Retry and per-attempt timeout settings. The timeout defaults to 70 s, above the gateway's own deadline. Each requested name appears in either ``results`` or ``errors``. Request failures and malformed responses raise. """ return await super().evaluate( input=input, decisions=decisions, conn_options=conn_options_or_default(conn_options), ) async def _evaluate_impl( self, *, input: str, decisions: Mapping[str, Decision], conn_options: APIConnectOptions, ) -> DecisionResponse: questions = {name: systemone.encode_question(d) for name, d in decisions.items()} session = self._session or utils.http_context.http_session() try: async with session.post( self._url, headers=self._headers, json={"model": self._model, "state": input, "questions": questions}, timeout=aiohttp.ClientTimeout(total=conn_options.timeout), allow_redirects=False, ) as response: request_id = response.headers.get("x-request-id", "") if response.status != 200: raise APIStatusError( "VLM Run decision request failed", status_code=response.status, request_id=request_id or None, body=await response.text(), retryable=response.status in (408, 429) or 500 <= response.status < 600, ) parsed = systemone.Response.model_validate(await response.json()) request_id = parsed.id or request_id results, errors = systemone.decode_answers( parsed.answers, invalid="invalid VLM Run decision answer" ) return DecisionResponse( results=results, errors=errors, model=parsed.model, provider=self.provider, request_id=request_id, input_tokens=parsed.usage.input_tokens, output_tokens=parsed.usage.output_tokens, ) except asyncio.TimeoutError: raise APITimeoutError("VLM Run decision request timed out") from None except aiohttp.ContentTypeError: raise APIError("invalid VLM Run decision response", retryable=False) from None except (aiohttp.ClientError, ConnectionError, OSError): raise APIConnectionError("VLM Run decision transport failed") from None except ValueError: raise APIError("invalid VLM Run decision response", retryable=False) from NoneTyped decisions from the VLM Run gateway's System One API.
Example::
model = vlmrun.DecisionModel() response = await model.evaluate( input=chat_ctx, decisions={"handoff": llm.Probability("The caller requests a human.")}, )Args
model- Gateway model id. Defaults to
qwen/qwen3.8-27b. api_key- VLM Run API key. Defaults to
VLMRUN_API_KEY. base_url- Gateway root URL. Defaults to
VLMRUN_GATEWAY_BASE_URL, thenVLMRUN_GATEWAY_URL, thenhttps://gateway.vlm.run. http_session- Optional caller-owned aiohttp session. Defaults to the job's HTTP
session; outside a job, use
async with utils.http_context.open().
Each attempt times out after 70 s by default, above the gateway's own deadline. Pass
conn_optionstoevaluate()to change it.Ancestors
- livekit.agents.llm.decisions.DecisionModel
- abc.ABC
- EventEmitter
- typing.Generic
Instance variables
prop model : str-
Expand source code
@property def model(self) -> str: return self._model prop provider : str-
Expand source code
@property def provider(self) -> str: return self._provider
Methods
async def evaluate(self,
*,
input: str | llm.ChatContext,
decisions: Mapping[str, Decision],
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=70.0, liveness_timeout=6.0)) ‑> livekit.agents.llm.decisions.DecisionResponse-
Expand source code
async def evaluate( self, *, input: str | llm.ChatContext, decisions: Mapping[str, Decision], conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS, ) -> DecisionResponse: """Evaluate a batch of decisions. Args: input: Text to evaluate, sent as-is, or a conversation converted to a transcript of user and assistant text. decisions: Decisions keyed by name. conn_options: Retry and per-attempt timeout settings. The timeout defaults to 70 s, above the gateway's own deadline. Each requested name appears in either ``results`` or ``errors``. Request failures and malformed responses raise. """ return await super().evaluate( input=input, decisions=decisions, conn_options=conn_options_or_default(conn_options), )Evaluate a batch of decisions.
Args
input- Text to evaluate, sent as-is, or a conversation converted to a transcript of user and assistant text.
decisions- Decisions keyed by name.
conn_options- Retry and per-attempt timeout settings. The timeout defaults to 70 s, above the gateway's own deadline.
Each requested name appears in either
resultsorerrors. Request failures and malformed responses raise.
Inherited members
class LLM (*,
model: str | ChatModels = 'qwen/qwen3.8-27b',
api_key: NotGivenOr[str] = NOT_GIVEN,
base_url: NotGivenOr[str] = NOT_GIVEN,
client: openai.AsyncClient | None = None,
temperature: NotGivenOr[float] = NOT_GIVEN,
top_p: NotGivenOr[float] = NOT_GIVEN,
parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
max_completion_tokens: NotGivenOr[int] = NOT_GIVEN,
extra_body: NotGivenOr[dict[str, Any]] = NOT_GIVEN,
max_retries: NotGivenOr[int] = NOT_GIVEN)-
Expand source code
class LLM(OpenAILLM): def __init__( self, *, model: str | ChatModels = "qwen/qwen3.8-27b", api_key: NotGivenOr[str] = NOT_GIVEN, base_url: NotGivenOr[str] = NOT_GIVEN, client: openai.AsyncClient | None = None, temperature: NotGivenOr[float] = NOT_GIVEN, top_p: NotGivenOr[float] = NOT_GIVEN, parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN, tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN, max_completion_tokens: NotGivenOr[int] = NOT_GIVEN, extra_body: NotGivenOr[dict[str, Any]] = NOT_GIVEN, max_retries: NotGivenOr[int] = NOT_GIVEN, ) -> None: """Create a new instance of a VLM Run gateway LLM. The VLM Run gateway serves open-weight and hosted vision-language models through an OpenAI-compatible Chat Completions API. Images in the chat context (``ImageContent``, including an ``rtc.VideoFrame``) are sent to the model. An ``AgentSession`` forwards live video only to realtime models, so the agent must add camera frames to the chat context itself. Requests time out after 70 s by default, above the gateway's first-token deadline. Pass ``conn_options`` to ``chat()``, or ``AgentSession(conn_options=...)``, to change it. ``api_key`` defaults to the ``VLMRUN_API_KEY`` environment variable and is required. ``base_url`` defaults to ``VLMRUN_GATEWAY_BASE_URL``, then ``VLMRUN_GATEWAY_URL``, then ``https://gateway.vlm.run``. Tool calling is supported by ``qwen/qwen3.8-27b`` and the hosted models (Gemini, Kimi, MiniMax, Muse), not by ``qwen/qwen3.5-0.8b`` or the Gemma models. Only parameters you set are sent. ``temperature`` and ``top_p`` are checked against what the model accepts, since the gateway rejects unsupported sampling fields. Args: model: Gateway model id. Defaults to ``qwen/qwen3.8-27b``. api_key: VLM Run API key. Defaults to ``VLMRUN_API_KEY``. base_url: Gateway root URL, without the ``/openai`` suffix. client: A preconfigured ``openai.AsyncClient``; overrides key and URL. temperature: Sampling temperature. top_p: Nucleus sampling. parallel_tool_calls: Allow parallel tool calls. tool_choice: Tool choice strategy. max_completion_tokens: Maximum tokens per reply. extra_body: Gateway-specific fields, such as ``method`` or ``method_params``. max_retries: Retries made by the openai client. """ _check_sampling(model, temperature=temperature, top_p=top_p) key = resolve_api_key(api_key if is_given(api_key) else None) created_client = False if client is None: if not key: raise ValueError( "VLMRUN_API_KEY is required, either as argument or set VLMRUN_API_KEY environment variable" # noqa: E501 ) client = openai.AsyncClient( api_key=key, base_url=openai_base_url(base_url if is_given(base_url) else None), default_headers=client_headers(None), max_retries=max_retries if is_given(max_retries) else 0, http_client=httpx.AsyncClient( follow_redirects=True, limits=httpx.Limits( max_connections=50, max_keepalive_connections=50, keepalive_expiry=120, ), ), ) created_client = True super().__init__( model=model, client=client, temperature=temperature, top_p=top_p, parallel_tool_calls=parallel_tool_calls, tool_choice=tool_choice, max_completion_tokens=max_completion_tokens, extra_body=extra_body, _strict_tool_schema=False, ) if created_client: self._owns_client = True def chat( self, *, chat_ctx: ChatContext, tools: list[Tool] | None = None, conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS, parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN, tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN, response_format: NotGivenOr[Any] = NOT_GIVEN, extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN, ) -> LLMStream: return super().chat( chat_ctx=_gateway_chat_ctx(chat_ctx), tools=tools, conn_options=conn_options_or_default(conn_options), parallel_tool_calls=parallel_tool_calls, tool_choice=tool_choice, response_format=response_format, extra_kwargs=extra_kwargs, )Helper class that provides a standard way to create an ABC using inheritance.
Create a new instance of a VLM Run gateway LLM.
The VLM Run gateway serves open-weight and hosted vision-language models through an OpenAI-compatible Chat Completions API. Images in the chat context (
ImageContent, including anrtc.VideoFrame) are sent to the model. AnAgentSessionforwards live video only to realtime models, so the agent must add camera frames to the chat context itself.Requests time out after 70 s by default, above the gateway's first-token deadline. Pass
conn_optionstochat(), orAgentSession(conn_options=...), to change it.api_keydefaults to theVLMRUN_API_KEYenvironment variable and is required.base_urldefaults toVLMRUN_GATEWAY_BASE_URL, thenVLMRUN_GATEWAY_URL, thenhttps://gateway.vlm.run.Tool calling is supported by
qwen/qwen3.8-27band the hosted models (Gemini, Kimi, MiniMax, Muse), not byqwen/qwen3.5-0.8bor the Gemma models.Only parameters you set are sent.
temperatureandtop_pare checked against what the model accepts, since the gateway rejects unsupported sampling fields.Args
model- Gateway model id. Defaults to
qwen/qwen3.8-27b. api_key- VLM Run API key. Defaults to
VLMRUN_API_KEY. base_url- Gateway root URL, without the
/openaisuffix. client- A preconfigured
openai.AsyncClient; overrides key and URL. temperature- Sampling temperature.
top_p- Nucleus sampling.
parallel_tool_calls- Allow parallel tool calls.
tool_choice- Tool choice strategy.
max_completion_tokens- Maximum tokens per reply.
extra_body- Gateway-specific fields, such as
methodormethod_params. max_retries- Retries made by the openai client.
Ancestors
- livekit.plugins.openai.llm.LLM
- livekit.agents.llm.llm.LLM
- abc.ABC
- EventEmitter
- typing.Generic
Methods
def chat(self,
*,
chat_ctx: ChatContext,
tools: list[Tool] | None = None,
conn_options: APIConnectOptions = APIConnectOptions(max_retry=3, retry_interval=2.0, timeout=70.0, liveness_timeout=6.0),
parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN,
tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN,
response_format: NotGivenOr[Any] = NOT_GIVEN,
extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN) ‑> livekit.plugins.openai.llm.LLMStream-
Expand source code
def chat( self, *, chat_ctx: ChatContext, tools: list[Tool] | None = None, conn_options: APIConnectOptions = DEFAULT_CONN_OPTIONS, parallel_tool_calls: NotGivenOr[bool] = NOT_GIVEN, tool_choice: NotGivenOr[ToolChoice] = NOT_GIVEN, response_format: NotGivenOr[Any] = NOT_GIVEN, extra_kwargs: NotGivenOr[dict[str, Any]] = NOT_GIVEN, ) -> LLMStream: return super().chat( chat_ctx=_gateway_chat_ctx(chat_ctx), tools=tools, conn_options=conn_options_or_default(conn_options), parallel_tool_calls=parallel_tool_calls, tool_choice=tool_choice, response_format=response_format, extra_kwargs=extra_kwargs, )
Inherited members