Agents v0.x migration guide - Node.js

import {
  type JobContext,
  WorkerOptions,
  defineAgent,
  llm,
  pipeline,
} from '@livekit/agents';
import * as deepgram from '@livekit/agents-plugin-deepgram';
import * as livekit from '@livekit/agents-plugin-livekit';
import * as openai from '@livekit/agents-plugin-openai';
import * as silero from '@livekit/agents-plugin-silero';

export default defineAgent({
  entry: async (ctx: JobContext) => {
    const vad = await silero.VAD.load() as silero.VAD;
    const initialContext = new llm.ChatContext().append({
      role: llm.ChatRole.SYSTEM,
      text: 'You are a helpful voice AI assistant.',
    });

    const agent = new pipeline.VoicePipelineAgent(
      vad,
      new deepgram.STT(),
      new openai.LLM(),
      new openai.TTS(),
      { chatCtx: initialContext, fncCtx, turnDetector: new livekit.turnDetector.EOUModel() },
    );
    
    await agent.start(ctx.room, participant);

    await agent.say('Hey, how can I help you today?', true);
  },
});

import {
  type JobContext,
  defineAgent,
  voice,
} from '@livekit/agents';
import * as deepgram from '@livekit/agents-plugin-deepgram';
import * as elevenlabs from '@livekit/agents-plugin-elevenlabs';
import * as livekit from '@livekit/agents-plugin-livekit';
import * as openai from '@livekit/agents-plugin-openai';
import * as silero from '@livekit/agents-plugin-silero';
import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node';

export default defineAgent({
  entry: async (ctx: JobContext) => {
    const agent = new voice.Agent({
      instructions:
        "You are a helpful voice AI assistant.",
    });

    const vad = await silero.VAD.load() as silero.VAD;

    const session = new voice.AgentSession({
      vad,
      stt: new deepgram.STT(),
      tts: new elevenlabs.TTS(),
      llm: new openai.LLM(),
      turnDetection: new livekit.turnDetector.MultilingualModel(),
    });
    
    // if using realtime api, use the following
    // session = AgentSession({
    //   llm: new openai.realtime.RealtimeModel({ voice: "echo" })
    // })

    await session.start({
      room: ctx.room,
      agent,
      inputOptions: {
        noiseCancellation: BackgroundVoiceCancellation(),
      },
    });

    await ctx.connect();

    // Instruct the agent to speak first
    const handle = session.generateReply('say hello to the user');
    await handle.waitForPlayout();
  },
});

const addRagContext: BeforeLLMCallback = (agent, chatCtx) => {
  const ragContext: string = retrieve(chatCtx);
  chatCtx.append({ text: ragContext, role: llm.ChatRole.SYSTEM });
};

const agent = new VoicePipelineAgent(
  ...
  { 
    ...
    beforeLLMCallback: addRagContext 
  }
);

class MyAgent extends voice.Agent {
  // override method from superclass to customize behavior
  async llmNode(
    chatCtx: llm.ChatContext,
    toolCtx: llm.ToolContext,
    modelSettings: voice.ModelSettings,
  ): Promise<ReadableStream<llm.ChatChunk | string> | null> {
    const ragContext: string = retrieve(chatCtx);
    chatCtx.addMessage({ content: ragContext, role: 'system' });

    return voice.Agent.default.llmNode(this, chatCtx, toolCtx, modelSettings);
  }
}

const beforeTtsCb: BeforeTTSCallback = (agent, source) => {
  // The TTS is incorrectly pronouncing "LiveKit", so we'll replace it
  if (typeof source === 'string') {
    return source.replace(/\bLiveKit\b/gi, 'Live Kit');
  }
  
  return (async function* () {
    for await (const chunk of source) {
      yield chunk.replace(/\bLiveKit\b/gi, 'Live Kit');
    }
  })();
};

const agent = new VoicePipelineAgent(
  ...
  { 
    ...
    beforeTTSCallback: beforeTtsCb 
  }
);

class MyAgent extends voice.Agent {
  async ttsNode(
    text: ReadableStream<string>,
    modelSettings: voice.ModelSettings,
  ): Promise<ReadableStream<AudioFrame> | null> {
    const replaceWords = (text: ReadableStream<string>): ReadableStream<string> => {
      // ...
    };

    // use default implementation, but pre-process the text
    return voice.Agent.default.ttsNode(this, replaceWords(text), modelSettings);
  }
}

import { llm, pipeline } from '@livekit/agents';
import { z } from 'zod';

const fncCtx: llm.FunctionContext = {
  getWeather: {
    description: 'Get weather information for a location',
    parameters: z.object({
      location: z.string(),
    }),
    execute: async ({ location }) => {
      ...
      return `The weather in ${location} right now is Sunny.`;
    },
  },
};

const agent = new pipeline.VoicePipelineAgent(
  ...
  { 
    ...
    fncCtx,
  }
);

import { llm, voice } from '@livekit/agents';
import { z } from 'zod';

const agent = new voice.Agent({
  instructions: "You are a helpful assistant.",
  tools: {
    getWeather: llm.tool({
      description: 'Look up weather information for a given location.',
      parameters: z.object({
        location: z.string().describe('The location to look up weather information for.'),
      }),
      execute: async ({ location }, { ctx }) => {
        return { weather: "sunny", temperatureF: 70 };
      },
    }),
  },
});

import { pipeline } from '@livekit/agents';

agent.on(pipeline.VPAEvent.USER_STARTED_SPEAKING, () => {
  console.log("User started speaking");
});

session.on(voice.AgentSessionEventTypes.UserStateChanged, (ev) => {
  // userState could be "speaking", "listening", or "away"
  console.log(`state change from ${ev.oldState} to ${ev.newState}`);
});

import { pipeline } from '@livekit/agents';

agent.on(pipeline.VPAEvent.AGENT_STARTED_SPEAKING, () => {
  // Log transcribed message from user
  console.log("Agent started speaking");
});

session.on(voice.AgentSessionEventTypes.AgentStateChanged, (ev) => {
  // AgentState could be "initializing", "idle", "listening", "thinking", "speaking"
  // newState is set as a participant attribute `lk.agent.state` to notify frontends
  console.log(`state change from ${ev.oldState} to ${ev.newState}`);
});