feat: add agent evals and migrate to vite/vitest

.github/workflows/template-check.yml

@@ -1,6 +1,6 @@
 # As this is a starter template project, we don't want to check in the pnpm-lock.yaml and livekit.toml files in its template form
 # However, once you have cloned this repo for your own use, LiveKit recommends you check them in and delete this github workflow entirely
-    
+
 name: Template Check
 
 on:
@@ -12,10 +12,10 @@ on:
 jobs:
   check-template-files:
     runs-on: ubuntu-latest
-    
+
     steps:
     - uses: actions/checkout@v4
-    
+
     - name: Check template files not tracked in git
       run: |
         if git ls-files | grep -q "^pnpm-lock.yaml$"; then
@@ -28,4 +28,4 @@ jobs:
           echo "Disable this test and commit the file once you have cloned this repo for your own use"
           exit 1
         fi
-        echo "✓ pnpm-lock.yaml and livekit.toml are correctly not tracked in git" 
+        echo "✓ pnpm-lock.yaml and livekit.toml are correctly not tracked in git" 

.github/workflows/tests.yml

@@ -14,11 +14,11 @@ jobs:
   test:
     name: Test (Node.js ${{ matrix.node-version }})
     runs-on: ubuntu-latest
-    
+
     strategy:
       matrix:
         node-version: [22]
-    
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -40,12 +40,16 @@ jobs:
         run: pnpm run build
 
       - name: Run tests
+        env:
+          LIVEKIT_API_KEY: ${{ secrets.LIVEKIT_API_KEY }}
+          LIVEKIT_API_SECRET: ${{ secrets.LIVEKIT_API_SECRET }}
+          LIVEKIT_URL: ${{ secrets.LIVEKIT_URL }}
         run: pnpm run test
-    
+
   format:
     name: Formatting & Linting
     runs-on: ubuntu-latest
-    
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

AGENTS.md

@@ -8,7 +8,7 @@ The following is a guide for working with this project.
 
 This Node.js project uses the `pnpm` package manager. You should always use `pnpm` to install dependencies, run the agent, and run tests.
 
-All app-level code is in the `src/` directory. In general, simple agents can be constructed with a single `agent.ts` file. Additional files can be added, but you must retain `agent.ts` as the entrypoint (see the associated Dockerfile for how this is deployed).
+All app-level code is in the `src/` directory. In general, simple agents can be constructed with a single `main.ts` file. Additional files can be added, but you must retain `main.ts` as the entrypoint (see the associated Dockerfile for how this is deployed).
 
 Be sure to maintain code formatting. You can use the prettier formatter and eslint to format and lint the code. Scripts are available in `package.json`, including `pnpm format` and `pnpm lint`.
 
@@ -46,6 +46,16 @@ If you are another agentic IDE, refer to your own documentation for how to insta
 
 Voice AI agents are highly sensitive to excessive latency. For this reason, it's important to design complex agents in a structured manner that minimizes the amount of irrelevant context and unnecessary tools present on requests to the LLM. LiveKit Agents supports handoffs (one agent hands control to another) to support building reliable workflows. You should make use of these features, instead of writing long instruction prompts that cover multiple phases of a conversation. Refer to the [documentation](https://docs.livekit.io/agents/build/workflows/) for more information.
 
+### Submit docs feedback to LiveKit
+
+If you use the LiveKit Docs MCP Server to search or browse documentation, also submit constructive feedback to LiveKit with the submit_docs_feedback tool. Add this to your to-do list when planning work, to ensure you file feedback after completing your other tasks. Useful feedback includes incomplete documentation, broken examples, missing steps or any other documentation gap or issue.
+
+## Testing
+
+When possible, add tests for agent behavior. Read the [documentation](https://docs.livekit.io/agents/build/testing/), and refer to existing test files with the `.test.ts` extension. Run tests with `pnpm test`.
+
+Important: When modifying core agent behavior such as instructions, tool descriptions, and tasks/workflows/handoffs, never just guess what will work. Always use test-driven development (TDD) and begin by writing tests for the desired behavior. For instance, if you're planning to add a new tool, write one or more tests for the tool's behavior, then iterate on the tool until the tests pass correctly. This will ensure you are able to produce a working, reliable agent for the user.
+
 ## Feature parity with Python SDK
 
 The Node.js SDK for LiveKit Agents has most, but not all, of the same features available in Python SDK for LiveKit Agents. You should always check the documentation for feature availability, and avoid using features that are not available in the Node.js SDK.

package.json

@@ -4,38 +4,39 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "build": "tsc",
+    "build": "vite build",
     "clean": "rm -rf dist",
     "typecheck": "tsc --noEmit",
     "lint": "eslint \"**/*.{ts,js}\"",
     "lint:fix": "eslint --fix \"**/*.{ts,js}\"",
     "format": "prettier --write \"**/*.{ts,js,json,md}\"",
     "format:check": "prettier --check \"**/*.{ts,js,json,md}\"",
-    "test": "node --test --import tsx",
-    "test:watch": "node --test --watch --import tsx",
-    "dev": "tsx src/agent.ts dev",
-    "download-files": "pnpm run build && node dist/agent.js download-files",
-    "start": "node dist/agent.js start"
+    "test": "vitest --run",
+    "test:watch": "vitest",
+    "dev": "pnpm run build && node dist/main.js dev",
+    "download-files": "pnpm run build && node dist/main.js download-files",
+    "start": "node dist/main.js start"
   },
   "engines": {
     "node": ">=22.0.0",
     "pnpm": ">=10.0.0"
   },
   "devDependencies": {
-    "@eslint/js": "^9.38.0",
+    "@eslint/js": "^9.39.2",
     "@trivago/prettier-plugin-sort-imports": "^5.2.2",
-    "@types/node": "^22.18.11",
-    "eslint": "^9.38.0",
-    "globals": "^16.4.0",
+    "@types/node": "^22.19.7",
+    "eslint": "^9.39.2",
+    "globals": "^16.5.0",
     "jiti": "^2.6.1",
-    "tsx": "^4.20.6",
     "typescript": "^5.9.3",
-    "typescript-eslint": "^8.46.2"
+    "typescript-eslint": "^8.54.0",
+    "vite": "^7.3.1",
+    "vitest": "^4.0.18"
   },
   "dependencies": {
-    "@livekit/agents": "^1.0.21",
-    "@livekit/agents-plugin-livekit": "^1.0.21",
-    "@livekit/agents-plugin-silero": "^1.0.21",
+    "@livekit/agents": "^1.0.40",
+    "@livekit/agents-plugin-livekit": "^1.0.40",
+    "@livekit/agents-plugin-silero": "^1.0.40",
     "@livekit/noise-cancellation-node": "^0.1.9",
     "dotenv": "^17.2.3",
     "zod": "^3.25.76"

src/agent.test.ts

@@ -1,24 +1,46 @@
-import assert from 'node:assert';
-import { describe, it } from 'node:test';
-
-describe('Agent Configuration', () => {
-  it('should have Node.js environment available', () => {
-    // Basic test to ensure Node.js environment is working
-    assert.ok(typeof process !== 'undefined', 'Process should be available');
-    assert.ok(typeof process.env !== 'undefined', 'Environment variables should be accessible');
-  });
+import { inference, initializeLogger, voice } from '@livekit/agents';
+import dotenv from 'dotenv';
+import { afterEach, beforeEach, describe, it } from 'vitest';
+import { Assistant } from './agent.js';
+
+dotenv.config({ path: '.env.local' });
+
+// Initialize logger for testing
+initializeLogger({ pretty: false, level: 'debug' });
+
+describe('agent evaluation', () => {
+  let session: voice.AgentSession;
+  let llmInstance: inference.LLM;
 
-  it('should be able to import required modules', async () => {
-    // Test that core Node.js modules work
-    const path = await import('node:path');
-    const url = await import('node:url');
+  beforeEach(async () => {
+    llmInstance = new inference.LLM({ model: 'openai/gpt-5.1' });
+    session = new voice.AgentSession({ llm: llmInstance });
+    await session.start({ agent: new Assistant() });
+  });
 
-    assert.ok(typeof path.dirname === 'function', 'Path module should be available');
-    assert.ok(typeof url.fileURLToPath === 'function', 'URL module should be available');
+  afterEach(async () => {
+    await session?.close();
   });
 
-  it('should have TypeScript compilation working', () => {
-    // This test file being run means TypeScript compiled successfully
-    assert.ok(true, 'TypeScript compilation is working');
+  it('offers assistance', { timeout: 30000 }, async () => {
+    // Run an agent turn following the user's greeting
+    const result = await session.run({ userInput: 'Hello' }).wait();
+
+    // Evaluate the agent's response for friendliness
+    await result.expect
+      .nextEvent()
+      .isMessage({ role: 'assistant' })
+      .judge(llmInstance, {
+        intent: `\
+Greets the user in a friendly manner.
+
+Optional context that may or may not be included:
+- Offer of assistance with any request the user may have
+- Other small talk or chit chat is acceptable, so long as it is friendly and not too intrusive
+`,
+      });
+
+    // Assert that there are no unexpected further events
+    result.expect.noMoreEvents();
   });
 });

src/agent.ts

@@ -1,22 +1,7 @@
-import {
-  type JobContext,
-  type JobProcess,
-  ServerOptions,
-  cli,
-  defineAgent,
-  inference,
-  metrics,
-  voice,
-} from '@livekit/agents';
-import * as livekit from '@livekit/agents-plugin-livekit';
-import * as silero from '@livekit/agents-plugin-silero';
-import { BackgroundVoiceCancellation } from '@livekit/noise-cancellation-node';
-import dotenv from 'dotenv';
-import { fileURLToPath } from 'node:url';
+import { voice } from '@livekit/agents';
 
-dotenv.config({ path: '.env.local' });
-
-class Assistant extends voice.Agent {
+// Define a custom voice AI assistant by extending the base Agent class
+export class Assistant extends voice.Agent {
   constructor() {
     super({
       instructions: `You are a helpful voice AI assistant. The user is interacting with you via voice, even if you perceive the conversation as text.
@@ -47,84 +32,3 @@ class Assistant extends voice.Agent {
     });
   }
 }
-
-export default defineAgent({
-  prewarm: async (proc: JobProcess) => {
-    proc.userData.vad = await silero.VAD.load();
-  },
-  entry: async (ctx: JobContext) => {
-    // Set up a voice AI pipeline using OpenAI, Cartesia, AssemblyAI, and the LiveKit turn detector
-    const session = new voice.AgentSession({
-      // Speech-to-text (STT) is your agent's ears, turning the user's speech into text that the LLM can understand
-      // See all available models at https://docs.livekit.io/agents/models/stt/
-      stt: new inference.STT({
-        model: 'assemblyai/universal-streaming',
-        language: 'en',
-      }),
-
-      // A Large Language Model (LLM) is your agent's brain, processing user input and generating a response
-      // See all providers at https://docs.livekit.io/agents/models/llm/
-      llm: new inference.LLM({
-        model: 'openai/gpt-4.1-mini',
-      }),
-
-      // Text-to-speech (TTS) is your agent's voice, turning the LLM's text into speech that the user can hear
-      // See all available models as well as voice selections at https://docs.livekit.io/agents/models/tts/
-      tts: new inference.TTS({
-        model: 'cartesia/sonic-3',
-        voice: '9626c31c-bec5-4cca-baa8-f8ba9e84c8bc',
-      }),
-
-      // VAD and turn detection are used to determine when the user is speaking and when the agent should respond
-      // See more at https://docs.livekit.io/agents/build/turns
-      turnDetection: new livekit.turnDetector.MultilingualModel(),
-      vad: ctx.proc.userData.vad! as silero.VAD,
-      voiceOptions: {
-        // Allow the LLM to generate a response while waiting for the end of turn
-        preemptiveGeneration: true,
-      },
-    });
-
-    // To use a realtime model instead of a voice pipeline, use the following session setup instead.
-    // (Note: This is for the OpenAI Realtime API. For other providers, see https://docs.livekit.io/agents/models/realtime/))
-    // 1. Install '@livekit/agents-plugin-openai'
-    // 2. Set OPENAI_API_KEY in .env.local
-    // 3. Add import `import * as openai from '@livekit/agents-plugin-openai'` to the top of this file
-    // 4. Use the following session setup instead of the version above
-    // const session = new voice.AgentSession({
-    //   llm: new openai.realtime.RealtimeModel({ voice: 'marin' }),
-    // });
-
-    // Metrics collection, to measure pipeline performance
-    // For more information, see https://docs.livekit.io/agents/build/metrics/
-    const usageCollector = new metrics.UsageCollector();
-    session.on(voice.AgentSessionEventTypes.MetricsCollected, (ev) => {
-      metrics.logMetrics(ev.metrics);
-      usageCollector.collect(ev.metrics);
-    });
-
-    const logUsage = async () => {
-      const summary = usageCollector.getSummary();
-      console.log(`Usage: ${JSON.stringify(summary)}`);
-    };
-
-    ctx.addShutdownCallback(logUsage);
-
-    // Start the session, which initializes the voice pipeline and warms up the models
-    await session.start({
-      agent: new Assistant(),
-      room: ctx.room,
-      inputOptions: {
-        // LiveKit Cloud enhanced noise cancellation
-        // - If self-hosting, omit this parameter
-        // - For telephony applications, use `BackgroundVoiceCancellationTelephony` for best results
-        noiseCancellation: BackgroundVoiceCancellation(),
-      },
-    });
-
-    // Join the room and connect to the user
-    await ctx.connect();
-  },
-});
-
-cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) }));