feat: LiveKit AI answering agent (Gemini 2.5 Flash native audio)

- Hospital receptionist agent "Helix" with Gemini realtime speech-to-speech
- Tools wired to platform: lookupDoctor, bookAppointment, collectLeadInfo, transferToAgent
- Loads hospital context (doctors, departments) from platform GraphQL on startup
- Connects to LiveKit Cloud, joins rooms when participants connect
- Silero VAD for voice activity detection
- @livekit/agents + @livekit/agents-plugin-google + @livekit/agents-plugin-silero

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

package.json

@@ -23,6 +23,9 @@
     "@ai-sdk/anthropic": "^3.0.58",
     "@ai-sdk/openai": "^3.0.41",
     "@deepgram/sdk": "^5.0.0",
+    "@livekit/agents": "^1.2.1",
+    "@livekit/agents-plugin-google": "^1.2.1",
+    "@livekit/agents-plugin-silero": "^1.2.1",
     "@nestjs/common": "^11.0.1",
     "@nestjs/config": "^4.0.3",
     "@nestjs/core": "^11.0.1",
@@ -34,7 +37,8 @@
     "ioredis": "^5.10.1",
     "reflect-metadata": "^0.2.2",
     "rxjs": "^7.8.1",
-    "socket.io": "^4.8.3"
+    "socket.io": "^4.8.3",
+    "zod": "^4.3.6"
   },
   "devDependencies": {
     "@eslint/eslintrc": "^3.2.0",

src/auth/auth.controller.ts

@@ -116,55 +116,54 @@ export class AuthController {
 
             this.logger.log(`User ${body.email} logged in with role: ${appRole} (platform roles: ${roleLabels.join(', ')})`);
 
-            // Multi-agent: resolve agent config + session lock for CC agents
+            // Check if user has an Agent entity with SIP config — applies to ALL roles
             let agentConfigResponse: any = undefined;
+            const memberId = workspaceMember?.id;
 
-            if (appRole === 'cc-agent') {
+            if (memberId) {
-                const memberId = workspaceMember?.id;
-                if (!memberId) throw new HttpException('Workspace member not found', 400);
-
                 const agentConfig = await this.agentConfigService.getByMemberId(memberId);
-                if (!agentConfig) {
+
+                if (agentConfig) {
+                    // Agent entity found — set up SIP + Ozonetel
+                    const clientIp = (req.headers['x-forwarded-for'] as string)?.split(',')[0]?.trim() ?? req.ip ?? 'unknown';
+                    const existingSession = await this.sessionService.getSession(agentConfig.ozonetelAgentId);
+                    if (existingSession) {
+                        this.logger.warn(`Duplicate login blocked for ${body.email} — session held by IP ${existingSession.ip} since ${existingSession.lockedAt}`);
+                        throw new HttpException(`You are already logged in from another device (${existingSession.ip}). Please log out there first.`, 409);
+                    }
+
+                    await this.sessionService.lockSession(agentConfig.ozonetelAgentId, memberId, clientIp);
+
+                    this.ozonetelAgent.refreshToken().catch(err => {
+                        this.logger.warn(`Ozonetel token refresh on login failed: ${err.message}`);
+                    });
+
+                    const ozAgentPassword = process.env.OZONETEL_AGENT_PASSWORD ?? 'Test123$';
+                    this.ozonetelAgent.loginAgent({
+                        agentId: agentConfig.ozonetelAgentId,
+                        password: ozAgentPassword,
+                        phoneNumber: agentConfig.sipExtension,
+                        mode: 'blended',
+                    }).catch(err => {
+                        this.logger.warn(`Ozonetel agent login failed (non-blocking): ${err.message}`);
+                    });
+
+                    agentConfigResponse = {
+                        ozonetelAgentId: agentConfig.ozonetelAgentId,
+                        sipExtension: agentConfig.sipExtension,
+                        sipPassword: agentConfig.sipPassword,
+                        sipUri: agentConfig.sipUri,
+                        sipWsServer: agentConfig.sipWsServer,
+                        campaignName: agentConfig.campaignName,
+                    };
+
+                    this.logger.log(`Agent ${body.email} → Ozonetel ${agentConfig.ozonetelAgentId} / SIP ${agentConfig.sipExtension}`);
+                } else if (appRole === 'cc-agent') {
+                    // CC agent role but no Agent entity — block login
                     throw new HttpException('Agent account not configured. Contact administrator.', 403);
+                } else {
+                    this.logger.log(`User ${body.email} has no Agent entity — SIP disabled`);
                 }
-
-                // Check for duplicate login — strict: one device only
-                const clientIp = (req.headers['x-forwarded-for'] as string)?.split(',')[0]?.trim() ?? req.ip ?? 'unknown';
-                const existingSession = await this.sessionService.getSession(agentConfig.ozonetelAgentId);
-                if (existingSession) {
-                    this.logger.warn(`Duplicate login blocked for ${body.email} — session held by IP ${existingSession.ip} since ${existingSession.lockedAt}`);
-                    throw new HttpException(`You are already logged in from another device (${existingSession.ip}). Please log out there first.`, 409);
-                }
-
-                // Lock session in Redis with IP
-                await this.sessionService.lockSession(agentConfig.ozonetelAgentId, memberId, clientIp);
-
-                // Force-refresh Ozonetel API token on login
-                this.ozonetelAgent.refreshToken().catch(err => {
-                    this.logger.warn(`Ozonetel token refresh on login failed: ${err.message}`);
-                });
-
-                // Login to Ozonetel with agent-specific credentials
-                const ozAgentPassword = process.env.OZONETEL_AGENT_PASSWORD ?? 'Test123$';
-                this.ozonetelAgent.loginAgent({
-                    agentId: agentConfig.ozonetelAgentId,
-                    password: ozAgentPassword,
-                    phoneNumber: agentConfig.sipExtension,
-                    mode: 'blended',
-                }).catch(err => {
-                    this.logger.warn(`Ozonetel agent login failed (non-blocking): ${err.message}`);
-                });
-
-                agentConfigResponse = {
-                    ozonetelAgentId: agentConfig.ozonetelAgentId,
-                    sipExtension: agentConfig.sipExtension,
-                    sipPassword: agentConfig.sipPassword,
-                    sipUri: agentConfig.sipUri,
-                    sipWsServer: agentConfig.sipWsServer,
-                    campaignName: agentConfig.campaignName,
-                };
-
-                this.logger.log(`CC agent ${body.email} → Ozonetel ${agentConfig.ozonetelAgentId} / SIP ${agentConfig.sipExtension}`);
             }
 
             return {

src/livekit-agent/agent.ts

@@ -0,0 +1,279 @@
+import { WorkerOptions, defineAgent, llm, voice, VAD } from '@livekit/agents';
+import * as google from '@livekit/agents-plugin-google';
+import * as silero from '@livekit/agents-plugin-silero';
+import { z } from 'zod';
+
+// Platform GraphQL helper
+const SIDECAR_URL = process.env.SIDECAR_URL ?? 'http://localhost:4100';
+const PLATFORM_API_KEY = process.env.PLATFORM_API_KEY ?? '';
+
+async function gql<T = any>(query: string, variables?: Record<string, unknown>): Promise<T | null> {
+    if (!PLATFORM_API_KEY) return null;
+    try {
+        const res = await fetch(`${SIDECAR_URL}/graphql`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${PLATFORM_API_KEY}` },
+            body: JSON.stringify({ query, variables }),
+        });
+        const data = await res.json();
+        if (data.errors) {
+            console.error('[AGENT-GQL] Error:', data.errors[0]?.message);
+            return null;
+        }
+        return data.data;
+    } catch (err) {
+        console.error('[AGENT-GQL] Failed:', err);
+        return null;
+    }
+}
+
+// Hospital context — loaded on startup
+let hospitalContext = {
+    doctors: [] as Array<{ name: string; department: string; specialty: string; id: string }>,
+    departments: [] as string[],
+};
+
+async function loadHospitalContext() {
+    const data = await gql(`{ doctors(first: 20) { edges { node { id fullName { firstName lastName } department specialty } } } }`);
+    if (data?.doctors?.edges) {
+        hospitalContext.doctors = data.doctors.edges.map((e: any) => ({
+            id: e.node.id,
+            name: `Dr. ${e.node.fullName?.firstName ?? ''} ${e.node.fullName?.lastName ?? ''}`.trim(),
+            department: e.node.department ?? '',
+            specialty: e.node.specialty ?? '',
+        }));
+        hospitalContext.departments = [...new Set(hospitalContext.doctors.map(d => d.department))] as string[];
+        console.log(`[LIVEKIT-AGENT] Loaded ${hospitalContext.doctors.length} doctors, ${hospitalContext.departments.length} departments`);
+    } else {
+        // Fallback
+        hospitalContext.doctors = [
+            { id: '', name: 'Dr. Arun Sharma', department: 'Cardiology', specialty: 'Interventional Cardiology' },
+            { id: '', name: 'Dr. Rajesh Kumar', department: 'Orthopedics', specialty: 'Joint Replacement' },
+            { id: '', name: 'Dr. Meena Patel', department: 'Gynecology', specialty: 'Reproductive Medicine' },
+            { id: '', name: 'Dr. Lakshmi Reddy', department: 'General Medicine', specialty: 'Internal Medicine' },
+            { id: '', name: 'Dr. Harpreet Singh', department: 'ENT', specialty: 'Head & Neck Surgery' },
+        ];
+        hospitalContext.departments = ['Cardiology', 'Orthopedics', 'Gynecology', 'General Medicine', 'ENT'];
+        console.log('[LIVEKIT-AGENT] Using fallback doctor list');
+    }
+}
+
+// ─── Tools ────────────────────────────────────────────────────────────
+
+const lookupDoctor = llm.tool({
+    description: 'Look up available doctors by department or specialty. Call this when the patient asks about a specific department or type of doctor.',
+    parameters: z.object({
+        department: z.string().nullable().describe('Department name like Cardiology, Orthopedics, ENT'),
+        specialty: z.string().nullable().describe('Specialty or condition like joint pain, heart, ear'),
+    }),
+    execute: async ({ department, specialty }) => {
+        let results = hospitalContext.doctors;
+        if (department) {
+            results = results.filter(d => d.department.toLowerCase().includes(department.toLowerCase()));
+        }
+        if (specialty) {
+            results = results.filter(d =>
+                d.specialty.toLowerCase().includes(specialty.toLowerCase()) ||
+                d.department.toLowerCase().includes(specialty.toLowerCase()),
+            );
+        }
+        if (results.length === 0) return 'No matching doctors found. Available departments: ' + hospitalContext.departments.join(', ');
+        return results.map(d => `${d.name} — ${d.department} (${d.specialty})`).join('\n');
+    },
+});
+
+const bookAppointment = llm.tool({
+    description: 'Book an appointment for the caller. You MUST collect patient name, phone number, department, preferred date/time, and reason before calling this.',
+    parameters: z.object({
+        patientName: z.string().describe('Full name of the patient'),
+        phoneNumber: z.string().describe('Patient phone number with country code'),
+        department: z.string().describe('Department for the appointment'),
+        doctorName: z.string().nullable().describe('Preferred doctor name if specified'),
+        preferredDate: z.string().describe('Date in YYYY-MM-DD format or natural language'),
+        preferredTime: z.string().describe('Time slot like 10:00 AM, morning, afternoon'),
+        reason: z.string().describe('Reason for visit'),
+    }),
+    execute: async ({ patientName, phoneNumber, department, doctorName, preferredDate, preferredTime, reason }) => {
+        console.log(`[LIVEKIT-AGENT] Booking: ${patientName} | ${phoneNumber} | ${department} | ${doctorName ?? 'any'} | ${preferredDate} ${preferredTime}`);
+
+        // Parse date — try ISO format first, fallback to tomorrow
+        let scheduledAt: string;
+        try {
+            const parsed = new Date(preferredDate);
+            if (!isNaN(parsed.getTime())) {
+                // Map time to hour
+                const timeMap: Record<string, string> = { morning: '10:00', afternoon: '14:00', evening: '17:00' };
+                const timeStr = timeMap[preferredTime.toLowerCase()] ?? preferredTime.replace(/\s*(AM|PM)/i, (_, p) => '');
+                scheduledAt = new Date(`${parsed.toISOString().split('T')[0]}T${timeStr}:00`).toISOString();
+            } else {
+                scheduledAt = new Date(Date.now() + 86400000).toISOString(); // tomorrow
+            }
+        } catch {
+            scheduledAt = new Date(Date.now() + 86400000).toISOString();
+        }
+
+        // Find matching doctor
+        const doctor = doctorName
+            ? hospitalContext.doctors.find(d => d.name.toLowerCase().includes(doctorName.toLowerCase()))
+            : hospitalContext.doctors.find(d => d.department.toLowerCase().includes(department.toLowerCase()));
+
+        // Create appointment on platform
+        const result = await gql(
+            `mutation($data: AppointmentCreateInput!) { createAppointment(data: $data) { id } }`,
+            {
+                data: {
+                    name: `AI Booking — ${patientName} (${department})`,
+                    scheduledAt,
+                    status: 'SCHEDULED',
+                    doctorName: doctor?.name ?? doctorName ?? 'To be assigned',
+                    department,
+                    reasonForVisit: reason,
+                },
+            },
+        );
+
+        // Create or find lead
+        const cleanPhone = phoneNumber.replace(/[^0-9]/g, '').slice(-10);
+        await gql(
+            `mutation($data: LeadCreateInput!) { createLead(data: $data) { id } }`,
+            {
+                data: {
+                    name: `AI — ${patientName}`,
+                    contactName: {
+                        firstName: patientName.split(' ')[0],
+                        lastName: patientName.split(' ').slice(1).join(' ') || '',
+                    },
+                    contactPhone: { primaryPhoneNumber: `+91${cleanPhone}` },
+                    source: 'PHONE',
+                    status: 'APPOINTMENT_SET',
+                    interestedService: department,
+                },
+            },
+        );
+
+        const refNum = `GH-${Date.now().toString().slice(-6)}`;
+        if (result?.createAppointment?.id) {
+            console.log(`[LIVEKIT-AGENT] Appointment created: ${result.createAppointment.id}`);
+            return `Appointment booked successfully! Reference number ${refNum}. ${patientName} is scheduled for ${department} on ${preferredDate} at ${preferredTime} with ${doctor?.name ?? 'an available doctor'}. A confirmation SMS will be sent to ${phoneNumber}.`;
+        }
+        return `I have noted the appointment request. Reference number ${refNum}. Our team will confirm the booking and send an SMS to ${phoneNumber}.`;
+    },
+});
+
+const collectLeadInfo = llm.tool({
+    description: 'Save the caller as a lead/enquiry when they are interested but not ready to book. Collect their name and phone number.',
+    parameters: z.object({
+        name: z.string().describe('Caller name'),
+        phoneNumber: z.string().describe('Caller phone number'),
+        interest: z.string().describe('What they are interested in or enquiring about'),
+    }),
+    execute: async ({ name, phoneNumber, interest }) => {
+        console.log(`[LIVEKIT-AGENT] Lead: ${name} | ${phoneNumber} | ${interest}`);
+
+        const cleanPhone = phoneNumber.replace(/[^0-9]/g, '').slice(-10);
+        const result = await gql(
+            `mutation($data: LeadCreateInput!) { createLead(data: $data) { id } }`,
+            {
+                data: {
+                    name: `AI Enquiry — ${name}`,
+                    contactName: {
+                        firstName: name.split(' ')[0],
+                        lastName: name.split(' ').slice(1).join(' ') || '',
+                    },
+                    contactPhone: { primaryPhoneNumber: `+91${cleanPhone}` },
+                    source: 'PHONE',
+                    status: 'NEW',
+                    interestedService: interest,
+                },
+            },
+        );
+
+        if (result?.createLead?.id) {
+            console.log(`[LIVEKIT-AGENT] Lead created: ${result.createLead.id}`);
+        }
+        return `Thank you ${name}. I have noted your enquiry about ${interest}. One of our team members will call you back on ${phoneNumber} shortly.`;
+    },
+});
+
+const transferToAgent = llm.tool({
+    description: 'Transfer the call to a human agent. Use this when the caller explicitly asks to speak with a person, or when the query is too complex.',
+    parameters: z.object({
+        reason: z.string().describe('Why the caller needs a human agent'),
+    }),
+    execute: async ({ reason }) => {
+        console.log(`[LIVEKIT-AGENT] Transfer requested: ${reason}`);
+        // TODO: When SIP is connected, trigger Ozonetel transfer via sidecar API
+        return 'I am transferring you to one of our agents now. Please hold for a moment. If no agent is available, someone will call you back within 15 minutes.';
+    },
+});
+
+// ─── Agent ────────────────────────────────────────────────────────────
+
+const hospitalAgent = new voice.Agent({
+    instructions: `You are the AI receptionist for Global Hospital, Bangalore. Your name is Helix.
+
+PERSONALITY:
+- Warm, professional, and empathetic
+- Speak clearly and at a moderate pace
+- Use simple language — many callers may not be fluent in English
+- Be concise — this is a phone call, not a chat
+- Respond in the same language the caller uses (English, Hindi, Kannada)
+
+CAPABILITIES:
+- Answer questions about hospital departments, doctors, and specialties
+- Book appointments — collect: name, phone, department, preferred date/time, reason
+- Take messages and create enquiries for callback
+- Transfer to a human agent when needed
+
+HOSPITAL INFO:
+- Global Hospital, Bangalore
+- Open Monday to Saturday, 8 AM to 8 PM
+- Emergency services available 24/7
+- Departments: ${hospitalContext.departments.join(', ') || 'Cardiology, Orthopedics, Gynecology, General Medicine, ENT'}
+
+RULES:
+- Greet: "Hello, thank you for calling Global Hospital. This is Helix, how may I help you today?"
+- If caller asks about pricing, say you will have the team call back with details
+- Never give medical advice — always recommend consulting a doctor
+- If the caller is in an emergency, tell them to visit the ER immediately or call 108
+- Always confirm all details before booking an appointment
+- End calls politely: "Thank you for calling Global Hospital. Have a good day!"
+- If you cannot understand the caller, politely ask them to repeat`,
+    llm: new google.beta.realtime.RealtimeModel({
+        model: 'gemini-2.5-flash-native-audio-latest',
+        voice: 'Aoede',
+        temperature: 0.7,
+    }),
+    tools: { lookupDoctor, bookAppointment, collectLeadInfo, transferToAgent },
+});
+
+// ─── Entry Point ──────────────────────────────────────────────────────
+
+export default defineAgent({
+    prewarm: async (proc) => {
+        proc.userData.vad = await silero.VAD.load();
+        await loadHospitalContext();
+    },
+    entry: async (ctx) => {
+        await ctx.connect();
+        console.log(`[LIVEKIT-AGENT] Connected to room: ${ctx.room.name}`);
+
+        const session = new voice.AgentSession({
+            vad: ctx.proc.userData.vad as VAD,
+        });
+
+        await session.start({ agent: hospitalAgent, room: ctx.room });
+        console.log('[LIVEKIT-AGENT] Voice session started');
+
+        // Gemini Realtime handles greeting via instructions — no separate say() needed
+    },
+});
+
+// CLI runner
+if (require.main === module) {
+    const options = new WorkerOptions({
+        agent: __filename,
+    });
+    const { cli } = require('@livekit/agents');
+    cli.runApp(options);
+}

src/supervisor/supervisor.service.ts

@@ -77,6 +77,7 @@ export class SupervisorService implements OnModuleInit {
     private mapOzonetelAction(action: string, eventData: string): AgentOzonetelState | null {
         switch (action) {
             case 'release': return 'ready';
+            case 'IDLE': return 'ready'; // agent available after unanswered/canceled call
             case 'calling': return 'calling';
             case 'incall': return 'in-call';
             case 'ACW': return 'acw';