Build Your First Voice Agent using Vapi

1
const { SarvamAIClient } = require("sarvamai");
2
3
class SarvamTranscriber {
4
  constructor({ channel, sampleRate, language, onTranscript }) {
5
    this.channel = channel; // "customer" or "assistant"
6
    this.sampleRate = sampleRate;
7
    this.language = language;
8
    this.onTranscript = onTranscript;
9
    this.socket = null;
10
  }
11
12
  async connect() {
13
    const client = new SarvamAIClient({
14
      apiSubscriptionKey: process.env.SARVAM_API_KEY,
15
    });
16
17
    // Heads-up on naming: these keys mirror the WebSocket API's query
18
    // parameters, so "language-code" is hyphenated (quoted) while the
19
    // rest are snake_case. The SDK expects them as-is, so don't "fix" the names.
20
    this.socket = await client.speechToTextStreaming.connect({
21
      model: "saaras:v3",
22
      mode: "transcribe",
23
      "language-code": this.language,
24
      sample_rate: this.sampleRate,
25
      input_audio_codec: "pcm_s16le", // Vapi sends raw 16-bit PCM
26
      high_vad_sensitivity: "true",   // 0.5s silence boundary for snappier turns
27
    });
28
29
    this.socket.on("message", (message) => {
30
      if (message.type === "data" && message.data.transcript) {
31
        this.onTranscript(message.data.transcript, this.channel);
32
      }
33
    });
34
35
    this.socket.on("error", (err) => {
36
      console.error(`Sarvam STT error (${this.channel}):`, err);
37
    });
38
39
    await this.socket.waitForOpen();
40
    console.log(`Sarvam STT connected for ${this.channel} channel`);
41
  }
42
43
  sendAudio(pcmBuffer) {
44
    if (!this.socket) return;
45
    try {
46
      // encoding stays "audio/wav" even for raw PCM, since the codec was
47
      // already declared via input_audio_codec at connection time
48
      this.socket.transcribe({
49
        audio: pcmBuffer.toString("base64"),
50
        sample_rate: this.sampleRate,
51
        encoding: "audio/wav",
52
      });
53
    } catch (err) {
54
      console.error(`Sarvam STT send failed (${this.channel}):`, err);
55
    }
56
  }
57
58
  close() {
59
    this.socket?.close();
60
  }
61
}
62
63
module.exports = SarvamTranscriber;

1
require("dotenv").config();
2
const express = require("express");
3
const http = require("http");
4
const { WebSocketServer } = require("ws");
5
const { SarvamAIClient } = require("sarvamai");
6
const SarvamTranscriber = require("./sarvamTranscriber");
7
8
const app = express();
9
app.use(express.json({ limit: "5mb" }));
10
11
const sarvam = new SarvamAIClient({
12
  apiSubscriptionKey: process.env.SARVAM_API_KEY,
13
});
14
15
// ---------------------------------------------------------------
16
// LLM proxy (custom-llm): forwards to Sarvam with reasoning off
17
// ---------------------------------------------------------------
18
19
// Sarvam enables thinking mode by default, which delays the first
20
// spoken token and can consume the whole max_tokens budget. Vapi's
21
// custom-llm config can't pass reasoning_effort, so inject it here.
22
app.post("/llm/chat/completions", async (req, res) => {
23
  try {
24
    const body = { ...req.body, reasoning_effort: null };
25
26
    const upstream = await fetch("https://api.sarvam.ai/v1/chat/completions", {
27
      method: "POST",
28
      headers: {
29
        "api-subscription-key": process.env.SARVAM_API_KEY,
30
        "Content-Type": "application/json",
31
      },
32
      body: JSON.stringify(body),
33
    });
34
35
    res.status(upstream.status);
36
    res.setHeader(
37
      "Content-Type",
38
      upstream.headers.get("content-type") || "application/json"
39
    );
40
41
    // Stream SSE (or JSON) straight through
42
    const { Readable } = require("stream");
43
    Readable.fromWeb(upstream.body).pipe(res);
44
  } catch (error) {
45
    console.error("LLM proxy failed:", error);
46
    if (!res.headersSent) res.status(500).json({ error: "LLM proxy failed" });
47
  }
48
});
49
50
// ---------------------------------------------------------------
51
// TTS endpoint (custom-voice): text in, raw PCM out
52
// ---------------------------------------------------------------
53
54
// Sarvam returns base64 WAV; Vapi needs headerless 16-bit mono PCM.
55
// Locate the WAV "data" chunk and return everything after it.
56
function wavToRawPcm(wavBuffer) {
57
  const dataIndex = wavBuffer.indexOf("data");
58
  if (dataIndex === -1) throw new Error("Invalid WAV: no data chunk");
59
  return wavBuffer.subarray(dataIndex + 8); // skip "data" tag + 4-byte size
60
}
61
62
app.post("/api/synthesize", async (req, res) => {
63
  try {
64
    const { message } = req.body;
65
    if (message?.type !== "voice-request" || !message.text) {
66
      return res.status(400).json({ error: "Expected a voice-request with text" });
67
    }
68
69
    const response = await sarvam.textToSpeech.convert({
70
      text: message.text,
71
      model: "bulbul:v3",
72
      target_language_code: process.env.TTS_LANGUAGE || "en-IN",
73
      speaker: process.env.TTS_SPEAKER || "shubh",
74
      speech_sample_rate: message.sampleRate, // must match Vapi's request
75
    });
76
77
    const wav = Buffer.from(response.audios.join(""), "base64");
78
    const pcm = wavToRawPcm(wav);
79
80
    res.setHeader("Content-Type", "application/octet-stream");
81
    res.setHeader("Content-Length", pcm.length);
82
    res.end(pcm);
83
  } catch (error) {
84
    console.error("TTS synthesis failed:", error);
85
    if (!res.headersSent) {
86
      res.status(500).json({ error: "TTS synthesis failed" });
87
    }
88
  }
89
});
90
91
// ---------------------------------------------------------------
92
// STT endpoint (custom-transcriber): WebSocket audio in, JSON out
93
// ---------------------------------------------------------------
94
95
// Vapi streams interleaved stereo PCM: channel 0 = customer, 1 = assistant
96
function splitStereo(buffer) {
97
  const frames = Math.floor(buffer.length / 4); // 2 bytes x 2 channels
98
  const customer = Buffer.alloc(frames * 2);
99
  const assistant = Buffer.alloc(frames * 2);
100
  for (let i = 0; i < frames; i++) {
101
    buffer.copy(customer, i * 2, i * 4, i * 4 + 2);
102
    buffer.copy(assistant, i * 2, i * 4 + 2, i * 4 + 4);
103
  }
104
  return { customer, assistant };
105
}
106
107
const server = http.createServer(app);
108
const wss = new WebSocketServer({ server, path: "/api/custom-transcriber" });
109
110
wss.on("connection", (ws) => {
111
  console.log("Vapi connected to custom transcriber");
112
  const transcribers = {};
113
114
  const sendTranscript = (transcription, channel) => {
115
    if (ws.readyState !== ws.OPEN) return;
116
    console.log(`Transcript (${channel}): ${transcription}`);
117
    ws.send(
118
      JSON.stringify({
119
        type: "transcriber-response",
120
        transcription,
121
        channel,
122
        transcriptType: "final",
123
      })
124
    );
125
  };
126
127
  ws.on("message", async (data, isBinary) => {
128
    if (!isBinary) {
129
      let msg;
130
      try {
131
        msg = JSON.parse(data.toString());
132
      } catch (err) {
133
        console.error("Ignoring non-JSON control message from Vapi:", err);
134
        return;
135
      }
136
      if (msg.type === "start") {
137
        // One Sarvam session per audio channel
138
        const language = process.env.STT_LANGUAGE || "unknown";
139
        try {
140
          for (const channel of ["customer", "assistant"]) {
141
            transcribers[channel] = new SarvamTranscriber({
142
              channel,
143
              sampleRate: msg.sampleRate,
144
              language,
145
              onTranscript: sendTranscript,
146
            });
147
            await transcribers[channel].connect();
148
          }
149
        } catch (err) {
150
          // Most common cause: invalid SARVAM_API_KEY. Close the socket
151
          // so Vapi's fallback plan can take over instead of dead air.
152
          console.error("Failed to start Sarvam STT sessions:", err);
153
          ws.close(1011, "Transcriber startup failed");
154
        }
155
      }
156
      return;
157
    }
158
159
    if (!transcribers.customer) return;
160
    const { customer, assistant } = splitStereo(data);
161
    transcribers.customer.sendAudio(customer);
162
    transcribers.assistant.sendAudio(assistant);
163
  });
164
165
  ws.on("error", (err) => {
166
    console.error("Transcriber WebSocket error:", err);
167
  });
168
169
  ws.on("close", () => {
170
    console.log("Vapi disconnected");
171
    Object.values(transcribers).forEach((t) => t.close());
172
  });
173
});
174
175
const PORT = process.env.PORT || 3001;
176
server.listen(PORT, () => {
177
  console.log(`Sarvam bridge listening on port ${PORT}`);
178
});

1
this.socket = await client.speechToTextStreaming.connect({
2
  model: "saaras:v3",
3
  mode: "translate",              // Speech-to-English translation
4
  sample_rate: this.sampleRate,
5
  input_audio_codec: "pcm_s16le",
6
  high_vad_sensitivity: "true",
7
});