Build Your First Voice Agent using Twilio

1
import os
2
from dotenv import load_dotenv
3
from loguru import logger
4
from pipecat.frames.frames import LLMRunFrame
5
from pipecat.pipeline.pipeline import Pipeline
6
from pipecat.pipeline.runner import PipelineRunner
7
from pipecat.pipeline.task import PipelineParams, PipelineTask
8
from pipecat.processors.aggregators.llm_context import LLMContext
9
from pipecat.processors.aggregators.llm_response_universal import (
10
    LLMContextAggregatorPair,
11
)
12
from pipecat.runner.types import RunnerArguments
13
from pipecat.runner.utils import create_transport
14
from pipecat.services.sarvam.stt import SarvamSTTService
15
from pipecat.services.sarvam.tts import SarvamTTSService
16
from pipecat.services.sarvam.llm import SarvamLLMService
17
from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams
18
19
load_dotenv(override=True)
20
21
async def bot(runner_args: RunnerArguments):
22
    """Main bot entry point."""
23
24
    # create_transport auto-detects Twilio's WebSocket handshake and builds
25
    # the matching TwilioFrameSerializer, using TWILIO_ACCOUNT_SID and
26
    # TWILIO_AUTH_TOKEN from the environment.
27
    transport = await create_transport(
28
        runner_args,
29
        {
30
            "twilio": lambda: FastAPIWebsocketParams(
31
                audio_in_enabled=True, audio_out_enabled=True
32
            ),
33
        },
34
    )
35
36
    # Initialize AI services
37
    stt = SarvamSTTService(api_key=os.getenv("SARVAM_API_KEY"))
38
    tts = SarvamTTSService(api_key=os.getenv("SARVAM_API_KEY"))
39
    llm = SarvamLLMService(
40
        api_key=os.getenv("SARVAM_API_KEY"),
41
        settings=SarvamLLMService.Settings(model="sarvam-105b"),
42
    )
43
44
    # Set up conversation context
45
    messages = [
46
        {
47
            "role": "system",
48
            "content": "You are a friendly AI phone assistant. Keep your responses brief and conversational.",
49
        },
50
    ]
51
    context = LLMContext(messages)
52
    context_aggregator = LLMContextAggregatorPair(context)
53
54
    # Build pipeline
55
    pipeline = Pipeline(
56
        [
57
            transport.input(),
58
            stt,
59
            context_aggregator.user(),
60
            llm,
61
            tts,
62
            transport.output(),
63
            context_aggregator.assistant(),
64
        ]
65
    )
66
67
    # Twilio Media Streams send and receive 8kHz mono audio
68
    task = PipelineTask(
69
        pipeline,
70
        params=PipelineParams(
71
            audio_in_sample_rate=8000,
72
            audio_out_sample_rate=8000,
73
        ),
74
    )
75
76
    @transport.event_handler("on_client_connected")
77
    async def on_client_connected(transport, client):
78
        logger.info("Caller connected")
79
        messages.append(
80
            {"role": "system", "content": "Greet the caller and briefly introduce yourself."}
81
        )
82
        await task.queue_frames([LLMRunFrame()])
83
84
    @transport.event_handler("on_client_disconnected")
85
    async def on_client_disconnected(transport, client):
86
        logger.info("Caller disconnected")
87
        await task.cancel()
88
89
    runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
90
    await runner.run(task)
91
92
if __name__ == "__main__":
93
    from pipecat.runner.run import main
94
    main()

1
stt = SarvamSTTService(
2
    api_key=os.getenv("SARVAM_API_KEY"),
3
    settings=SarvamSTTService.Settings(
4
        model="saaras:v3",
5
        language="hi-IN",  # Hindi
6
    ),
7
    mode="transcribe",
8
)
9
10
tts = SarvamTTSService(
11
    api_key=os.getenv("SARVAM_API_KEY"),
12
    settings=SarvamTTSService.Settings(
13
        model="bulbul:v3",
14
        voice="simran",  # Or: priya, ishita, kavya, aditya, anand, rohan
15
        target_language_code="hi-IN",
16
    ),
17
)
18
19
llm = SarvamLLMService(
20
    api_key=os.getenv("SARVAM_API_KEY"),
21
    settings=SarvamLLMService.Settings(model="sarvam-105b"),
22
)

1
stt = SarvamSTTService(
2
    api_key=os.getenv("SARVAM_API_KEY"),
3
    settings=SarvamSTTService.Settings(
4
        model="saaras:v3",
5
        language="ta-IN",
6
    ),
7
    mode="transcribe",
8
)
9
10
tts = SarvamTTSService(
11
    api_key=os.getenv("SARVAM_API_KEY"),
12
    settings=SarvamTTSService.Settings(
13
        model="bulbul:v3",
14
        voice="shubh",
15
        target_language_code="ta-IN",
16
    ),
17
)
18
19
llm = SarvamLLMService(
20
    api_key=os.getenv("SARVAM_API_KEY"),
21
    settings=SarvamLLMService.Settings(model="sarvam-105b"),
22
)

1
# Auto-detect the caller's language
2
stt = SarvamSTTService(
3
    api_key=os.getenv("SARVAM_API_KEY"),
4
    settings=SarvamSTTService.Settings(
5
        model="saaras:v3",
6
        language="unknown",  # Auto-detects language
7
    ),
8
    mode="transcribe",
9
)
10
11
tts = SarvamTTSService(
12
    api_key=os.getenv("SARVAM_API_KEY"),
13
    settings=SarvamTTSService.Settings(
14
        model="bulbul:v3",
15
        voice="anand",
16
        target_language_code="en-IN",
17
    ),
18
)
19
20
llm = SarvamLLMService(
21
    api_key=os.getenv("SARVAM_API_KEY"),
22
    settings=SarvamLLMService.Settings(model="sarvam-105b"),
23
)

1
# Caller speaks Hindi, Saaras converts it to English, the LLM processes it,
2
# and the agent responds in English.
3
4
stt = SarvamSTTService(
5
    api_key=os.getenv("SARVAM_API_KEY"),
6
    settings=SarvamSTTService.Settings(model="saaras:v3"),
7
    mode="translate",  # Speech-to-English translation
8
)
9
10
tts = SarvamTTSService(
11
    api_key=os.getenv("SARVAM_API_KEY"),
12
    settings=SarvamTTSService.Settings(
13
        model="bulbul:v3",
14
        voice="aditya",
15
        target_language_code="en-IN",
16
    ),
17
)
18
19
llm = SarvamLLMService(
20
    api_key=os.getenv("SARVAM_API_KEY"),
21
    settings=SarvamLLMService.Settings(model="sarvam-105b"),
22
)

1
tts = SarvamTTSService(
2
    api_key=os.getenv("SARVAM_API_KEY"),
3
    settings=SarvamTTSService.Settings(
4
        model="bulbul:v3",
5
        voice="shubh",
6
        target_language_code="en-IN",
7
        pace=1.0,  # Range: 0.5 to 2.0
8
    ),
9
)