Model details
Magpie TTS is a multilingual neural text-to-speech model that generates speech in twelve different languages (English-US, Spanish-US, German-German, French-France, Italian, Vietnamese, Mandarin-Chinese, Hindi, Japanese, Modern Standard Arabic, Brazilian Portuguese, and Korean) by predicting discrete audio codec tokens autoregressively using a transformer encoder-decoder architecture. It supports at least one male and one female speaker for all the languages. It employs multi-codebook prediction (typically 8 codebooks) with local transformer refinement and frame stacking (factor = 2) for high-quality audio generation, and leverages techniques like attention priors, classifier-free guidance (CFG), and Group Relative Policy Optimization (GRPO) for improved alignment and robustness.
1import asyncio, base64, json, os, time, wave
2
3import websockets # >= 13
4
5URL = (
6 "wss://model-<MODEL_ID>.api.baseten.co"
7 "/environments/production/websocket?intent=synthesize"
8)
9API_KEY = os.environ["BASETEN_API_KEY"]
10RATE = 22050
11SESSION = {
12 "input_text_synthesis": {
13 "language_code": "en-US",
14 "voice_name": "Magpie-Multilingual.EN-US.Mia",
15 },
16 "output_audio_params": {
17 "sample_rate_hz": RATE,
18 "num_channels": 1,
19 "audio_format": "LINEAR_PCM",
20 },
21}
22# Text sent before both acks can use the default session.
23ACKS = {"conversation.created", "synthesize_session.updated"}
24
25
26async def send(ws, type_, **fields):
27 await ws.send(json.dumps({"type": type_, **fields}))
28
29
30async def main(
31 text="Hello from Magpie, streaming on Baseten.",
32 out="magpie.wav",
33):
34 headers = {"Authorization": f"Api-Key {API_KEY}"}
35 async with websockets.connect(
36 URL, additional_headers=headers, max_size=10 * 2**20
37 ) as ws:
38 await send(
39 ws, "synthesize_session.update", session=SESSION
40 )
41
42 pending = set(ACKS)
43 while pending:
44 event = json.loads(await ws.recv())
45 if event["type"] == "error":
46 raise RuntimeError(event)
47 pending.discard(event["type"])
48
49 start = time.perf_counter()
50 await send(ws, "input_text.append", text=text)
51 await send(ws, "input_text.commit")
52 await send(ws, "input_text.done")
53
54 pcm = bytearray()
55 async for message in ws:
56 event = json.loads(message)
57 kind = event["type"]
58 if kind == "conversation.item.speech.data":
59 if not pcm:
60 ttfa = time.perf_counter() - start
61 print(f"first audio after {ttfa:.3f}s")
62 pcm += base64.b64decode(event["audio"])
63 elif kind == "conversation.item.speech.completed":
64 break
65 elif kind == "error":
66 raise RuntimeError(event)
67
68 with wave.open(out, "wb") as w:
69 w.setnchannels(1)
70 w.setsampwidth(2)
71 w.setframerate(RATE)
72 w.writeframes(pcm)
73 seconds = len(pcm) / 2 / RATE
74 print(f"wrote {seconds:.2f}s of audio to {out}")
75
76
77asyncio.run(main())
