
NVIDIA/Magpie TTS
NVIDIA's 357M-parameter multilingual text-to-speech model
Magpie TTS is a multilingual neural text-to-speech model that generates speech in twelve different languages (English-US, Spanish-US, German-German, French-France, Italian, Vietnamese, Mandarin-Chinese, Hindi, Japanese, Modern Standard Arabic, Brazilian Portuguese, and Korean) by predicting discrete audio codec tokens autoregressively using a transformer encoder-decoder architecture. It supports at least one male and one female speaker for all the languages. It employs multi-codebook prediction (typically 8 codebooks) with local transformer refinement and frame stacking (factor = 2) for high-quality audio generation, and leverages techniques like attention priors, classifier-free guidance (CFG), and Group Relative Policy Optimization (GRPO) for improved alignment and robustness.
import asyncio, base64, json, os, time, wave
import websockets # >= 13
URL = (
"wss://model-<MODEL_ID>.api.baseten.co"
"/environments/production/websocket?intent=synthesize"
)
API_KEY = os.environ["BASETEN_API_KEY"]
RATE = 22050
SESSION = {
"input_text_synthesis": {
"language_code": "en-US",
"voice_name": "Magpie-Multilingual.EN-US.Mia",
},
"output_audio_params": {
"sample_rate_hz": RATE,
"num_channels": 1,
"audio_format": "LINEAR_PCM",
},
}
# Text sent before both acks can use the default session.
ACKS = {"conversation.created", "synthesize_session.updated"}
async def send(ws, type_, **fields):
await ws.send(json.dumps({"type": type_, **fields}))
async def main(
text="Hello from Magpie, streaming on Baseten.",
out="magpie.wav",
):
headers = {"Authorization": f"Api-Key {API_KEY}"}
async with websockets.connect(
URL, additional_headers=headers, max_size=10 * 2**20
) as ws:
await send(
ws, "synthesize_session.update", session=SESSION
)
pending = set(ACKS)
while pending:
event = json.loads(await ws.recv())
if event["type"] == "error":
raise RuntimeError(event)
pending.discard(event["type"])
start = time.perf_counter()
await send(ws, "input_text.append", text=text)
await send(ws, "input_text.commit")
await send(ws, "input_text.done")
pcm = bytearray()
async for message in ws:
event = json.loads(message)
kind = event["type"]
if kind == "conversation.item.speech.data":
if not pcm:
ttfa = time.perf_counter() - start
print(f"first audio after {ttfa:.3f}s")
pcm += base64.b64decode(event["audio"])
elif kind == "conversation.item.speech.completed":
break
elif kind == "error":
raise RuntimeError(event)
with wave.open(out, "wb") as w:
w.setnchannels(1)
w.setsampwidth(2)
w.setframerate(RATE)
w.writeframes(pcm)
seconds = len(pcm) / 2 / RATE
print(f"wrote {seconds:.2f}s of audio to {out}")
asyncio.run(main())