Please rotate your device

To continue exploring this website, please rotate your device to a vertical orientation

Magpie TTS

NVIDIA/Magpie TTS

NVIDIA's 357M-parameter multilingual text-to-speech model

text to speech

Magpie TTS is a multilingual neural text-to-speech model that generates speech in twelve different languages (English-US, Spanish-US, German-German, French-France, Italian, Vietnamese, Mandarin-Chinese, Hindi, Japanese, Modern Standard Arabic, Brazilian Portuguese, and Korean) by predicting discrete audio codec tokens autoregressively using a transformer encoder-decoder architecture. It supports at least one male and one female speaker for all the languages. It employs multi-codebook prediction (typically 8 codebooks) with local transformer refinement and frame stacking (factor = 2) for high-quality audio generation, and leverages techniques like attention priors, classifier-free guidance (CFG), and Group Relative Policy Optimization (GRPO) for improved alignment and robustness.

python
import asyncio, base64, json, os, time, wave

import websockets  # >= 13

URL = (
    "wss://model-<MODEL_ID>.api.baseten.co"
    "/environments/production/websocket?intent=synthesize"
)
API_KEY = os.environ["BASETEN_API_KEY"]
RATE = 22050
SESSION = {
    "input_text_synthesis": {
        "language_code": "en-US",
        "voice_name": "Magpie-Multilingual.EN-US.Mia",
    },
    "output_audio_params": {
        "sample_rate_hz": RATE,
        "num_channels": 1,
        "audio_format": "LINEAR_PCM",
    },
}
# Text sent before both acks can use the default session.
ACKS = {"conversation.created", "synthesize_session.updated"}


async def send(ws, type_, **fields):
    await ws.send(json.dumps({"type": type_, **fields}))


async def main(
    text="Hello from Magpie, streaming on Baseten.",
    out="magpie.wav",
):
    headers = {"Authorization": f"Api-Key {API_KEY}"}
    async with websockets.connect(
        URL, additional_headers=headers, max_size=10 * 2**20
    ) as ws:
        await send(
            ws, "synthesize_session.update", session=SESSION
        )

        pending = set(ACKS)
        while pending:
            event = json.loads(await ws.recv())
            if event["type"] == "error":
                raise RuntimeError(event)
            pending.discard(event["type"])

        start = time.perf_counter()
        await send(ws, "input_text.append", text=text)
        await send(ws, "input_text.commit")
        await send(ws, "input_text.done")

        pcm = bytearray()
        async for message in ws:
            event = json.loads(message)
            kind = event["type"]
            if kind == "conversation.item.speech.data":
                if not pcm:
                    ttfa = time.perf_counter() - start
                    print(f"first audio after {ttfa:.3f}s")
                pcm += base64.b64decode(event["audio"])
            elif kind == "conversation.item.speech.completed":
                break
            elif kind == "error":
                raise RuntimeError(event)

    with wave.open(out, "wb") as w:
        w.setnchannels(1)
        w.setsampwidth(2)
        w.setframerate(RATE)
        w.writeframes(pcm)
    seconds = len(pcm) / 2 / RATE
    print(f"wrote {seconds:.2f}s of audio to {out}")


asyncio.run(main())