Grok Voice 3.0 API

wss://api.apimodels.app/v1/realtime?model=grok-voice-3.0

curl -X POST https://api.apimodels.app/v1/realtime/client_secrets \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"expires_after": {"seconds": 60}}'

# { "value": "<token>", "expires_at": 1792130400 }

new WebSocket("wss://api.apimodels.app/v1/realtime?model=grok-voice-3.0", ["xai-client-secret." + value])

{
  "type": "session.update",
  "session": {
    "voice": "eve",
    "instructions": "You are Grok Voice 3.0, a realtime voice assistant built by xAI. You are a friendly support agent. Keep answers short.",
    "turn_detection": { "type": "server_vad" },
    "audio": {
      "input":  { "format": { "type": "audio/pcm", "rate": 24000 },
                  "transcription": { "language_hint": "zh-CN" } },
      "output": { "format": { "type": "audio/pcm", "rate": 24000 }, "speed": 1.0 }
    },
    "tools": [
      {
        "type": "function",
        "name": "get_order_status",
        "description": "Look up an order by its id",
        "parameters": {
          "type": "object",
          "properties": { "order_id": { "type": "string" } },
          "required": ["order_id"]
        }
      }
    ]
  }
}

// with turn_detection {"type":"server_vad"}: just keep appending,
// the server decides when you stopped talking and answers.
{ "type": "input_audio_buffer.append", "audio": "<base64 16-bit PCM, 24 kHz, mono>" }

// with turn_detection null (manual turns): append, then commit, then ask for a reply
{ "type": "input_audio_buffer.append", "audio": "<base64 ...>" }
{ "type": "input_audio_buffer.commit" }
{ "type": "response.create" }

// throw away audio you have not committed yet
{ "type": "input_audio_buffer.clear" }

{
  "type": "conversation.item.create",
  "item": {
    "type": "message",
    "role": "user",
    "content": [{ "type": "input_text", "text": "What can you do?" }]
  }
}
{ "type": "response.create" }

// server -> you
{ "type": "response.function_call_arguments.done",
  "call_id": "call_123", "name": "get_order_status", "arguments": "{\"order_id\":\"A-1001\"}" }

// you -> server: the result, then ask the model to continue speaking
{ "type": "conversation.item.create",
  "item": { "type": "function_call_output", "call_id": "call_123", "output": "{\"status\":\"shipped\"}" } }
{ "type": "response.create" }

# pip install websockets   (websockets 14+; on older versions pass extra_headers= instead)
import asyncio, base64, json, os, wave
import websockets

URL = "wss://api.apimodels.app/v1/realtime?model=grok-voice-3.0"
API_KEY = os.environ["APIMODELS_API_KEY"]


async def main():
    headers = {"Authorization": f"Bearer {API_KEY}"}
    pcm = bytearray()
    asked = False

    async with websockets.connect(URL, additional_headers=headers) as ws:
        # 1. Configure the session. turn_detection None = manual turns (no microphone here).
        await ws.send(json.dumps({
            "type": "session.update",
            "session": {
                "voice": "eve",
                "instructions": "You are Grok Voice 3.0, a realtime voice assistant built by xAI. Keep answers short.",
                "turn_detection": None,
                "audio": {"output": {"format": {"type": "audio/pcm", "rate": 24000}}},
            },
        }))

        async for raw in ws:
            event = json.loads(raw)
            kind = event["type"]

            if kind == "session.updated" and not asked:
                asked = True
                # 2. One text message ($0.0052), then ask for a spoken reply.
                await ws.send(json.dumps({
                    "type": "conversation.item.create",
                    "item": {
                        "type": "message",
                        "role": "user",
                        "content": [{"type": "input_text", "text": "Introduce yourself in one sentence."}],
                    },
                }))
                await ws.send(json.dumps({"type": "response.create"}))

            elif kind == "response.output_audio.delta":
                pcm += base64.b64decode(event["delta"])

            elif kind == "response.output_audio_transcript.delta":
                print(event["delta"], end="", flush=True)

            elif kind == "error":
                raise RuntimeError(event)

            elif kind == "response.done":
                break

    # 3. The audio is 16-bit PCM, 24 kHz, mono: wrap it in a WAV header.
    with wave.open("reply.wav", "wb") as wav:
        wav.setnchannels(1)
        wav.setsampwidth(2)
        wav.setframerate(24000)
        wav.writeframes(bytes(pcm))
    print()
    print("saved reply.wav")


asyncio.run(main())

// npm install ws   (Node 18+, save as reply.mjs)
import fs from "node:fs";
import WebSocket from "ws";

const ws = new WebSocket("wss://api.apimodels.app/v1/realtime?model=grok-voice-3.0", {
  headers: { Authorization: "Bearer " + process.env.APIMODELS_API_KEY },
});
const chunks = [];
let asked = false;

ws.on("open", () => {
  ws.send(JSON.stringify({
    type: "session.update",
    session: {
      voice: "eve",
      instructions: "You are Grok Voice 3.0, a realtime voice assistant built by xAI. Keep answers short.",
      turn_detection: null,
    },
  }));
});

ws.on("message", (data) => {
  const event = JSON.parse(data.toString());

  if (event.type === "session.updated" && !asked) {
    asked = true;
    ws.send(JSON.stringify({
      type: "conversation.item.create",
      item: {
        type: "message",
        role: "user",
        content: [{ type: "input_text", text: "Introduce yourself in one sentence." }],
      },
    }));
    ws.send(JSON.stringify({ type: "response.create" }));
  } else if (event.type === "response.output_audio.delta") {
    chunks.push(Buffer.from(event.delta, "base64"));
  } else if (event.type === "error") {
    console.error(event);
  } else if (event.type === "response.done") {
    // Raw 16-bit PCM, 24 kHz, mono. To get a WAV:
    //   ffmpeg -f s16le -ar 24000 -ac 1 -i reply.pcm reply.wav
    fs.writeFileSync("reply.pcm", Buffer.concat(chunks));
    ws.close();
  }
});

// ── On your server (Node 18+, any framework). The API key never leaves the server. ──
export async function mintRealtimeToken() {
  const res = await fetch("https://api.apimodels.app/v1/realtime/client_secrets", {
    method: "POST",
    headers: {
      Authorization: "Bearer " + process.env.APIMODELS_API_KEY,
      "Content-Type": "application/json",
    },
    body: JSON.stringify({ expires_after: { seconds: 60 } }), // 10 to 600, default 60
  });
  return res.json(); // { value, expires_at }
}
// Expose it to your own logged-in users, e.g. POST /api/realtime-token -> mintRealtimeToken()