Whisper Realtime

# pip install websockets
# audio.pcm: mono, 16 kHz, signed PCM16 little-endian; <= 30 seconds.
import asyncio, json, os
from pathlib import Path
from websockets.asyncio.client import connect

async def main():
    headers = {"Authorization": "Bearer " + os.environ["SPEECH_API_KEY"]}
    async with connect("wss://api.speechinfra.com/v1/inference/whisper-realtime/realtime", additional_headers=headers) as socket:
        await socket.send("{\"turn_detection\":\"server_vad\",\"threshold\":0.3,\"min_silence_duration_ms\":500,\"min_speech_duration_ms\":250,\"max_speech_duration_s\":5,\"speech_pad_ms\":250,\"type\":\"session.configure\"}")
        while True:
            event = json.loads(await socket.recv())
            if event["type"] == "error":
                raise RuntimeError(event)
            if event["type"] == "session.updated":
                break
        async def send():
            audio = Path("audio.pcm").read_bytes()
            assert 0 < len(audio) <= 960000 and len(audio) % 2 == 0
            for i in range(0, len(audio), 3200):
                await socket.send(audio[i:i+3200])
                await asyncio.sleep(0.1)
            await socket.send('{"type":"session.close"}')
        sender = asyncio.create_task(send())
        try:
            async with asyncio.timeout(90):
                async for message in socket:
                    print(message)
        finally:
            sender.cancel()

asyncio.run(main())

// Use from the console origin with a Speech Cloud project key.
// apiKey is a Speech Cloud project key, never a provider key.
const socket = new WebSocket("wss://api.speechinfra.com/v1/inference/whisper-realtime/realtime", ['speech', apiKey]);
socket.onopen = () => socket.send("{\"turn_detection\":\"server_vad\",\"threshold\":0.3,\"min_silence_duration_ms\":500,\"min_speech_duration_ms\":250,\"max_speech_duration_s\":5,\"speech_pad_ms\":250,\"type\":\"session.configure\"}");
socket.onmessage = ({data}) => {
  const event = JSON.parse(data);
  console.log(event);
  if (event.type === 'session.updated') {
    // Send mono PCM16 / 16 kHz binary frames, <= 30 s in total.
    // After the final frame: socket.send(JSON.stringify({type:'session.close'}));
  }
};
// For a complete file-streaming example: scripts/test_models.py

{
  "type": "object",
  "additionalProperties": false,
  "required": [],
  "properties": {
    "turn_detection": {
      "type": "string",
      "enum": [
        "server_vad",
        "manual"
      ],
      "default": "server_vad"
    },
    "threshold": {
      "type": "number",
      "minimum": 0,
      "maximum": 1,
      "default": 0.3,
      "visibleWhen": {
        "turn_detection": "server_vad"
      }
    },
    "min_silence_duration_ms": {
      "type": "integer",
      "minimum": 0,
      "maximum": 10000,
      "default": 500,
      "visibleWhen": {
        "turn_detection": "server_vad"
      }
    },
    "min_speech_duration_ms": {
      "type": "integer",
      "minimum": 0,
      "maximum": 10000,
      "default": 250,
      "visibleWhen": {
        "turn_detection": "server_vad"
      }
    },
    "max_speech_duration_s": {
      "type": "number",
      "minimum": 0.1,
      "maximum": 300,
      "default": 5,
      "visibleWhen": {
        "turn_detection": "server_vad"
      }
    },
    "speech_pad_ms": {
      "type": "integer",
      "minimum": 0,
      "maximum": 5000,
      "default": 250,
      "visibleWhen": {
        "turn_detection": "server_vad"
      }
    },
    "type": {
      "type": "string",
      "enum": [
        "session.configure"
      ],
      "default": "session.configure"
    }
  }
}

{
  "type": "object",
  "description": "JSON server events. Send binary mono PCM16 at 16 kHz after session.updated; session.close drains and returns session.closed.",
  "properties": {
    "type": {
      "enum": [
        "session.created",
        "session.updated",
        "session.routed",
        "transcript.partial",
        "transcript.final",
        "error",
        "session.closed"
      ]
    }
  }
}