Build a voice agent (Next.js)

export async function POST(req: Request) {
  const { text, speaker = "astra", modelId = "coda" } = await req.json();

  if (!text) {
    return Response.json({ error: "text is required" }, { status: 400 });
  }

  const rimeRes = await fetch("https://users.rime.ai/v1/rime-tts", {
    method: "POST",
    headers: {
      Authorization: `Bearer ${process.env.RIME_API_KEY}`,
      "Content-Type": "application/json",
      Accept: "audio/mpeg", // MP3; see the API reference for Opus, WAV, PCM, μ-law
    },
    body: JSON.stringify({ text, speaker, modelId }),
  });

  if (!rimeRes.ok) {
    const detail = await rimeRes.text();
    return Response.json(
      { error: `Rime API error ${rimeRes.status}`, detail },
      { status: rimeRes.status },
    );
  }

  // Stream the audio straight through to the browser
  return new Response(rimeRes.body, {
    headers: { "Content-Type": "audio/mpeg" },
  });
}

"use client";

import { useRef, useState } from "react";

// Replace this stub with a call to your LLM of choice.
function respond(userText: string): string {
  return `You said: ${userText}. This voice is Rime's Coda model.`;
}

export default function VoiceAgent() {
  const [messages, setMessages] = useState<string[]>([]);
  const [listening, setListening] = useState(false);
  const audioRef = useRef<HTMLAudioElement>(null);

  async function speak(text: string) {
    const res = await fetch("/api/tts", {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ text }),
    });
    if (!res.ok) throw new Error(`TTS failed: ${res.status}`);
    const url = URL.createObjectURL(await res.blob());
    audioRef.current!.src = url;
    await audioRef.current!.play();
  }

  async function handleUserText(userText: string) {
    const reply = respond(userText);
    setMessages((m) => [...m, `You: ${userText}`, `Agent: ${reply}`]);
    await speak(reply);
  }

  function listen() {
    const SR =
      (window as any).SpeechRecognition ??
      (window as any).webkitSpeechRecognition;
    if (!SR) return alert("SpeechRecognition not supported in this browser.");
    const rec = new SR();
    rec.onresult = (e: any) => handleUserText(e.results[0][0].transcript);
    rec.onend = () => setListening(false);
    setListening(true);
    rec.start();
  }

  return (
    <main style={{ maxWidth: 480, margin: "4rem auto", fontFamily: "sans-serif" }}>
      <h1>Rime voice agent</h1>
      <button onClick={listen} disabled={listening}>
        {listening ? "Listening…" : "🎤 Speak"}
      </button>
      <ul>{messages.map((m, i) => <li key={i}>{m}</li>)}</ul>
      <audio ref={audioRef} hidden />
    </main>
  );
}

import { createServer } from "node:http";
import next from "next";
import { WebSocketServer, WebSocket } from "ws";

const app = next({ dev: process.env.NODE_ENV !== "production" });
await app.prepare();
const handle = app.getRequestHandler();

const server = createServer(handle);
const wss = new WebSocketServer({ server, path: "/ws/tts" });

wss.on("connection", (client) => {
  // One upstream Rime connection per browser client.
  // speaker/modelId/audioFormat are query params; auth is a header.
  const rime = new WebSocket(
    "wss://users-ws.rime.ai/ws3?speaker=astra&modelId=coda&audioFormat=mp3",
    { headers: { Authorization: `Bearer ${process.env.RIME_API_KEY}` } },
  );

  // Browser → Rime: {text: "..."} to speak, {operation: "clear"} to interrupt.
  // Queue anything that arrives before the upstream connection finishes opening;
  // the browser side is usually connected first, and ws drops sends on a
  // CONNECTING socket.
  const pending = [];
  client.on("message", (data) => {
    const msg = data.toString();
    if (rime.readyState === WebSocket.OPEN) rime.send(msg);
    else pending.push(msg);
  });
  rime.on("open", () => {
    for (const msg of pending) rime.send(msg);
    pending.length = 0;
  });

  // Rime → browser: {type: "chunk", data: <base64 mp3>}, {type: "timestamps", ...},
  // {type: "done"}. Relay verbatim and let the client decode.
  rime.on("message", (data) => {
    if (client.readyState === WebSocket.OPEN) client.send(data.toString());
  });

  const closeBoth = () => { rime.close(); client.close(); };
  client.on("close", closeBoth);
  rime.on("close", closeBoth);
  rime.on("error", (err) => {
    client.send(JSON.stringify({ type: "error", message: String(err) }));
  });
});

server.listen(3000, () => {
  console.log("Next.js + Rime WS bridge on http://localhost:3000");
});

{
  "scripts": {
    "dev": "node server.mjs",
    "build": "next build",
    "start": "NODE_ENV=production node server.mjs"
  }
}