Chatbot with Livepeer LLM

import { NextRequest } from 'next/server';

export const runtime = 'edge';

const GATEWAY_URL = process.env.LIVEPEER_GATEWAY_URL!;
const MODEL = process.env.LIVEPEER_LLM_MODEL!;

interface Message {
  role: 'user' | 'assistant' | 'system';
  content: string;
}

export async function POST(req: NextRequest) {
  const { messages } = (await req.json()) as { messages: Message[] };

  const response = await fetch(`${GATEWAY_URL}/llm`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      model: MODEL,
      messages,
      stream: true,
    }),
  });

  if (!response.ok || !response.body) {
    return new Response(`Gateway returned ${response.status}`, {
      status: 502,
    });
  }

  // Pipe the SSE stream straight through to the client.
  return new Response(response.body, {
    headers: {
      'Content-Type': 'text/event-stream',
      'Cache-Control': 'no-cache',
      Connection: 'keep-alive',
    },
  });
}

data: {"choices":[{"delta":{"content":"Live","role":"assistant"},"finish_reason":null}]}

data: {"choices":[{"delta":{"content":"peer","role":"assistant"},"finish_reason":null}]}

data: {"choices":[{"delta":{"content":" is","role":"assistant"},"finish_reason":null}]}

...

data: {"choices":[{"delta":{"content":"","role":"assistant"},"finish_reason":"stop"}]}

'use client';

const { useState } = React;

interface Message {
  role: 'user' | 'assistant' | 'system';
  content: string;
}

export function Chat() {
  const [messages, setMessages] = useState<Message[]>([
    {
      role: 'system',
      content: 'You are a helpful assistant. Keep responses concise.',
    },
  ]);
  const [input, setInput] = useState('');
  const [streaming, setStreaming] = useState(false);

  async function sendMessage() {
    if (!input.trim() || streaming) return;

    const userMessage: Message = { role: 'user', content: input };
    const newMessages = [...messages, userMessage];
    setMessages(newMessages);
    setInput('');
    setStreaming(true);

    // Add an empty assistant message that we'll fill as tokens arrive.
    setMessages((prev) => [...prev, { role: 'assistant', content: '' }]);

    const response = await fetch('/api/chat', {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ messages: newMessages }),
    });

    if (!response.ok || !response.body) {
      setStreaming(false);
      return;
    }

    const reader = response.body.getReader();
    const decoder = new TextDecoder();
    let buffer = '';

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      buffer += decoder.decode(value, { stream: true });
      const lines = buffer.split('\n');
      buffer = lines.pop() ?? '';

      for (const line of lines) {
        if (!line.startsWith('data: ')) continue;
        const data = line.slice(6).trim();
        if (!data) continue;

        try {
          const chunk = JSON.parse(data);
          const token = chunk.choices?.[0]?.delta?.content ?? '';
          const finished = chunk.choices?.[0]?.finish_reason === 'stop';

          if (token) {
            setMessages((prev) => {
              const next = [...prev];
              next[next.length - 1] = {
                ...next[next.length - 1],
                content: next[next.length - 1].content + token,
              };
              return next;
            });
          }

          if (finished) break;
        } catch {
          // Skip malformed chunks
        }
      }
    }

    setStreaming(false);
  }

  return (
    <div className="max-w-2xl mx-auto p-4 space-y-4">
      <div className="space-y-2 min-h-[400px]">
        {messages
          .filter((m) => m.role !== 'system')
          .map((m, i) => (
            <div
              key={i}
              className={`p-3 rounded ${
                m.role === 'user' ? 'bg-blue-100' : 'bg-gray-100'
              }`}
            >
              <p className="text-xs text-gray-600 mb-1">{m.role}</p>
              <p className="whitespace-pre-wrap">{m.content}</p>
            </div>
          ))}
      </div>
      <div className="flex gap-2">
        <input
          type="text"
          value={input}
          onChange={(e) => setInput(e.target.value)}
          onKeyDown={(e) => e.key === 'Enter' && sendMessage()}
          placeholder="Ask anything"
          disabled={streaming}
          className="flex-1 border rounded p-2"
        />
        <button
          onClick={sendMessage}
          disabled={streaming}
          className="bg-blue-600 text-white px-4 py-2 rounded disabled:opacity-50"
        >
          {streaming ? 'Streaming…' : 'Send'}
        </button>
      </div>
    </div>
  );
}

// Import Chat from ./components/Chat.

export default function HomePage() {
  return (
    <main className="min-h-screen bg-white">
      <header className="border-b p-4">
        <h1 className="text-xl font-bold">Livepeer LLM Chatbot</h1>
        <p className="text-sm text-gray-600">
          Streaming chat via the decentralised LLM pipeline.
        </p>
      </header>
      <Chat />
    </main>
  );
}

npm run dev

Build the "Chatbot with Livepeer LLM" tutorial as a Next.js App Router project. Create a TypeScript app, add LIVEPEER_GATEWAY_URL=https://dream-gateway.livepeer.cloud to .env.local, implement src/app/api/chat/route.ts as a streaming Server-Sent Events route that forwards OpenAI-compatible chat completion requests to the Livepeer LLM endpoint, and build a client chat UI that appends streamed tokens in place. Use model "meta-llama/Meta-Llama-3.1-8B-Instruct" by default and expose a small model selector for Mistral, Gemma, and Qwen variants. Include run commands, a curl test for the route, browser verification at http://localhost:3000, and production notes that any LIVEPEER_API_KEY must stay server-side. Do not use Studio.