ForgeLearn
|$ curl https://forge-ai.dev/api/markdown?path=docs/ai/streaming
$cat docs/streaming-ai-responses.md
updated Recently·35 min read·published

Streaming AI Responses

◆AI◆Intermediate🎯Free Tools
Introduction

Streaming enables token-by-token delivery of LLM responses, dramatically improving perceived latency. Instead of waiting 5-30 seconds for a complete response, users see text appear in real-time as the model generates it. This is essential for production AI applications.

This guide covers the full streaming stack: server-sent events (SSE) for transport, API-level streaming with OpenAI, server-side middleware, and frontend rendering patterns for smooth, responsive AI interfaces.

Server-Sent Events (SSE)

SSE is the standard protocol for streaming data from server to client over HTTP. Unlike WebSockets, SSE is unidirectional (server to client), simpler to implement, and works through proxies and load balancers.

route.ts
TypeScript
1// Next.js API route — streaming endpoint
2import { OpenAIStream, StreamingTextResponse } from "ai";
3import OpenAI from "openai";
4
5const openai = new OpenAI();
6
7export async function POST(req: Request) {
8 const { messages } = await req.json();
9
10 const response = await openai.chat.completions.create({
11 model: "gpt-4o",
12 messages,
13 stream: true,
14 });
15
16 // Convert OpenAI stream to ReadableStream
17 const stream = OpenAIStream(response);
18 return new StreamingTextResponse(stream);
19}
20
21// Raw SSE implementation (without SDK)
22export async function GET() {
23 const encoder = new TextEncoder();
24
25 const stream = new ReadableStream({
26 async start(controller) {
27 const response = await openai.chat.completions.create({
28 model: "gpt-4o",
29 messages: [{ role: "user", content: "Hello" }],
30 stream: true,
31 });
32
33 for await (const chunk of response) {
34 const text = chunk.choices[0]?.delta?.content || "";
35 if (text) {
36 // SSE format: data: <payload>\n\n
37 controller.enqueue(
38 encoder.encode(`data: ${JSON.stringify({ text })}\n\n`)
39 );
40 }
41 }
42 controller.enqueue(encoder.encode("data: [DONE]\n\n"));
43 controller.close();
44 },
45 });
46
47 return new Response(stream, {
48 headers: {
49 "Content-Type": "text/event-stream",
50 "Cache-Control": "no-cache",
51 Connection: "keep-alive",
52 },
53 });
54}
ℹ

info

The data: [DONE] message is the SSE convention for signaling stream completion. Always send it so clients know when to stop listening.
OpenAI Streaming API

OpenAI's streaming API yields response chunks in real-time. Each chunk contains a delta with partial content, which accumulates into the full response.

openai_stream.py
Python
1# Python — async streaming with OpenAI
2import asyncio
3from openai import AsyncOpenAI
4
5client = AsyncOpenAI()
6
7async def stream_response(prompt: str):
8 """Stream tokens from OpenAI API."""
9 response = await client.chat.completions.create(
10 model="gpt-4o",
11 messages=[{"role": "user", "content": prompt}],
12 stream=True,
13 )
14
15 full_response = ""
16 async for chunk in response:
17 delta = chunk.choices[0].delta
18 if delta.content:
19 full_response += delta.content
20 print(delta.content, end="", flush=True)
21
22 print() # Newline after completion
23 return full_response
24
25# Usage
26asyncio.run(stream_response("Explain streaming in 3 sentences."))
ai-sdk.ts
TypeScript
1// TypeScript — stream with Vercel AI SDK
2import { openai } from "@ai-sdk/openai";
3import { streamText } from "ai";
4
5export async function POST(req: Request) {
6 const { messages } = await req.json();
7
8 const result = streamText({
9 model: openai("gpt-4o"),
10 messages,
11 onFinish: ({ text, usage }) => {
12 console.log("Complete:", text.length, "chars");
13 console.log("Tokens used:", usage);
14 },
15 });
16
17 return result.toDataStreamResponse();
18}
19
20// Client-side consumption
21import { useChat } from "ai/react";
22
23function Chat() {
24 const { messages, input, handleInputChange, handleSubmit } = useChat({
25 api: "/api/chat",
26 });
27
28 return (
29 <div>
30 {messages.map((m) => (
31 <div key={m.id}>
32 <strong>{m.role}:</strong> {m.content}
33 </div>
34 ))}
35 <form onSubmit={handleSubmit}>
36 <input value={input} onChange={handleInputChange} />
37 </form>
38 </div>
39 );
40}
✓

best practice

Use the Vercel AI SDK (useChat hook) for Next.js apps. It handles streaming, error recovery, message persistence, and optimistic updates out of the box.
Chunk Processing & Buffering

Raw stream chunks arrive irregularly — sometimes partial words, sometimes multiple tokens. Client-side processing must handle buffering, parsing, and rendering efficiently.

chunk-processing.ts
TypeScript
1// Client-side stream consumer with buffering
2async function consumeStream(url: string, prompt: string) {
3 const response = await fetch(url, {
4 method: "POST",
5 headers: { "Content-Type": "application/json" },
6 body: JSON.stringify({ messages: [{ role: "user", content: prompt }] }),
7 });
8
9 const reader = response.body!.getReader();
10 const decoder = new TextDecoder();
11 let buffer = "";
12 let fullText = "";
13
14 while (true) {
15 const { done, value } = await reader.read();
16 if (done) break;
17
18 // Decode chunk and add to buffer
19 buffer += decoder.decode(value, { stream: true });
20
21 // Process complete SSE lines
22 const lines = buffer.split("\n");
23 buffer = lines.pop() || ""; // Keep incomplete line in buffer
24
25 for (const line of lines) {
26 if (line.startsWith("data: ")) {
27 const data = line.slice(6);
28 if (data === "[DONE]") break;
29
30 try {
31 const parsed = JSON.parse(data);
32 if (parsed.text) {
33 fullText += parsed.text;
34 renderToken(parsed.text); // Update UI incrementally
35 }
36 } catch (e) {
37 // Skip malformed JSON
38 }
39 }
40 }
41 }
42
43 return fullText;
44}
45
46function renderToken(token: string) {
47 const el = document.getElementById("output");
48 if (el) el.textContent += token;
49}
StreamingMessage.tsx
TypeScript
1// Token-by-token rendering with React
2import { useState, useCallback } from "react";
3
4function StreamingMessage({ stream }: { stream: ReadableStream }) {
5 const [text, setText] = useState("");
6 const [isStreaming, setIsStreaming] = useState(false);
7
8 const processStream = useCallback(async () => {
9 setIsStreaming(true);
10 const reader = stream.getReader();
11 const decoder = new TextDecoder();
12 let buffer = "";
13
14 try {
15 while (true) {
16 const { done, value } = await reader.read();
17 if (done) break;
18
19 buffer += decoder.decode(value, { stream: true });
20 const lines = buffer.split("\n");
21 buffer = lines.pop() || "";
22
23 for (const line of lines) {
24 if (line.startsWith("data: ") && line !== "data: [DONE]") {
25 try {
26 const { text: token } = JSON.parse(line.slice(6));
27 if (token) setText((prev) => prev + token);
28 } catch {}
29 }
30 }
31 }
32 } finally {
33 setIsStreaming(false);
34 }
35 }, [stream]);
36
37 return (
38 <div className="prose">
39 {text}
40 {isStreaming && <span className="animate-pulse">|</span>}
41 </div>
42 );
43}
ℹ

info

Always buffer partial SSE lines — chunks rarely align with line boundaries. Accumulate data and split on \\n, keeping any remainder in the buffer for the next iteration.
UI Streaming Patterns

Production AI interfaces use specific streaming patterns to maximize perceived performance and user experience. These patterns handle the gap between request and first token, provide feedback during generation, and manage errors gracefully.

chat-pattern.tsx
TypeScript
1// Pattern 1: Optimistic UI with streaming
2function ChatInterface() {
3 const [messages, setMessages] = useState<Message[]>([]);
4 const [isGenerating, setIsGenerating] = useState(false);
5
6 const sendMessage = async (content: string) => {
7 // Optimistic: add user message immediately
8 const userMsg = { role: "user", content, id: Date.now() };
9 setMessages((prev) => [...prev, userMsg]);
10
11 // Add placeholder for AI response
12 const aiMsg = { role: "assistant", content: "", id: Date.now() + 1 };
13 setMessages((prev) => [...prev, aiMsg]);
14 setIsGenerating(true);
15
16 try {
17 const response = await fetch("/api/chat", {
18 method: "POST",
19 headers: { "Content-Type": "application/json" },
20 body: JSON.stringify({ messages: [...messages, userMsg] }),
21 });
22
23 const reader = response.body!.getReader();
24 const decoder = new TextDecoder();
25 let accumulated = "";
26
27 while (true) {
28 const { done, value } = await reader.read();
29 if (done) break;
30
31 const chunk = decoder.decode(value, { stream: true });
32 const lines = chunk.split("\n");
33
34 for (const line of lines) {
35 if (line.startsWith("data: ") && line !== "data: [DONE]") {
36 try {
37 const { text } = JSON.parse(line.slice(6));
38 accumulated += text;
39 // Update only the last message
40 setMessages((prev) => {
41 const updated = [...prev];
42 updated[updated.length - 1] = {
43 ...updated[updated.length - 1],
44 content: accumulated,
45 };
46 return updated;
47 });
48 } catch {}
49 }
50 }
51 }
52 } catch (error) {
53 // Handle stream error — remove empty AI message
54 setMessages((prev) => prev.slice(0, -1));
55 } finally {
56 setIsGenerating(false);
57 }
58 };
59}
streaming-ui-patterns.tsx
TypeScript
1// Pattern 2: Markdown rendering during streaming
2import ReactMarkdown from "react-markdown";
3
4function StreamingMarkdown({ content }: { content: string }) {
5 // Re-render markdown on every token — optimized with useMemo
6 const rendered = useMemo(() => (
7 <ReactMarkdown
8 components={{
9 // Prevent layout shift for code blocks
10 code: ({ children, ...props }) => (
11 <code {...props}>{children}</code>
12 ),
13 }}
14 >
15 {content}
16 </ReactMarkdown>
17 ), [content]);
18
19 return <div className="prose prose-invert">{rendered}</div>;
20}
21
22// Pattern 3: Typewriter effect with variable speed
23function TypewriterText({ text }: { text: string }) {
24 const [displayed, setDisplayed] = useState(0);
25
26 useEffect(() => {
27 if (displayed < text.length) {
28 const delay = text[displayed] === "." ? 80 : 15;
29 const timer = setTimeout(() => setDisplayed((d) => d + 1), delay);
30 return () => clearTimeout(timer);
31 }
32 }, [displayed, text]);
33
34 return <span>{text.slice(0, displayed)}<span className="animate-pulse">|</span></span>;
35}
⚠

warning

Re-rendering React components on every token can cause performance issues with large messages. Use useMemo for expensive renders, and consider virtualizing long messages with react-window.
Error Handling & Recovery

Streams can disconnect mid-generation due to network issues, rate limits, or server errors. Robust streaming implementations must handle partial responses and enable resumption.

error-handling.ts
TypeScript
1// Resilient streaming with retry and partial recovery
2async function resilientStream(
3 url: string,
4 messages: Message[],
5 onToken: (token: string) => void,
6 onError: (error: Error) => void,
7) {
8 const MAX_RETRIES = 3;
9 let accumulated = "";
10 let retryCount = 0;
11
12 while (retryCount < MAX_RETRIES) {
13 try {
14 const response = await fetch(url, {
15 method: "POST",
16 headers: { "Content-Type": "application/json" },
17 body: JSON.stringify({ messages }),
18 });
19
20 if (!response.ok) {
21 if (response.status === 429) {
22 // Rate limited — exponential backoff
23 const waitTime = Math.pow(2, retryCount) * 1000;
24 await new Promise((r) => setTimeout(r, waitTime));
25 retryCount++;
26 continue;
27 }
28 throw new Error(`HTTP ${response.status}`);
29 }
30
31 const reader = response.body!.getReader();
32 const decoder = new TextDecoder();
33
34 while (true) {
35 const { done, value } = await reader.read();
36 if (done) break;
37
38 const chunk = decoder.decode(value, { stream: true });
39 // Parse and emit tokens...
40 }
41
42 return accumulated; // Success
43 } catch (error) {
44 retryCount++;
45 if (retryCount >= MAX_RETRIES) {
46 onError(error as Error);
47 return accumulated; // Return what we have
48 }
49 }
50 }
51}
✓

best practice

Always preserve partially received content on stream errors. Users should see what was generated before the error, with an option to retry from where it left off rather than starting over.
Performance Optimization

Streaming performance depends on both the server-side generation speed and the client-side rendering efficiency. Optimizing both ends is critical for smooth user experiences.

performance.ts
TypeScript
1// Performance: batch DOM updates with requestAnimationFrame
2function createBatchedRenderer(container: HTMLElement) {
3 let buffer = "";
4 let scheduled = false;
5
6 return function appendToken(token: string) {
7 buffer += token;
8
9 if (!scheduled) {
10 scheduled = true;
11 requestAnimationFrame(() => {
12 container.textContent += buffer;
13 buffer = "";
14 scheduled = false;
15 });
16 }
17 };
18}
19
20// Performance: throttle rendering for fast streams
21function throttledRenderer(
22 element: HTMLElement,
23 tokensPerFrame: number = 3
24) {
25 let buffer = "";
26 let count = 0;
27
28 return (token: string) => {
29 buffer += token;
30 count++;
31
32 if (count >= tokensPerFrame) {
33 requestAnimationFrame(() => {
34 element.textContent += buffer;
35 buffer = "";
36 count = 0;
37 });
38 }
39 };
40}
41
42// Usage
43const output = document.getElementById("output")!;
44const render = createBatchedRenderer(output);
45
46// In stream processing loop:
47for await (const chunk of stream) {
48 render(chunk); // Automatically batches DOM updates
49}
ℹ

info

Use requestAnimationFrame to batch DOM updates. Rendering on every token causes layout thrashing — batching 3-5 tokens per frame is imperceptible to humans but dramatically reduces reflows.
← PreviousLlamaIndex
Next →AI in UIs
🎯

35+ Free Developer Tools

AI Chat, Mermaid, Excalidraw, JSON Editor, API Tester & more

→
$Blueprint — Engineering Documentation·Section ID: AI-ST-01·Revision: 1.0
Follow