Building LLM Applications with RAG, Function Calling, and Production Patterns

Building LLM applications in 2026 means working with a mature but fast-moving ecosystem. RAG reduces hallucinations; function calling enables structured outputs; streaming improves UX.

RAG (Retrieval-Augmented Generation)

from openai import OpenAI
import numpy as np
from typing import List

client = OpenAI()

def chunk_document(text: str, chunk_size: int = 500, overlap: int = 50) -> List[str]:
    words = text.split()
    return [
        ' '.join(words[i:i + chunk_size])
        for i in range(0, len(words), chunk_size - overlap)
        if words[i:i + chunk_size]
    ]

def embed(text: str) -> List[float]:
    return client.embeddings.create(
        model="text-embedding-3-small", input=text
    ).data[0].embedding

def cosine_similarity(a, b) -> float:
    a, b = np.array(a), np.array(b)
    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))

def retrieve(query: str, index: list, top_k: int = 5) -> list:
    q_emb = embed(query)
    scored = sorted(index, key=lambda d: cosine_similarity(q_emb, d['embedding']), reverse=True)
    return scored[:top_k]

def answer_with_rag(query: str, index: list) -> str:
    docs = retrieve(query, index)
    context = "

".join(f"Source: {d['title']}
{d['content']}" for d in docs)

    return client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {"role": "system", "content": "Answer ONLY using the provided context. Cite your sources."},
            {"role": "user", "content": f"Context:
{context}

Question: {query}"}
        ],
        temperature=0.1
    ).choices[0].message.content

Function Calling for Structured Output

tools = [{
    "type": "function",
    "function": {
        "name": "search_products",
        "description": "Search the product catalog",
        "parameters": {
            "type": "object",
            "properties": {
                "query": {"type": "string"},
                "category": {"type": "string", "enum": ["electronics", "clothing", "books"]},
                "max_price": {"type": "number"}
            },
            "required": ["query"]
        }
    }
}]

def process_with_tools(user_message: str, history: list) -> str:
    messages = history + [{"role": "user", "content": user_message}]
    resp = client.chat.completions.create(model="gpt-4o", messages=messages, tools=tools)
    msg = resp.choices[0].message

    if msg.tool_calls:
        messages.append(msg)
        for call in msg.tool_calls:
            fn_name = call.function.name
            args = json.loads(call.function.arguments)
            result = search_products(**args) if fn_name == "search_products" else {}
            messages.append({"role": "tool", "tool_call_id": call.id, "content": json.dumps(result)})
        final = client.chat.completions.create(model="gpt-4o", messages=messages)
        return final.choices[0].message.content

    return msg.content

Streaming Responses

from fastapi.responses import StreamingResponse

@app.post("/chat/stream")
async def chat_stream(request: ChatRequest):
    async def generate():
        stream = await client.chat.completions.create(
            model="gpt-4o",
            messages=[{"role": "user", "content": request.message}],
            stream=True
        )
        async for chunk in stream:
            if chunk.choices[0].delta.content:
                yield f"data: {json.dumps({'text': chunk.choices[0].delta.content})}

"
        yield "data: [DONE]

"

    return StreamingResponse(generate(), media_type="text/event-stream")

Cost Optimization

# Cache embeddings (deterministic)
from functools import lru_cache

@lru_cache(maxsize=10000)
def cached_embed(text: str) -> tuple:
    return tuple(embed(text))

# Route to cheaper model for simple queries
def select_model(query: str) -> str:
    simple_patterns = ['what is', 'define ', 'list the', 'when was']
    return "gpt-4o-mini" if any(query.lower().startswith(p) for p in simple_patterns) else "gpt-4o"

# Compress context to stay within token budget
def compress_context(docs: list, max_tokens: int = 2000) -> str:
    context, total = [], 0
    for doc in docs:
        if total + len(doc['content']) > max_tokens * 4:
            break
        context.append(doc['content'])
        total += len(doc['content'])
    return "

".join(context)

Evaluation

def evaluate_rag(cases: list, rag_fn) -> dict:
    results = []
    for case in cases:
        answer = rag_fn(case['question'])
        results.append({
            'question': case['question'],
            'answer': answer,
            'relevance': check_relevance(answer, case['expected']),
            'faithfulness': check_faithfulness(answer, case['context']),
        })
    avg_relevance = sum(r['relevance'] for r in results) / len(results)
    avg_faithfulness = sum(r['faithfulness'] for r in results) / len(results)
    return {'avg_relevance': avg_relevance, 'avg_faithfulness': avg_faithfulness, 'n': len(results)}

LLM applications require a different mindset: probabilistic systems need evaluation from day one, not as an afterthought.

→ Format your prompt templates with the JSON Viewer tool.