Deploying LLMs in Production

Running LLMs in production is expensive and latency-sensitive. This guide covers practical techniques to reduce costs and improve performance.

Model Serving with vLLM

pip install vllm

# Serve with OpenAI-compatible API
python -m vllm.entrypoints.openai.api_server   --model meta-llama/Llama-3-8B-Instruct   --tensor-parallel-size 2   --max-model-len 4096   --gpu-memory-utilization 0.90   --max-num-seqs 256

from vllm import LLM, SamplingParams

llm = LLM(
    model="meta-llama/Llama-3-8B-Instruct",
    tensor_parallel_size=2,
    quantization="awq",  # Use quantized model
)

outputs = llm.generate(
    ["Explain async/await in Python", "What is Docker?"],
    SamplingParams(temperature=0.7, max_tokens=512),
)

Prompt Caching

import hashlib
import redis
import json

class CachedLLMClient:
    def __init__(self, client, redis_client):
        self.client = client
        self.redis = redis_client
        self.ttl = 3600  # 1 hour
    
    def _cache_key(self, messages, **kwargs):
        content = json.dumps({"messages": messages, "kwargs": kwargs}, sort_keys=True)
        return f"llm:{hashlib.sha256(content.encode()).hexdigest()}"
    
    def generate(self, messages, **kwargs):
        cache_key = self._cache_key(messages, **kwargs)
        
        # Check cache first
        cached = self.redis.get(cache_key)
        if cached:
            return json.loads(cached)
        
        # Generate
        response = self.client.chat.completions.create(
            model="gpt-4o-mini",
            messages=messages,
            **kwargs,
        )
        result = response.choices[0].message.content
        
        # Cache result
        self.redis.setex(cache_key, self.ttl, json.dumps(result))
        return result

Request Batching

import asyncio
from dataclasses import dataclass

@dataclass
class BatchRequest:
    messages: list
    future: asyncio.Future

class BatchedLLMClient:
    def __init__(self, max_batch_size=10, max_wait_ms=50):
        self.max_batch_size = max_batch_size
        self.max_wait_ms = max_wait_ms
        self.queue = []
        self.lock = asyncio.Lock()
    
    async def generate(self, messages):
        future = asyncio.get_event_loop().create_future()
        async with self.lock:
            self.queue.append(BatchRequest(messages, future))
            if len(self.queue) == 1:
                asyncio.create_task(self._process_batch())
        return await future
    
    async def _process_batch(self):
        await asyncio.sleep(self.max_wait_ms / 1000)
        async with self.lock:
            batch = self.queue[:self.max_batch_size]
            self.queue = self.queue[self.max_batch_size:]
        
        tasks = [self._call_single(r.messages) for r in batch]
        responses = await asyncio.gather(*tasks)
        for req, resp in zip(batch, responses):
            req.future.set_result(resp)

Quantization for Cost Reduction

# AWQ 4-bit quantization reduces VRAM by 4x with minimal quality loss
from vllm import LLM
llm = LLM(
    model="TheBloke/Llama-2-7B-Chat-AWQ",
    quantization="awq",
)

# Memory comparison:
# Full precision (float16): 7B model = 14GB VRAM
# AWQ 4-bit: 7B model = ~4GB VRAM

Cost Optimization: Model Routing

class LLMRouter:
    def classify_complexity(self, prompt):
        if len(prompt) < 100:
            return "simple"
        if any(kw in prompt.lower() for kw in ["code", "analyze", "compare"]):
            return "complex"
        return "simple"
    
    def generate(self, prompt):
        complexity = self.classify_complexity(prompt)
        model = "gpt-4o-mini" if complexity == "simple" else "gpt-4o"
        # gpt-4o-mini is 10x cheaper than gpt-4o
        return self.client.chat.completions.create(
            model=model,
            messages=[{"role": "user", "content": prompt}],
        ).choices[0].message.content

Monitoring

from prometheus_client import Histogram, Counter

LATENCY = Histogram("llm_request_duration_seconds", "LLM latency", ["model"])
TOKENS = Counter("llm_tokens_total", "Total tokens", ["model", "type"])

def monitored_generate(client, messages, model):
    import time
    start = time.time()
    
    response = client.chat.completions.create(model=model, messages=messages)
    
    TOKENS.labels(model=model, type="prompt").inc(response.usage.prompt_tokens)
    TOKENS.labels(model=model, type="completion").inc(response.usage.completion_tokens)
    LATENCY.labels(model=model).observe(time.time() - start)
    
    return response.choices[0].message.content

Summary

Production LLM deployment needs:

• vLLM/TGI: Purpose-built serving for maximum throughput
• Caching: Exact and semantic caching for repeated queries
• Batching: Group requests for better GPU utilization
• Routing: Send simple tasks to cheaper models
• Quantization: AWQ/GGUF reduces VRAM 4x with minimal quality loss
• Monitoring: Track latency, tokens, and costs per model