Observability: Distributed Tracing with OpenTelemetry

The Three Pillars of Observability

Logs:    What happened (discrete events)
         "User 123 failed login at 10:23:45"

Metrics: How much / how often (aggregated numbers)
         HTTP requests/sec, error rate, P99 latency

Traces:  Where time was spent (distributed request flow)
         Request: 250ms total
           API Gateway: 5ms
           User Service: 30ms
             Database: 25ms
           Order Service: 195ms
             Database: 150ms
             Payment API: 40ms

OpenTelemetry Setup

// tracing.ts - Must be imported BEFORE your app
import { NodeSDK } from '@opentelemetry/sdk-node';
import { Resource } from '@opentelemetry/resources';
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http';
import { ExpressInstrumentation } from '@opentelemetry/instrumentation-express';

const sdk = new NodeSDK({
  resource: new Resource({ 'service.name': 'user-service', 'service.version': '1.2.3' }),
  traceExporter: new OTLPTraceExporter({ url: 'http://jaeger:4318/v1/traces' }),
  instrumentations: [new HttpInstrumentation(), new ExpressInstrumentation()],
});
sdk.start();

Manual Instrumentation

import { trace, SpanStatusCode } from '@opentelemetry/api';
const tracer = trace.getTracer('user-service');

async function processOrder(orderId: string): Promise<Order> {
  return tracer.startActiveSpan('processOrder', async (span) => {
    span.setAttributes({ 'order.id': orderId });
    try {
      const order = await fetchOrder(orderId);
      await chargePayment(order);
      span.setStatus({ code: SpanStatusCode.OK });
      return order;
    } catch (err) {
      span.setStatus({ code: SpanStatusCode.ERROR, message: err.message });
      span.recordException(err);
      throw err;
    } finally {
      span.end();
    }
  });
}

Context Propagation

import { propagation, context } from '@opentelemetry/api';

// Producer: inject trace context into Kafka message headers
async function publishEvent(event: object): Promise<void> {
  const headers: Record<string, string> = {};
  propagation.inject(context.active(), headers); // Adds traceparent header
  await kafka.producer.send({ topic: 'events', messages: [{ value: JSON.stringify(event), headers }] });
}

// Consumer: extract and continue trace
async function handleMessage(message: KafkaMessage): Promise<void> {
  const parentContext = propagation.extract(context.active(), message.headers ?? {});
  await context.with(parentContext, () =>
    tracer.startActiveSpan('handleEvent', span => {
      processEvent(JSON.parse(message.value!.toString()));
      span.end();
    })
  );
}

Sampling

import { ParentBasedSampler, TraceIdRatioBasedSampler } from '@opentelemetry/sdk-trace-node';

const sdk = new NodeSDK({
  sampler: new ParentBasedSampler({
    root: new TraceIdRatioBasedSampler(0.1), // Sample 10% of traces
  }),
});
// If parent is sampled, child is sampled (distributed traces stay together)

Jaeger Quick Start

# docker-compose.yml
services:
  jaeger:
    image: jaegertracing/all-in-one:latest
    ports:
      - "16686:16686"  # UI
      - "4318:4318"    # OTLP HTTP
    environment:
      COLLECTOR_OTLP_ENABLED: true

Distributed tracing answers "why is this request slow?" across microservice boundaries.