Building Kubernetes Operators with Go
Kubernetes operators extend the Kubernetes API to automate the management of complex stateful applications. This guide covers building production-ready operators using controller-runtime.
What is an Operator?
An operator encodes human operational knowledge into software:
- Custom Resource Definition (CRD): Define your API
- Controller: Watch resources and reconcile desired vs actual state
- Reconciliation loop: The core logic that makes state happen
Prerequisites
# Install Operator SDK
curl -LO https://github.com/operator-framework/operator-sdk/releases/latest/download/operator-sdk_linux_amd64
chmod +x operator-sdk_linux_amd64
mv operator-sdk_linux_amd64 /usr/local/bin/operator-sdk
# Create new operator project
operator-sdk init --domain=example.com --repo=github.com/myorg/myoperator
operator-sdk create api --group=apps --version=v1alpha1 --kind=Database --resource --controller
Defining the CRD
// api/v1alpha1/database_types.go
package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
// DatabaseSpec defines the desired state
type DatabaseSpec struct {
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=10
Replicas int32 `json:"replicas"`
// +kubebuilder:validation:Enum=postgres;mysql;mariadb
Engine string `json:"engine"`
Version string `json:"version"`
Storage StorageSpec `json:"storage"`
}
type StorageSpec struct {
// +kubebuilder:validation:Pattern=`^[0-9]+[KMGT]i
Building Kubernetes Operators with Go: Automate Complex Workloads | MyUtl
MyUtl
正在加载，请稍候…
Size string `json:"size"`
// +optional
StorageClassName *string `json:"storageClassName,omitempty"`
}
// DatabaseStatus defines the observed state
type DatabaseStatus struct {
// +patchMergeKey=type
// +patchStrategy=merge
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty"`
ReadyReplicas int32 `json:"readyReplicas,omitempty"`
Phase string `json:"phase,omitempty"` // Pending, Running, Failed
}
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="Engine",type="string",JSONPath=".spec.engine"
// +kubebuilder:printcolumn:name="Replicas",type="integer",JSONPath=".spec.replicas"
// +kubebuilder:printcolumn:name="Ready",type="integer",JSONPath=".status.readyReplicas"
// +kubebuilder:printcolumn:name="Phase",type="string",JSONPath=".status.phase"
// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
type Database struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec DatabaseSpec `json:"spec,omitempty"`
Status DatabaseStatus `json:"status,omitempty"`
}
The Controller: Reconciliation Loop
// controllers/database_controller.go
package controllers
import (
"context"
"fmt"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/log"
appsv1alpha1 "github.com/myorg/myoperator/api/v1alpha1"
)
type DatabaseReconciler struct {
client.Client
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=apps.example.com,resources=databases,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=apps.example.com,resources=databases/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
func (r *DatabaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logger := log.FromContext(ctx)
// Fetch the Database instance
db := &appsv1alpha1.Database{}
if err := r.Get(ctx, req.NamespacedName, db); err != nil {
if errors.IsNotFound(err) {
return ctrl.Result{}, nil // Resource deleted
}
return ctrl.Result{}, err
}
// Set the phase to Pending initially
if db.Status.Phase == "" {
db.Status.Phase = "Pending"
if err := r.Status().Update(ctx, db); err != nil {
return ctrl.Result{}, err
}
}
// Reconcile StatefulSet
sts, err := r.reconcileStatefulSet(ctx, db)
if err != nil {
logger.Error(err, "Failed to reconcile StatefulSet")
return ctrl.Result{}, err
}
// Reconcile Service
if err := r.reconcileService(ctx, db); err != nil {
logger.Error(err, "Failed to reconcile Service")
return ctrl.Result{}, err
}
// Update status
db.Status.ReadyReplicas = sts.Status.ReadyReplicas
if sts.Status.ReadyReplicas == db.Spec.Replicas {
db.Status.Phase = "Running"
} else {
db.Status.Phase = "Pending"
}
if err := r.Status().Update(ctx, db); err != nil {
return ctrl.Result{}, err
}
logger.Info("Reconciled Database", "phase", db.Status.Phase)
return ctrl.Result{}, nil
}
Creating Child Resources
func (r *DatabaseReconciler) reconcileStatefulSet(
ctx context.Context,
db *appsv1alpha1.Database,
) (*appsv1.StatefulSet, error) {
desired := r.desiredStatefulSet(db)
// Set owner reference for garbage collection
if err := ctrl.SetControllerReference(db, desired, r.Scheme); err != nil {
return nil, err
}
// Get existing StatefulSet
existing := &appsv1.StatefulSet{}
err := r.Get(ctx, client.ObjectKeyFromObject(desired), existing)
if errors.IsNotFound(err) {
// Create
if err := r.Create(ctx, desired); err != nil {
return nil, fmt.Errorf("creating StatefulSet: %w", err)
}
return desired, nil
}
if err != nil {
return nil, err
}
// Update if spec changed
existing.Spec.Replicas = desired.Spec.Replicas
existing.Spec.Template = desired.Spec.Template
if err := r.Update(ctx, existing); err != nil {
return nil, fmt.Errorf("updating StatefulSet: %w", err)
}
return existing, nil
}
func (r *DatabaseReconciler) desiredStatefulSet(db *appsv1alpha1.Database) *appsv1.StatefulSet {
labels := map[string]string{
"app": db.Name,
"managed-by": "database-operator",
}
return &appsv1.StatefulSet{
ObjectMeta: metav1.ObjectMeta{
Name: db.Name,
Namespace: db.Namespace,
Labels: labels,
},
Spec: appsv1.StatefulSetSpec{
Replicas: &db.Spec.Replicas,
Selector: &metav1.LabelSelector{MatchLabels: labels},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{Labels: labels},
Spec: corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "database",
Image: fmt.Sprintf("%s:%s", db.Spec.Engine, db.Spec.Version),
Ports: []corev1.ContainerPort{
{ContainerPort: 5432, Name: "db"},
},
},
},
},
},
},
}
}
Status Conditions
// Update conditions following Kubernetes API conventions
func (r *DatabaseReconciler) setCondition(
db *appsv1alpha1.Database,
condType string,
status metav1.ConditionStatus,
reason, message string,
) {
condition := metav1.Condition{
Type: condType,
Status: status,
Reason: reason,
Message: message,
LastTransitionTime: metav1.Now(),
ObservedGeneration: db.Generation,
}
// Check if condition already exists with same status
for i, c := range db.Status.Conditions {
if c.Type == condType {
if c.Status == status {
condition.LastTransitionTime = c.LastTransitionTime
}
db.Status.Conditions[i] = condition
return
}
}
db.Status.Conditions = append(db.Status.Conditions, condition)
}
// Usage
r.setCondition(db, "Available", metav1.ConditionTrue, "ReplicasReady", "All replicas are running")
r.setCondition(db, "Available", metav1.ConditionFalse, "ReplicasPending", "Waiting for replicas to start")
Watching Related Resources
func (r *DatabaseReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&appsv1alpha1.Database{}).
// Watch owned StatefulSets and trigger reconcile
Owns(&appsv1.StatefulSet{}).
// Watch owned Services
Owns(&corev1.Service{}).
// Custom predicate: only reconcile when spec changes
WithEventFilter(predicate.GenerationChangedPredicate{}).
Complete(r)
}
Deploying the Operator
# Generate CRD manifests from Go types
make manifests
# Install CRDs
make install
# Run locally for testing
make run
# Build and push image
make docker-build IMG=my-registry/myoperator:v0.1.0
make docker-push IMG=my-registry/myoperator:v0.1.0
# Deploy to cluster
make deploy IMG=my-registry/myoperator:v0.1.0
# Create a Database resource
cat <<EOF | kubectl apply -f -
apiVersion: apps.example.com/v1alpha1
kind: Database
metadata:
name: my-postgres
namespace: default
spec:
engine: postgres
version: "15.4"
replicas: 3
storage:
size: 10Gi
EOF
Best Practices
- Idempotent reconciliation: Always reconcile toward desired state, never assume state
- Owner references: Set
ctrl.SetControllerReferencefor child resources - Status subresource: Update
.statusseparately from.spec - Requeue on error: Return
ctrl.Result{RequeueAfter: time.Second * 10}for transient errors - Finalizers: Use finalizers for cleanup before deletion
- RBAC: Use minimal required permissions
Summary
Kubernetes operators automate complex operational tasks:
- Define the desired state with CRDs
- Implement reconciliation loops that converge to desired state
- Use owner references for automatic garbage collection
- Update status conditions following Kubernetes conventions
- Test with envtest for controller unit tests