Data Preprocessing and Feature Engineering

Quality features matter more than algorithm choice in most ML problems.

Missing Value Strategies

import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer, KNNImputer
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

df = pd.read_csv("dataset.csv")

# Analyze missingness
missing = df.isnull().sum()
missing_pct = missing / len(df) * 100
print(missing_pct[missing_pct > 0])

# Strategy by type
def impute_dataset(df: pd.DataFrame) -> pd.DataFrame:
    df = df.copy()

    # Drop columns with >50% missing
    df = df.dropna(thresh=len(df) * 0.5, axis=1)

    # Numerical: median imputation
    num_cols = df.select_dtypes(include=[np.number]).columns
    num_imputer = SimpleImputer(strategy="median")
    df[num_cols] = num_imputer.fit_transform(df[num_cols])

    # Categorical: mode imputation
    cat_cols = df.select_dtypes(include=["object"]).columns
    cat_imputer = SimpleImputer(strategy="most_frequent")
    df[cat_cols] = cat_imputer.fit_transform(df[cat_cols])

    return df

# Advanced: iterative imputation (MICE)
mice_imputer = IterativeImputer(max_iter=10, random_state=42)
df_imputed = pd.DataFrame(mice_imputer.fit_transform(df), columns=df.columns)

Encoding Categorical Variables

from sklearn.preprocessing import LabelEncoder, OrdinalEncoder
import category_encoders as ce

# Target encoding (for high-cardinality features)
target_encoder = ce.TargetEncoder(cols=["city", "product_category"])
X_encoded = target_encoder.fit_transform(X_train, y_train)

# Frequency encoding
def frequency_encode(df: pd.DataFrame, col: str) -> pd.Series:
    freq_map = df[col].value_counts(normalize=True)
    return df[col].map(freq_map)

# WOE encoding for binary classification
woe_encoder = ce.WOEEncoder(cols=["customer_segment"])
X_woe = woe_encoder.fit_transform(X_train, y_train)

# Hash encoding for very high cardinality
hash_encoder = ce.HashingEncoder(cols=["user_id"], n_components=16)
X_hash = hash_encoder.fit_transform(X_train)

Feature Scaling

from sklearn.preprocessing import StandardScaler, RobustScaler, MinMaxScaler, PowerTransformer

# Check distribution before scaling
import matplotlib.pyplot as plt
df[num_cols].hist(bins=30, figsize=(15, 10))

# Standard scaling (assume normal distribution)
scaler = StandardScaler()

# Robust scaling (for outliers)
robust_scaler = RobustScaler()

# Log transform for skewed features
from scipy.stats import skew

def auto_transform(df: pd.DataFrame, threshold: float = 0.5) -> pd.DataFrame:
    df = df.copy()
    for col in df.select_dtypes(include=[np.number]).columns:
        if df[col].skew() > threshold:
            df[col] = np.log1p(df[col].clip(lower=0))
    return df

# PowerTransformer for Gaussian normalization
pt = PowerTransformer(method="yeo-johnson")
df_transformed = pd.DataFrame(pt.fit_transform(df[num_cols]), columns=num_cols)

Automated Feature Engineering

import featuretools as ft

# Create entity set
es = ft.EntitySet(id="customer_data")

es = es.add_dataframe(
    dataframe=orders_df,
    dataframe_name="orders",
    index="order_id",
    time_index="order_date",
)

es = es.add_dataframe(
    dataframe=customers_df,
    dataframe_name="customers",
    index="customer_id",
)

es = es.add_relationship("customers", "customer_id", "orders", "customer_id")

# Auto-generate features
feature_matrix, feature_defs = ft.dfs(
    entityset=es,
    target_dataframe_name="customers",
    agg_primitives=["count", "sum", "mean", "max", "min", "std"],
    trans_primitives=["year", "month", "weekday"],
    max_depth=2,
)

print(f"Generated {len(feature_defs)} features")

Feature Selection

from sklearn.feature_selection import (
    SelectKBest, f_classif, mutual_info_classif,
    RFECV, SelectFromModel,
)
from sklearn.ensemble import RandomForestClassifier

# Univariate selection
selector = SelectKBest(score_func=mutual_info_classif, k=20)
X_selected = selector.fit_transform(X_train, y_train)
selected_features = X_train.columns[selector.get_support()].tolist()

# Recursive Feature Elimination
rfecv = RFECV(estimator=RandomForestClassifier(n_estimators=100), cv=5, scoring="roc_auc")
rfecv.fit(X_train, y_train)
important_features = X_train.columns[rfecv.support_].tolist()

# SHAP-based selection
import shap

rf = RandomForestClassifier(n_estimators=100).fit(X_train, y_train)
explainer = shap.TreeExplainer(rf)
shap_values = explainer.shap_values(X_test)

mean_shap = np.abs(shap_values[1]).mean(axis=0)
feature_importance = pd.Series(mean_shap, index=X_train.columns).sort_values(ascending=False)
top_features = feature_importance.head(20).index.tolist()

Pipeline Composition

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer

num_pipeline = Pipeline([
    ("imputer", SimpleImputer(strategy="median")),
    ("scaler", RobustScaler()),
])

cat_pipeline = Pipeline([
    ("imputer", SimpleImputer(strategy="most_frequent")),
    ("encoder", ce.TargetEncoder()),
])

preprocessor = ColumnTransformer([
    ("num", num_pipeline, num_cols),
    ("cat", cat_pipeline, cat_cols),
])

full_pipeline = Pipeline([
    ("preprocessor", preprocessor),
    ("classifier", RandomForestClassifier(n_estimators=200)),
])

full_pipeline.fit(X_train, y_train)
predictions = full_pipeline.predict(X_test)

Feature Engineering Checklist