scikit-learn: Estimators, Preprocessing & Feature Engineering

scikit-learn is Python's standard machine learning library. All objects follow a consistent API: estimators have fit(), transformers have fit_transform(), and predictors have predict(). This uniformity makes it easy to build pipelines.

Data Splitting

from sklearn.model_selection import train_test_split
import numpy as np

X, y = load_data()   # features and labels

# Train/test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y,
    test_size=0.2,
    random_state=42,
    stratify=y        # preserve class distribution (for classification)
)

# Train/validation/test split
X_temp, X_test, y_temp, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_temp, y_temp, test_size=0.25, random_state=42)
# Result: 60% train / 20% val / 20% test

Preprocessing

from sklearn.preprocessing import (
    StandardScaler, MinMaxScaler, RobustScaler,
    LabelEncoder, OrdinalEncoder, OneHotEncoder,
    PolynomialFeatures, Normalizer, PowerTransformer
)

# Numerical scaling
scaler = StandardScaler()       # zero mean, unit variance (z-score)
X_train_scaled = scaler.fit_transform(X_train)  # fit on train only!
X_test_scaled = scaler.transform(X_test)         # only transform test

MinMaxScaler()          # scale to [0, 1]
RobustScaler()          # uses median/IQR — robust to outliers
Normalizer(norm='l2')   # normalize each sample to unit norm

# Categorical encoding
enc = OneHotEncoder(sparse_output=False, handle_unknown='ignore')
X_encoded = enc.fit_transform(X_categorical)
# Creates one column per category (0 or 1)

OrdinalEncoder()        # encode categories as integers (for ordinal data)
LabelEncoder()          # encode target labels (y), not features

# Missing values
from sklearn.impute import SimpleImputer, KNNImputer
imputer = SimpleImputer(strategy='mean')    # or 'median', 'most_frequent', 'constant'
imputer = KNNImputer(n_neighbors=5)         # fill using nearest neighbors

# Feature engineering
poly = PolynomialFeatures(degree=2, include_bias=False)
X_poly = poly.fit_transform(X)              # adds x1^2, x2^2, x1*x2, etc.

PowerTransformer(method='yeo-johnson')      # makes distribution more Gaussian

Feature Selection

from sklearn.feature_selection import (
    SelectKBest, f_classif, chi2, mutual_info_classif,
    RFE, SelectFromModel, VarianceThreshold
)
from sklearn.linear_model import Lasso

# Remove low-variance features
selector = VarianceThreshold(threshold=0.01)
X_selected = selector.fit_transform(X)

# Statistical tests (for classification)
selector = SelectKBest(score_func=f_classif, k=10)   # ANOVA F-value
selector = SelectKBest(score_func=chi2, k=10)         # Chi-squared (non-negative features)
selector = SelectKBest(score_func=mutual_info_classif, k=10)
X_selected = selector.fit_transform(X, y)

# Recursive Feature Elimination
from sklearn.ensemble import RandomForestClassifier
rfe = RFE(estimator=RandomForestClassifier(), n_features_to_select=10)
rfe.fit(X_train, y_train)
X_selected = rfe.transform(X)

# L1-based selection (Lasso zeroes out unimportant features)
lasso = Lasso(alpha=0.01)
selector = SelectFromModel(lasso)
selector.fit(X_train, y_train)
X_selected = selector.transform(X)

# Feature importance from tree-based models
rf = RandomForestClassifier().fit(X_train, y_train)
importances = rf.feature_importances_
sorted_idx = np.argsort(importances)[::-1]

scikit-learn: Classification, Regression & Clustering

Classification Algorithms

from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import (
    RandomForestClassifier, GradientBoostingClassifier,
    AdaBoostClassifier, BaggingClassifier, VotingClassifier
)
from sklearn.svm import SVC, LinearSVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.neural_network import MLPClassifier

# Common classifiers with sensible defaults
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000, C=1.0),
    'Random Forest':       RandomForestClassifier(n_estimators=100, random_state=42),
    'Gradient Boosting':   GradientBoostingClassifier(n_estimators=100, learning_rate=0.1),
    'SVM (RBF kernel)':    SVC(C=1.0, kernel='rbf', probability=True),
    'KNN':                 KNeighborsClassifier(n_neighbors=5),
    'Naive Bayes':         GaussianNB(),
}

# Fit and predict
for name, model in models.items():
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    print(f"{name}: {score:.4f}")

# Predict probabilities
model.predict(X_test)                  # class labels
model.predict_proba(X_test)            # probability per class
model.predict_proba(X_test)[:, 1]     # positive class probability (binary)

# Multiclass strategies
from sklearn.multiclass import OneVsRestClassifier, OneVsOneClassifier
ovr = OneVsRestClassifier(SVC())
ovr.fit(X_train, y_train)

Regression Algorithms

from sklearn.linear_model import (
    LinearRegression, Ridge, Lasso, ElasticNet, BayesianRidge
)
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor

# Linear models
LinearRegression()              # OLS — no regularization
Ridge(alpha=1.0)                # L2 regularization — shrinks coefficients
Lasso(alpha=0.1)                # L1 regularization — zeroes out features
ElasticNet(alpha=0.1, l1_ratio=0.5)  # L1 + L2

# Tree-based
RandomForestRegressor(n_estimators=100, random_state=42)
GradientBoostingRegressor(n_estimators=200, learning_rate=0.05, max_depth=4)

# Fit and predict
reg = Ridge(alpha=1.0)
reg.fit(X_train, y_train)
y_pred = reg.predict(X_test)

# Coefficients
reg.coef_                   # feature coefficients
reg.intercept_              # bias term

from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

Clustering

from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering, MeanShift
from sklearn.mixture import GaussianMixture

# K-Means (must specify k)
kmeans = KMeans(n_clusters=5, random_state=42, n_init=10)
labels = kmeans.fit_predict(X)
centers = kmeans.cluster_centers_

# Elbow method to find optimal k
inertias = []
for k in range(1, 11):
    km = KMeans(n_clusters=k, random_state=42, n_init=10).fit(X)
    inertias.append(km.inertia_)

# DBSCAN (density-based — finds clusters of arbitrary shape, handles noise)
dbscan = DBSCAN(eps=0.5, min_samples=5)
labels = dbscan.fit_predict(X)
# Labels: -1 = noise, 0, 1, 2, ... = clusters

# Agglomerative (hierarchical)
agg = AgglomerativeClustering(n_clusters=5, linkage='ward')
labels = agg.fit_predict(X)

# Evaluate clustering (no ground truth)
from sklearn.metrics import silhouette_score, davies_bouldin_score
sil_score = silhouette_score(X, labels)      # higher = better (-1 to 1)
db_score = davies_bouldin_score(X, labels)   # lower = better

# Dimensionality reduction for visualization
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE

pca = PCA(n_components=2)
X_2d = pca.fit_transform(X)

tsne = TSNE(n_components=2, random_state=42, perplexity=30)
X_tsne = tsne.fit_transform(X)

scikit-learn: Model Evaluation, Pipelines & Hyperparameter Tuning

Model Evaluation

from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    roc_auc_score, average_precision_score,
    confusion_matrix, classification_report,
    mean_squared_error, mean_absolute_error, r2_score
)

y_pred = model.predict(X_test)
y_proba = model.predict_proba(X_test)[:, 1]   # binary positive class

# Classification metrics
print(f"Accuracy:  {accuracy_score(y_test, y_pred):.4f}")
print(f"Precision: {precision_score(y_test, y_pred):.4f}")
print(f"Recall:    {recall_score(y_test, y_pred):.4f}")
print(f"F1:        {f1_score(y_test, y_pred):.4f}")
print(f"ROC-AUC:   {roc_auc_score(y_test, y_proba):.4f}")

# Full report
print(classification_report(y_test, y_pred, target_names=['neg', 'pos']))

# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
# [[TN, FP],
#  [FN, TP]]

# Cross-validation
from sklearn.model_selection import cross_val_score, StratifiedKFold

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=cv, scoring='f1')
print(f"CV F1: {scores.mean():.4f} ± {scores.std():.4f}")

# Multiple metrics at once
from sklearn.model_selection import cross_validate
results = cross_validate(model, X, y, cv=5,
    scoring=['accuracy', 'f1', 'roc_auc'], return_train_score=True)

Pipelines

Pipelines chain preprocessing and model steps. They prevent data leakage by fitting transformers only on training data, and make deployment simpler.

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.compose import ColumnTransformer, make_column_selector
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier

# Column transformer (different preprocessing per column type)
numeric_features = ['age', 'income', 'credit_score']
categorical_features = ['job', 'marital', 'education']

numeric_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler()),
])

categorical_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(handle_unknown='ignore')),
])

preprocessor = ColumnTransformer([
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features),
])

# Full pipeline
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42)),
])

# Fit and evaluate — preprocessing applied correctly to train/test
pipeline.fit(X_train, y_train)
score = pipeline.score(X_test, y_test)
y_pred = pipeline.predict(X_test)

# Save the entire pipeline (includes scaler, encoder, model)
import joblib
joblib.dump(pipeline, 'model.pkl')
loaded = joblib.load('model.pkl')
loaded.predict(X_new)    # preprocessing applied automatically

Hyperparameter Tuning

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import randint, uniform

# Grid search (exhaustive — all combinations)
param_grid = {
    'classifier__n_estimators': [50, 100, 200],
    'classifier__max_depth': [None, 5, 10, 20],
    'classifier__min_samples_split': [2, 5, 10],
}
grid_search = GridSearchCV(
    pipeline, param_grid,
    cv=5, scoring='f1',
    n_jobs=-1,           # use all CPU cores
    verbose=2,
)
grid_search.fit(X_train, y_train)
print(f"Best params: {grid_search.best_params_}")
print(f"Best CV F1:  {grid_search.best_score_:.4f}")
best_model = grid_search.best_estimator_

# Randomized search (faster for large search spaces)
param_dist = {
    'classifier__n_estimators': randint(50, 500),
    'classifier__max_depth': [None, 5, 10, 20, 30],
    'classifier__min_samples_split': randint(2, 20),
    'classifier__max_features': ['sqrt', 'log2', None],
}
random_search = RandomizedSearchCV(
    pipeline, param_dist,
    n_iter=50,           # try 50 random combinations
    cv=5, scoring='roc_auc',
    n_jobs=-1, random_state=42,
)
random_search.fit(X_train, y_train)

Tips & Common Pitfalls

• Always fit scalers/imputers on training data only — transform both train and test. Pipelines enforce this.

• stratify=y in train_test_split for imbalanced classification — preserves class ratios.

• For imbalanced classes: use class_weight="balanced", oversample (SMOTE via imbalanced-learn), or adjust threshold.

• Check for data leakage: no future information in features, no test data used in preprocessing.

• Feature importance != causation. Correlated features share importance — use permutation importance for reliable estimates.

• Cross-validation score overestimates performance if hyperparameters were tuned on the same data — use nested CV.

• Use joblib for saving models — more reliable than pickle for numpy arrays.

• LightGBM and XGBoost outperform scikit-learn's GradientBoostingClassifier in speed and accuracy for tabular data.