scikit-learn: Model Evaluation, Pipelines & Hyperparameter Tuning

Model Evaluation

from sklearn.metrics import (
    accuracy_score, precision_score, recall_score, f1_score,
    roc_auc_score, average_precision_score,
    confusion_matrix, classification_report,
    mean_squared_error, mean_absolute_error, r2_score
)

y_pred = model.predict(X_test)
y_proba = model.predict_proba(X_test)[:, 1]   # binary positive class

# Classification metrics
print(f"Accuracy:  {accuracy_score(y_test, y_pred):.4f}")
print(f"Precision: {precision_score(y_test, y_pred):.4f}")
print(f"Recall:    {recall_score(y_test, y_pred):.4f}")
print(f"F1:        {f1_score(y_test, y_pred):.4f}")
print(f"ROC-AUC:   {roc_auc_score(y_test, y_proba):.4f}")

# Full report
print(classification_report(y_test, y_pred, target_names=['neg', 'pos']))

# Confusion matrix
cm = confusion_matrix(y_test, y_pred)
# [[TN, FP],
#  [FN, TP]]

# Cross-validation
from sklearn.model_selection import cross_val_score, StratifiedKFold

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=cv, scoring='f1')
print(f"CV F1: {scores.mean():.4f} ± {scores.std():.4f}")

# Multiple metrics at once
from sklearn.model_selection import cross_validate
results = cross_validate(model, X, y, cv=5,
    scoring=['accuracy', 'f1', 'roc_auc'], return_train_score=True)

Pipelines

Pipelines chain preprocessing and model steps. They prevent data leakage by fitting transformers only on training data, and make deployment simpler.

from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.compose import ColumnTransformer, make_column_selector
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.ensemble import RandomForestClassifier

# Column transformer (different preprocessing per column type)
numeric_features = ['age', 'income', 'credit_score']
categorical_features = ['job', 'marital', 'education']

numeric_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler()),
])

categorical_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='most_frequent')),
    ('encoder', OneHotEncoder(handle_unknown='ignore')),
])

preprocessor = ColumnTransformer([
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features),
])

# Full pipeline
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100, random_state=42)),
])

# Fit and evaluate — preprocessing applied correctly to train/test
pipeline.fit(X_train, y_train)
score = pipeline.score(X_test, y_test)
y_pred = pipeline.predict(X_test)

# Save the entire pipeline (includes scaler, encoder, model)
import joblib
joblib.dump(pipeline, 'model.pkl')
loaded = joblib.load('model.pkl')
loaded.predict(X_new)    # preprocessing applied automatically

Hyperparameter Tuning

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import randint, uniform

# Grid search (exhaustive — all combinations)
param_grid = {
    'classifier__n_estimators': [50, 100, 200],
    'classifier__max_depth': [None, 5, 10, 20],
    'classifier__min_samples_split': [2, 5, 10],
}
grid_search = GridSearchCV(
    pipeline, param_grid,
    cv=5, scoring='f1',
    n_jobs=-1,           # use all CPU cores
    verbose=2,
)
grid_search.fit(X_train, y_train)
print(f"Best params: {grid_search.best_params_}")
print(f"Best CV F1:  {grid_search.best_score_:.4f}")
best_model = grid_search.best_estimator_

# Randomized search (faster for large search spaces)
param_dist = {
    'classifier__n_estimators': randint(50, 500),
    'classifier__max_depth': [None, 5, 10, 20, 30],
    'classifier__min_samples_split': randint(2, 20),
    'classifier__max_features': ['sqrt', 'log2', None],
}
random_search = RandomizedSearchCV(
    pipeline, param_dist,
    n_iter=50,           # try 50 random combinations
    cv=5, scoring='roc_auc',
    n_jobs=-1, random_state=42,
)
random_search.fit(X_train, y_train)

Tips & Common Pitfalls

• Always fit scalers/imputers on training data only — transform both train and test. Pipelines enforce this.

• stratify=y in train_test_split for imbalanced classification — preserves class ratios.

• For imbalanced classes: use class_weight="balanced", oversample (SMOTE via imbalanced-learn), or adjust threshold.

• Check for data leakage: no future information in features, no test data used in preprocessing.

• Feature importance != causation. Correlated features share importance — use permutation importance for reliable estimates.

• Cross-validation score overestimates performance if hyperparameters were tuned on the same data — use nested CV.

• Use joblib for saving models — more reliable than pickle for numpy arrays.

• LightGBM and XGBoost outperform scikit-learn's GradientBoostingClassifier in speed and accuracy for tabular data.