Scikit-learn Guide: Machine Learning in Python

Machine Learning Machine Learning 7 min read 1455 words Beginner ExcellentWiki Editorial Team

Scikit-learn is the most widely used machine learning library in Python. It provides a consistent API for classification, regression, clustering, dimensionality reduction, model selection, and preprocessing. This guide covers the essential workflows.

The Estimator API

Every scikit-learn model follows the same interface:

fit(X, y) — train the model
predict(X) — make predictions
score(X, y) — evaluate accuracy/R²
transform(X) — transform data (preprocessing, dimensionality reduction)

from sklearn.ensemble import RandomForestClassifier

model = RandomForestClassifier(n_estimators=100)
model.fit(X_train, y_train)
predictions = model.predict(X_test)
accuracy = model.score(X_test, y_test)

Supervised Learning

Classification

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB

# Common classifiers
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000),
    'SVM': SVC(kernel='rbf', probability=True),
    'Decision Tree': DecisionTreeClassifier(max_depth=5),
    'Random Forest': RandomForestClassifier(n_estimators=100),
    'Gradient Boosting': GradientBoostingClassifier(n_estimators=100),
    'KNN': KNeighborsClassifier(n_neighbors=5),
---

# Evaluate all
for name, model in models.items():
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    print(f"{name}: {score:.3f}")

Regression

from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.svm import SVR

regressors = {
    'Linear Regression': LinearRegression(),
    'Ridge': Ridge(alpha=1.0),
    'Lasso': Lasso(alpha=0.1),
    'Random Forest': RandomForestRegressor(n_estimators=100),
    'Gradient Boosting': GradientBoostingRegressor(n_estimators=100),
---

for name, model in regressors.items():
    model.fit(X_train, y_train)
    score = model.score(X_test, y_test)
    print(f"{name} R²: {score:.3f}")

Unsupervised Learning

Clustering

from sklearn.cluster import KMeans, DBSCAN, AgglomerativeClustering
from sklearn.metrics import silhouette_score

# K-Means
kmeans = KMeans(n_clusters=3, random_state=42, n_init=10)
kmeans.fit(X)
labels = kmeans.labels_
centroids = kmeans.cluster_centers_

# Find optimal k using silhouette score
scores = []
for k in range(2, 11):
    kmeans = KMeans(n_clusters=k, random_state=42, n_init=10)
    kmeans.fit(X)
    score = silhouette_score(X, kmeans.labels_)
    scores.append(score)
    print(f"k={k}: silhouette={score:.3f}")

# DBSCAN (density-based, detects outliers)
dbscan = DBSCAN(eps=0.5, min_samples=5)
dbscan.fit(X)
n_clusters = len(set(dbscan.labels_)) - (1 if -1 in dbscan.labels_ else 0)
n_noise = list(dbscan.labels_).count(-1)
print(f"DBSCAN: {n_clusters} clusters, {n_noise} noise points")

Dimensionality Reduction

from sklearn.decomposition import PCA, TruncatedSVD
from sklearn.manifold import TSNE, Isomap

# PCA — linear dimensionality reduction
pca = PCA(n_components=0.95)  # keep 95% of variance
X_pca = pca.fit_transform(X)
print(f"Reduced from {X.shape[1]} to {X_pca.shape[1]} dimensions")
print(f"Explained variance ratio: {pca.explained_variance_ratio_.sum():.3f}")

# t-SNE — non-linear visualization (2D or 3D only)
tsne = TSNE(n_components=2, perplexity=30, random_state=42)
X_tsne = tsne.fit_transform(X)

Preprocessing

from sklearn.preprocessing import (
    StandardScaler,
    MinMaxScaler,
    RobustScaler,
    OneHotEncoder,
    LabelEncoder,
    PolynomialFeatures,
)
from sklearn.impute import SimpleImputer

# Feature scaling (essential for distance-based models)
scaler = StandardScaler()       # zero mean, unit variance
# scaler = MinMaxScaler()       # range [0, 1]
# scaler = RobustScaler()       # robust to outliers (uses median/IQR)

X_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)  # use same scaler!

# Handle missing values
imputer = SimpleImputer(strategy='median')  # or 'mean', 'most_frequent'
X_imputed = imputer.fit_transform(X)

# Encoding categorical variables
encoder = OneHotEncoder(drop='first', sparse_output=False)
X_encoded = encoder.fit_transform(X_categorical)

# Polynomial features (capture interactions)
poly = PolynomialFeatures(degree=2, interaction_only=True)
X_poly = poly.fit_transform(X)

Pipelines

Pipelines chain preprocessing and modeling into a single estimator:

from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer

# Define preprocessing for different column types
numeric_features = ['age', 'income', 'total_purchases']
categorical_features = ['plan_type', 'country']

preprocessor = ColumnTransformer(
    transformers=[
        ('num', StandardScaler(), numeric_features),
        ('cat', OneHotEncoder(drop='first'), categorical_features),
    ]
)

# Full pipeline
pipeline = Pipeline([
    ('preprocessor', preprocessor),
    ('classifier', RandomForestClassifier(n_estimators=100)),
])

# Train and predict
pipeline.fit(X_train, y_train)
predictions = pipeline.predict(X_test)

Model Selection

from sklearn.model_selection import (
    train_test_split,
    cross_val_score,
    GridSearchCV,
    RandomizedSearchCV,
    StratifiedKFold,
)

# Train-test split
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# Cross-validation
scores = cross_val_score(
    model, X, y, cv=5, scoring='accuracy'
)
print(f"CV accuracy: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})")

# Grid search (exhaustive)
param_grid = {
    'n_estimators': [100, 200, 300],
    'max_depth': [5, 10, None],
    'min_samples_split': [2, 5, 10],
---

grid_search = GridSearchCV(
    RandomForestClassifier(),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    verbose=1,
)
grid_search.fit(X_train, y_train)
print(f"Best params: {grid_search.best_params_}")
print(f"Best CV score: {grid_search.best_score_:.3f}")

# Random search (more efficient for high dimensions)
random_search = RandomizedSearchCV(
    RandomForestClassifier(),
    param_distributions=param_grid,
    n_iter=20,      # try 20 random combinations
    cv=5,
    scoring='accuracy',
    n_jobs=-1,
    random_state=42,
)

Evaluation Metrics

from sklearn.metrics import (
    accuracy_score,
    precision_score,
    recall_score,
    f1_score,
    confusion_matrix,
    classification_report,
    roc_auc_score,
    mean_squared_error,
    r2_score,
)

# Classification
y_pred = model.predict(X_test)
print(classification_report(y_test, y_pred))

# Regression
y_pred = regressor.predict(X_test)
mse = mean_squared_error(y_test, y_pred)
rmse = np.sqrt(mse)
r2 = r2_score(y_test, y_pred)

Practical Tips

Always scale features for distance-based and linear models
Use pipelines to prevent data leakage between train and test
Cross-validate hyperparameter searches (nested CV for unbiased performance)
Start simple (linear/logistic regression) before complex models
Monitor for overfitting with cross-validation scores
Use randomized search before grid search for large parameter spaces
Set random_state for reproducibility

Real-World Implementation Tips

Production Considerations

When moving from development to production, several factors become critical. Error handling should be comprehensive — every external call (database, API, file system) should have proper error checking, logging, and retry logic where appropriate. Performance monitoring through metrics and structured logging helps identify bottlenecks before they affect users.

Testing Strategy

A thorough testing approach combines multiple levels:

Unit tests verify individual functions and methods in isolation
Integration tests validate that components work together correctly
Edge case tests cover boundary conditions, empty inputs, and error states
Performance tests ensure the system meets latency and throughput requirements

Test data should be realistic but controlled. Mock external dependencies to make tests fast and deterministic. Aim for tests that are independent, repeatable, and fast enough to run on every commit.

Documentation

Good documentation is essential for maintainable code. Follow these principles:

Document the “why” not just the “what” — explain design decisions
Keep examples up to date with the code
Include usage examples for public APIs
Document configuration options and their defaults
Explain error conditions and recovery strategies

Security Best Practices

Security should be considered throughout development:

Validate all inputs at system boundaries
Use parameterized queries for database access
Store secrets in environment variables or secret managers
Keep dependencies updated to patch vulnerabilities
Apply the principle of least privilege

Performance Optimization

Optimize based on measured data, not assumptions:

Profile before optimizing — identify actual bottlenecks
Measure the impact of each change
Consider the trade-off between speed and readability
Cache expensive operations with appropriate invalidation
Use connection pooling for database and network resources

Monitoring and Observability

Production systems need visibility:

Structured logging with correlation IDs for request tracking
Metrics for latency, throughput, error rates, and resource usage
Health check endpoints for load balancers and orchestration
Distributed tracing for request flows across services
Alerts for anomaly detection based on baselines

These patterns apply across all programming languages and frameworks. The specific implementation varies, but the principles remain consistent.

FAQ

Q: Do I need to scale features for tree-based models? A: No — decision trees, random forests, and gradient boosting are invariant to feature scaling.

Q: How do I handle imbalanced classes? A: Use class_weight='balanced', SMOTE for oversampling, or try different thresholds. Evaluate with precision/recall or ROC-AUC, not accuracy.

Q: What is the difference between fit and fit_transform? A: fit learns parameters from data (e.g., mean/std for scaler). transform applies the learned transformation. fit_transform does both on training data. Always use transform (not fit_transform) on test data.

Q: How do I save and load a trained model? A: Use joblib.dump(model, 'model.pkl') and joblib.load('model.pkl'). This preserves the entire fitted estimator, including learned parameters.

Q: What is the difference between predict and predict_proba? A: predict returns the class label. predict_proba returns per-class probabilities. Use probabilities for threshold tuning, confidence scoring, and ROC curve calculation.

Q: Can scikit-learn handle large datasets (10GB+)? A: Scikit-learn loads data into memory. For large datasets, use incremental learning (partial_fit), subsample, or switch to a framework designed for out-of-core learning.

Scikit-Learn Best Practices

Pipeline and ColumnTransformer

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.impute import SimpleImputer
from sklearn.pipeline import Pipeline

numeric_features = ['age', 'fare', 'family_size']
categorical_features = ['sex', 'embarked', 'cabin_letter']

numeric_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('scaler', StandardScaler())
])

categorical_transformer = Pipeline([
    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),
    ('onehot', OneHotEncoder(handle_unknown='ignore', sparse_output=False))
])

preprocessor = ColumnTransformer([
    ('num', numeric_transformer, numeric_features),
    ('cat', categorical_transformer, categorical_features)
])

Model Selection and Hyperparameter Tuning

from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from sklearn.ensemble import RandomForestClassifier

param_dist = {
    'n_estimators': [100, 200, 500],
    'max_depth': [5, 10, 20, None],
    'min_samples_split': [2, 5, 10],
    'min_samples_leaf': [1, 2, 4],
    'max_features': ['sqrt', 'log2']
---

rf = RandomForestClassifier(random_state=42)
random_search = RandomizedSearchCV(
    rf, param_distributions=param_dist,
    n_iter=50, cv=5, scoring='accuracy',
    n_jobs=-1, verbose=1, random_state=42
)
random_search.fit(X_train, y_train)
print(f"Best params: {random_search.best_params_}")
print(f"Best score: {random_search.best_score_:.4f}")

Advanced Scikit-Learn Features

Custom Transformers

from sklearn.base import BaseEstimator, TransformerMixin

class LogTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, offset=1):
        self.offset = offset
    
    def fit(self, X, y=None):
        return self
    
    def transform(self, X):
        return np.log(X + self.offset)

# Use in pipeline
pipeline = Pipeline([
    ('log', LogTransformer(offset=0.5)),
    ('scaler', StandardScaler()),
    ('model', RandomForestClassifier())
])

Feature Importance and Interpretation

# Permutation importance (model-agnostic)
from sklearn.inspection import permutation_importance

result = permutation_importance(model, X_test, y_test, n_repeats=10, random_state=42)
importance_df = pd.DataFrame({
    'feature': X.columns,
    'importance': result.importances_mean,
    'std': result.importances_std
---).sort_values('importance', ascending=False)

# Partial dependence plots
from sklearn.inspection import PartialDependenceDisplay
PartialDependenceDisplay.from_estimator(model, X, ['age', 'income'])

Model Calibration

from sklearn.calibration import CalibratedClassifierCV

# Platt scaling (sigmoid) for probability calibration
calibrated_model = CalibratedClassifierCV(model, method='sigmoid', cv=5)
calibrated_model.fit(X_train, y_train)
well_calibrated_probs = calibrated_model.predict_proba(X_test)