Hyperparamter tuning - DATA 442/610 Fall 2026

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
plt.rcParams['xtick.direction'] = 'in'
plt.rcParams['ytick.direction'] = 'in'
plt.rcParams['axes.grid'] = True
plt.rcParams['axes.axisbelow'] = True
plt.rcParams['figure.dpi'] = 150

Dataset¶

We’re going to use a different dataset this time, which is more interesting than flowers I hope.

https://archive.ics.uci.edu/dataset/17/breast+cancer+wisconsin+diagnostic

base_feature_names = [
    'radius',
    'texture',
    'perimeter',
    'area',
    'smoothness',
    'compactness',
    'concavity',
    'concave_points',
    'symmetry',
    'fractal_dimension',
]

feature_names = []
for i in range(1, 4):
    for feature in base_feature_names:
        feature_names.append(feature + str(i))

feature_names

df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data', 
                 header=None, index_col=0, names=['Diagnosis', *feature_names])

print(f"Loaded the Wisconsin Breast Cancer dataset of size {df.shape}")
df.head()

This will be a bit harder to visualize since we don’t have 30-dimensional brains. First, visualize the features in a 2-D space using PCA

from sklearn.decomposition import PCA

def scatter_benign_malignant(ax, X, y, pca, label='Train', **kwargs):
    z = pca.transform(X)
    mask = y == 1
    ax.scatter(z[mask, 0], z[mask, 1], color='red', label=f'Malignant ({label})', **kwargs)
    ax.scatter(z[~mask, 0], z[~mask, 1], color='blue', label=f'Benign ({label})', **kwargs)

Next, use seaborn to plot the feature correlation matrix

import seaborn as sns

Now we need to get our dataset, that is our X, y variables. Make sure to encode the benign/malignant labels into [0, 1] labels

Finally, we should create and plot three subsets of our dataset:

Training dataset, for model fitting
Validation dataset, to track model fitting
Test dataset, to evaluate our model

from sklearn.model_selection import train_test_split

# Visualize train, validation, and test sets

Define Polynomial Logistic Regression with L1 and L2 regularization¶

from sklearn.preprocessing import StandardScaler, PolynomialFeatures

def cross_entropy(y_true, y_pred, eps=1e-10):
    pass

class LogisticRegression:
    def __init__(self):
        pass

    def fit(self, X_train, y_train, X_val, y_val):
        pass

    def predict_proba(self, X):
        """ Compute the class probability and return, for ROC-AUC """
        pass
    
    def predict(self, X):
        pass

    def score(self, X, y):
        """ Compute accuracy - what fraction do we predict correctly? """
        pass

# Fit model, plot loss curves

# Compare accuracy on training, validation, and testing sets

# Plot the decision boundary in a PCA space using the helper function below
from matplotlib.colors import ListedColormap
from sklearn.decomposition import PCA

def plot_PCA_decision_boundary(ax, X, y, model, d=0.1, colors=['blue', 'red']):
    pca = PCA(n_components=2)
    X_pca = pca.fit_transform(model.scaler_.transform(X))

    x1, x2 = np.meshgrid(
        np.arange(X_pca[:, 0].min()-10*d, X_pca[:, 0].max()+10*d, d),
        np.arange(X_pca[:, 1].min()-10*d, X_pca[:, 1].max()+10*d, d),
    )
    grid_pca = np.stack([x1.ravel(), x2.ravel()]).T

    grid_raw = model.scaler_.inverse_transform(pca.inverse_transform(grid_pca))
    y_model = model.predict(grid_raw)
    y_model = y_model.reshape(x1.shape)

    pred_mask = model.predict(X) == y

    cmap = ListedColormap(colors)
    im=ax.pcolormesh(x1, x2, y_model, alpha=0.5, cmap=cmap)
    for ii, label in enumerate(np.unique(y)):
        label_mask = y == label
        ax.scatter(X_pca[label_mask & pred_mask, 0], X_pca[label_mask & pred_mask, 1], 
                   facecolor=colors[ii], edgecolor=colors[ii])
        ax.scatter(X_pca[label_mask & ~pred_mask, 0], X_pca[label_mask & ~pred_mask, 1], 
                   facecolor='white', edgecolor=colors[ii])
    ax.set(xlabel='PC 1', ylabel='PC 2')
    return ax

ax = plt.figure().gca()
plot_PCA_decision_boundary(ax, X_test, y_test, model)

# Compute and plot a ROC curve
from sklearn.metrics import roc_curve, auc

y_prob = model.predict_proba(X_test)
fpr, tpr, _ = roc_curve(y_test, y_prob, pos_label=1)
roc_auc = auc(fpr, tpr)

ax = plt.figure().gca()
ax.plot(fpr, tpr, color='black', label=f'ROC-AUC = {roc_auc:.3f}')
ax.plot([0,1], [0,1], color='grey', linestyle='--') # Chance level
ax.set(xlabel='FPR', ylabel='TPR')
ax.legend();

K-fold cross validation¶

Before, we only used one validation set to estimate performance during training. But this is a partial snapshot. The idea behind cross-validation is you create several distinct train/val splits so that you have a performance estimate over the entire training set.

Using cross-validation, you can more accurately assess and tune model hyperparameters before applying the model to test data

from sklearn.model_selection import KFold, StratifiedKFold

kfold = KFold(n_splits=5, shuffle=True, random_state=42)

Hyperparameter tuning with cross validation¶

We want to get the training parameters lr, l1_weight that will maximize our generalization accuracy. To do this, we can do a grid search and track the cross-validation accuracy for each parameter combination

from itertools import product

l1_grid = np.geomspace(1e-4, 1, 5)
lr_grid = np.geomspace(1e-3, 1, 5)

tracking = pd.DataFrame(columns=['l1_weight', 'lr', 'fold', 'score'])
model = LogisticRegression(degree=1)

Nonlinear optimization and kernel methods¶

We have seen with Polynomial Logistic Regression that the use of nonlinear features can allow us to represent more complex functions and decision boundaries. Here, we will compare linear ridge (L2) regression with nonlinear features $\phi(x)$ to kernel ridge regression (KRR). KRR represents the inner product $\langle \phi(x), \phi(y) \rangle$ via a kernel function $k(x, y)$ .

At first this may seem uninteresting. What is the point of using a function to represent a dot product? Let’s consider 2-dimensional inputs $\mathbf{x} = (x_1, x_2)$ and $\mathbf{y} = (y_1, y_2)$ .

Some kernel functions are uninteresting, like the linear kernel: $k_1(\mathbf{x}, \mathbf{y}) = \mathbf{x} \cdot \mathbf{y} = x_1 \,y_1 + x_2 \,y_2$

Some are slightly more interesting, like the quadratic kernel:

\begin{align} k_2(\mathbf{x}, \mathbf{y}) &= (\mathbf{x} \cdot \mathbf{y})^2 \\ &= (x_1 \, y_1 + x_2 \, y_2)^2 \\ &= x_1^2 \, y_1^2 + x_2^2 \, y_2^2 + 2 x_1 \, x_2 \, y_1 \, y_2 \\ &= (x_1^2,\, x_2^2,\, x_1\, x_2 \sqrt{2}) \cdot (y_1^2,\, y_2^2,\, y_1\, y_2 \sqrt{2}) \end{align}

Here, the quadratic kernel allows us to compute a dot product between two 3-dimensional feature vectors, but we did not have to actually compute either feature vector!

One of the most interesting kernels is the RBF kernel:

\begin{align} k_{\text{RBF}}(\mathbf{x}, \mathbf{y}) = \exp \left(-\frac12 \Vert \mathbf{x} - \mathbf{y} \Vert^2 \right) \end{align}

It turns out that this is equivalent to the product of two infinite-dimensional feature vectors (read more here). The RBF kernel is particularly convenient because we do not have the computational resources to compute an infinite-size feature vector for each element of our dataset. But computing the kernel is very easy, and can allow us to represent a variety of complex behaviors.

from sklearn.datasets import make_moons

def make_xor_dataset(N=256, random_seed=42):
    np.random.seed(random_seed)
    X = 2 * np.random.uniform(size=(N, 2)) - 1
    y = np.logical_xor(X[:, 0] > 0, X[:, 1] > 0)
    y = np.where(y, 1, 0)
    return X, y

def make_sin_dataset(N=1024, random_seed=42):
    np.random.seed(random_seed)
    X = 2 * np.random.uniform(size=(N, 2)) - 1
    y = np.sin(2 * np.pi * X[:, 0]) * np.cos(2 * np.pi * X[:, 1])
    y = np.where(y > 0, 1, 0)
    return X, y

nl_datasets = {
    'xor': make_xor_dataset(N=256),
    'sin': make_sin_dataset(N=256),
    'moon': make_moons(n_samples=256)
}

# Plot each of these datasets to see nonlinear character
fig, ax = plt.subplots(1, 3)

for ii, (label, (X_nl, y_nl)) in enumerate(nl_datasets.items()):
    ax[ii].scatter(X_nl[:, 0], X_nl[:, 1], c=y_nl, cmap='bwr')
    ax[ii].set(xlabel='$x_0$', ylabel='$x_1$', aspect='equal')
    ax[ii].set_title(label)

plt.tight_layout()

from matplotlib.colors import ListedColormap
def plot_decision_boundary(ax, X, y, model, d=0.05, colors=['red', 'blue'], **kwargs):
    x1, x2 = np.meshgrid(
        np.arange(X[:, 0].min() - 10*d, X[:, 0].max() + 10*d, d),
        np.arange(X[:, 1].min() - 10*d, X[:, 1].max() + 10*d, d),
    )
    y_grid = model.predict(np.stack([x1.ravel(), x2.ravel()]).T) # [N, 2]
    y_grid = y_grid.reshape(x1.shape)

    cmap = ListedColormap(colors)
    im = ax.pcolormesh(x1, x2, y_grid, cmap=cmap, alpha=0.5)

    for ii, label in enumerate(np.unique(y)):
        mask = y == label
        ax.scatter(X[mask, 0], X[mask, 1], color=colors[int(label)], **kwargs)

Feature Ridge Regression¶

We are going to do linear ridge regression on a set of nonlinear features. Specifically, we would like to optimize

\begin{align} \mathcal{L} = \sum_{i=1}^N (y_i - \hat{y_i})^2 + \lambda \vec{w} \cdot \vec{w} \end{align}

where

\begin{align} y(x) &= \vec{w} \cdot \vec{\phi}(x) \\ &= \left[ \sum_{i=1}^{N_{\text{train}}} \alpha_i \vec{\phi} (x_i) \right] \cdot \vec{\phi}(x) \end{align}

In the second line, we have rewritten the parameter vector $\vec{w}$ as a weighted sum over training examples. Our goal will be to figure out the sample weights $\alpha_i$ and use them to compute $\vec{w}$

from sklearn.preprocessing import PolynomialFeatures

class FeatureRidgeRegression:
    def __init__(self, feature_transform, ridge_weight=0.1):
        pass

    def fit(self, X, y):
        pass

    def predict(self, X):
        pass

model = FeatureRidgeRegression(feature_transform=PolynomialFeatures(degree=100))

fig, ax = plt.subplots(1, 3)

for ii, (label, (X_nl, y_nl)) in enumerate(nl_datasets.items()):
    model.fit(X_nl, y_nl)
    
    plot_decision_boundary(ax[ii], X_nl, y_nl, model, s=5)    
    ax[ii].set(xlabel='$x_0$', ylabel='$x_1$', aspect='equal')
    ax[ii].set_title(label)

plt.tight_layout()

Kernel Ridge Regression¶

In the above example, we had to compute a matrix whose entries contained the dot products between feature vectors from our training set $\vec{\phi}(x_{\text{train}})$ The kernel trick recasts this dot product as the evaluation of a kernel function $k(x_i, x_j) = \vec{\phi}(x_i) \cdot \vec{\phi(x_j)}$ . Using this trick, one can compute a kernel matrix $K \equiv \Phi \Phi^T$ whose entries consist of kernel evaluations

\begin{align} K_{ij} = k(x_i, x_j) = \vec{\phi}(x_i) \cdot \vec{\phi(x_j)} \end{align}

In kernel ridge regression, we no longer care about the feature function and feature vectors. Instead, we care about the kernel matrix and kernel function to perform our optimization.

from sklearn.metrics.pairwise import rbf_kernel

class KernelRidgeRegression:
    def __init__(self, kernel_function, ridge_weight=0.1):
        pass
        
    def fit(self, X, y):
        pass
    
    def predict(self, X):
        pass

model = KernelRidgeRegression(kernel_function=lambda x1,x2: rbf_kernel(x1, x2, gamma=5.))

fig, ax = plt.subplots(1, 3)

for ii, (label, (X_nl, y_nl)) in enumerate(nl_datasets.items()):
    model.fit(X_nl, y_nl)
    
    plot_decision_boundary(ax[ii], X_nl, y_nl, model, s=5)    
    ax[ii].set(xlabel='$x_0$', ylabel='$x_1$', aspect='equal')
    ax[ii].set_title(label)

plt.tight_layout()