187 KiB
187 KiB
In [ ]:
# runnable — linear algebra verification toolkit
import numpy as np
def cosine_similarity(u, v):
u, v = np.asarray(u, dtype=float), np.asarray(v, dtype=float)
return np.dot(u, v) / (np.linalg.norm(u) * np.linalg.norm(v))
def project(a, b):
a, b = np.asarray(a, dtype=float), np.asarray(b, dtype=float)
return (np.dot(a, b) / np.dot(b, b)) * b
def is_orthogonal(u, v, tol=1e-12):
return abs(float(np.dot(u, v))) < tol
A = np.array([[1, 2], [3, 4]])
x = np.array([5, 6])
print("A @ x =", A @ x) # matrix-vector: use @ or np.dot, NEVER *
print("A * I =\n", A @ np.eye(2)) # identity leaves A unchanged
print("norm (3,4) =", np.linalg.norm([3, 4]))
print("cos((3,4),(5,12))=", cosine_similarity([3, 4], [5, 12]), "= 63/65")
print("cos((1,2),(2,4)) =", cosine_similarity([1, 2], [2, 4]), "(parallel -> 1)")
print("proj (2,2)->(1,0)=", project([2, 2], [1, 0]))
print("(5,2) ⟂ (4,-10)? =", is_orthogonal([5, 2], [4, -10]))In [ ]:
# runnable — gradient descent, one step at a time
import numpy as np
import matplotlib.pyplot as plt
def f(x, y):
return x**2 + 2 * y**2
def grad_f(x, y):
return np.array([2 * x, 4 * y])
# ---- one step, by the book -------------------------------------------------
p0 = np.array([1.0, 1.0])
eta = 0.1
g = grad_f(*p0)
p1 = p0 - eta * g
print(f"grad at {tuple(p0)} = {tuple(g)}")
print(f"step: {tuple(p0)} - {eta}*{tuple(g)} = {tuple(np.round(p1, 4))}")
print(f"f before = {f(*p0):.4f} f after = {f(*p1):.4f} -> decreased (downhill)")
# ---- full descent + contour plot ------------------------------------------
path = [p0.copy()]
p = p0.copy()
for _ in range(15):
p = p - eta * grad_f(*p)
path.append(p.copy())
path = np.array(path)
X, Y = np.meshgrid(np.linspace(-2, 2, 400), np.linspace(-2, 2, 400))
plt.figure(figsize=(6, 5))
plt.contour(X, Y, f(X, Y), levels=20)
plt.plot(path[:, 0], path[:, 1], "o-", color="crimson", markersize=4, label="GD path")
plt.xlabel("x"); plt.ylabel("y")
plt.title(r"Gradient descent on $f(x,y)=x^2+2y^2$")
plt.legend(); plt.show()In [ ]:
# runnable — losses computed the exam way (no torch needed)
import numpy as np
from math import log
# --- MSE --------------------------------------------------------------------
x = np.array([1, 2, 3]); y = np.array([2, 4, 6]); w = 1.5
y_hat = w * x
mse = np.mean((y - y_hat) ** 2)
print(f"MSE = {mse:.2f}")
# --- MSE on 0/1 predictions (Exam SS25 2.2) ---------------------------------
labels = np.array([1, 1, 1, 0, 0, 0, 1, 1])
preds = np.array([1, 0, 1, 1, 0, 0, 0, 0])
print(f"MSE (0/1) = {np.mean((labels - preds) ** 2)} = {int(((labels-preds)**2).sum())}/{len(labels)}")
# --- Binary Cross-Entropy (Exam AS25 2.2) -----------------------------------
label_prob = [(1, 0.9), (1, 0.7), (1, 0.4), (0, 0.3),
(0, 0.1), (0, 0.2), (1, 0.8), (0, 0.6)]
terms = [-(log(p) if t == 1 else log(1 - p)) for t, p in label_prob]
for (t, p), term in zip(label_prob, terms):
print(f" y={t} p={p} -> -log({p if t==1 else round(1-p,2)}) = {term:.4f}")
print(f"BCE = {np.mean(terms):.2f}")In [ ]:
# runnable — the Assignment-1 readiness check, condensed
import math
import numpy as np
# f-strings
name, age = "Olivia", 29
print(f"My name is {name}, I am {age} years old.")
# def vs lambda
def square(x):
return x ** 2
add_one = lambda x: x + 1
print(square(5), add_one(10))
# types and length
for var in [3, 3.14, "hello", [1, 2, 3]]:
print(var, type(var))
my_list = [1, 2, 3, 4, 5]
print(my_list, "len =", len(my_list))
# classes and inheritance (the pattern every nn.Module follows)
class Animal:
def __init__(self):
self.legs = 4
def speak(self):
print("The animal makes a sound")
class Dog(Animal): # inherits __init__, so Dog also has .legs
def speak(self): # overrides the parent method
print("The dog barks")
Animal().speak()
d = Dog(); print(d.legs); d.speak()In [ ]:
# runnable
import numpy as np
A = np.array([[1, 2, 3, 4],
[5, 6, 7, 8],
[9, 10, 11, 12]])
print(A)
print("len :", len(A)) # 3 -> first dimension only
print("shape :", A.shape) # (3, 4)
print("mean :", np.mean(A), "| axis=0:", np.mean(A, axis=0))
print("linspace:", np.linspace(-2, 2, 5))
print("meshgrid shapes:", [g.shape for g in np.meshgrid(np.arange(3), np.arange(4))])
print("argmax:", np.argmax([0.1, 0.7, 0.2]), "| where:", np.where(np.array([0, 1, 0]) == 0, -1, 1))
print("permutation:", np.random.default_rng(0).permutation(5))In [ ]:
# runnable — THE SETUP CELL. Run this before anything else.
import os, copy, math, random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import torchvision
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torchvision.transforms import v2
from torch.utils.data import Dataset, DataLoader, TensorDataset, Subset, random_split
from tqdm.auto import tqdm
sns.set_theme(rc={"figure.figsize": (8, 6)}, style="whitegrid")
def set_seed(seed):
"""Full reproducibility across random / numpy / torch / cuda."""
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
set_seed(42)
# Exams add this line to force bit-identical results:
# torch.use_deterministic_algorithms(True)
g = torch.Generator().manual_seed(42) # for DataLoader(generator=g)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("device:", device, "| torch", torch.__version__)In [ ]:
# runnable — shape surgery cheat-run
import torch
x = torch.randn(8, 3, 32, 32) # batch of 8 RGB 32x32 images
print("batch :", tuple(x.shape))
print("flatten for MLP :", tuple(x.view(x.shape[0], -1).shape)) # (8, 3072)
print("one image :", tuple(x[0].shape)) # (3, 32, 32)
print("channels-last (plot) :", tuple(x[0].permute(1, 2, 0).shape)) # (32, 32, 3)
print("add batch dim :", tuple(x[0].unsqueeze(0).shape)) # (1, 3, 32, 32)
gray = torch.randn(1, 28, 28)
print("squeeze grayscale :", tuple(gray.squeeze().shape)) # (28, 28)
# unfold = the ViT patch trick
img = torch.randn(3, 112, 112)
patches = img.unfold(1, 14, 14).unfold(2, 14, 14) # (3, 8, 8, 14, 14)
patches = patches.permute(1, 2, 0, 3, 4).reshape(-1, 3, 14, 14)
print("patches :", tuple(patches.shape), "-> 8*8 = 64 tokens")
logits = torch.randn(4, 10)
print("predicted classes :", logits.argmax(dim=1).tolist())
print("torch.max variant :", torch.max(logits, 1).indices.tolist())In [ ]:
# runnable — logistic regression on Iris + decision boundary
import numpy as np, seaborn as sns, matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
iris = sns.load_dataset("iris")
X = iris[["sepal_length", "petal_length"]].values
y = iris["species"].astype("category").cat.codes.values
y = (y != 0).astype(int) # setosa (0) vs the rest (1)
print("shapes:", X.shape, y.shape)
log_reg = LogisticRegression().fit(X, y)
def wb2mq(w, b):
"""Weights+bias -> slope, intercept of the decision boundary (2D only)."""
assert len(w) == 2, "Only works in 2D"
m = -w[0] / w[1] if w[1] != 0 else float("inf")
q = -b / w[1] if w[1] != 0 else (float("inf") if w[0] != 0 else 0)
return m, q
def params2boundary(w, b, verbose=True):
m, q = wb2mq(w, b)
if verbose:
print(f"m: {m}, q: {q}")
return lambda x: m * x + q
def plot_decision_boundary(w, b, X, y, x1_name, x2_name, title, label=None, ax=None):
boundary = params2boundary(w, b, verbose=False)
x_vals = np.linspace(X[:, 0].min() - 1, X[:, 0].max() + 1, 100)
if ax is None:
plt.figure(figsize=(8, 6)); ax = plt.gca()
sns.scatterplot(x=X[:, 0], y=X[:, 1], hue=y, palette="RdYlBu",
edgecolor="k", s=50, ax=ax)
ax.plot(x_vals, boundary(x_vals), "k--", label=label or "Decision Boundary")
ax.set_xlabel(x1_name); ax.set_ylabel(x2_name); ax.set_title(title)
ax.legend()
return ax
plot_decision_boundary(log_reg.coef_[0], log_reg.intercept_[0], X, y,
"Sepal Length", "Petal Length",
"Logistic Regression Decision Boundary")
plt.show()In [ ]:
# runnable — perceptron activation calculator (exam style)
def weighted_sum(x, w, b):
return sum(wi * xi for wi, xi in zip(w, x)) + b
def relu(z):
return max(0.0, z)
def leaky_relu(z, alpha=0.01):
return z if z > 0 else alpha * z
cases = [
("Midterm 2026 (LeakyReLU)", [2.0, -2.0, 1.0], [1.0, -1.0, 1.0], -2, leaky_relu),
("Exam AS25 2.3 (ReLU)", [1.5, -2.0, 3.0], [0.7, 1.2, -0.8], -0.3, relu),
("Exam SS25 2.3 (ReLU)", [2.0, -1.5, 0.5], [0.8, -0.5, 1.0], 0.2, relu),
]
for name, x, w, b, act in cases:
z = weighted_sum(x, w, b)
terms = " + ".join(f"({wi}*{xi})" for wi, xi in zip(w, x))
print(f"{name}\n z = {terms} + ({b}) = {z:.2f}\n sigma({z:.2f}) = {act(z):.2f}\n")In [ ]:
# runnable — MyPerceptron from scratch, compared to scikit-learn
import numpy as np, seaborn as sns, matplotlib.pyplot as plt
from sklearn.datasets import load_iris
from sklearn.linear_model import Perceptron
class MyPerceptron:
"""Bias is stored LAST in the weight array (exam requirement)."""
def __init__(self, dim, learning_rate=0.01, epochs=100):
self.weights = np.zeros(dim + 1) # dim weights + 1 bias
self.learning_rate = learning_rate
self.epochs = epochs
def predict(self, x):
z = np.dot(x, self.weights[:-1]) + self.weights[-1]
return 1 if z >= 0 else 0
def train(self, X, y):
for _ in range(self.epochs):
for i in range(X.shape[0]):
prediction = self.predict(X[i])
delta = self.learning_rate * (y[i] - prediction)
self.weights[:-1] += delta * X[i]
self.weights[-1] += delta
def score(self, X, y):
predictions = np.array([self.predict(x) for x in X])
return np.mean(predictions == y)
iris = load_iris()
X = iris.data[:, [0, 2]] # sepal length, petal length
y = (iris.target != 0).astype(int) # setosa vs rest
mine = MyPerceptron(dim=2, learning_rate=0.1, epochs=100)
mine.train(X, y)
print("Trained Weights:", mine.weights)
print(f"Accuracy: {mine.score(X, y) * 100:.2f}%")
skl = Perceptron().fit(X, y)
*my_w, my_b = mine.weights
w_skl, b_skl = skl.coef_[0], skl.intercept_[0]
x_vals = np.array([X[:, 0].min(), X[:, 0].max()])
sns.scatterplot(x=X[:, 0], y=X[:, 1], hue=y, palette="coolwarm", edgecolor="k")
plt.plot(x_vals, (-my_w[0] / my_w[1]) * x_vals - my_b / my_w[1],
"k--", label="Implemented Perceptron")
plt.plot(x_vals, (-w_skl[0] / w_skl[1]) * x_vals - b_skl / w_skl[1],
"--", color="darkgreen", label="SKL Perceptron")
plt.xlabel("Sepal Length"); plt.ylabel("Petal Length")
plt.title("Perceptron Decision Boundary (Setosa vs Rest)")
plt.legend(); plt.show()In [ ]:
# runnable — learning rate & initialisation effects
import numpy as np, seaborn as sns, matplotlib.pyplot as plt
from sklearn.linear_model import Perceptron
from sklearn.datasets import make_circles
# Non-linearly separable data: the perceptron cannot solve it
X, y = make_circles(n_samples=100, noise=0.1, factor=0.4, random_state=42)
sns.scatterplot(x=X[:, 0], y=X[:, 1], hue=y, palette="coolwarm", edgecolor="k")
for state in range(3):
p = Perceptron(random_state=state).fit(X, y)
w, b = p.coef_[0], p.intercept_[0]
xs = np.linspace(X[:, 0].min() - 1, X[:, 0].max() + 1, 100)
plt.plot(xs, (-w[0] / w[1]) * xs - b / w[1], "--", label=f"Random State {state}")
plt.title("Perceptron Decision Boundaries for Different Random States")
plt.legend(); plt.show()In [ ]:
# runnable — hill climbing and random restarts
import numpy as np, matplotlib.pyplot as plt
from scipy import optimize
# --- naive hill climbing (maximisation) -------------------------------------
def f_hill(x):
return np.sin(6 * x) + x**2 * np.cos(x**2)
rng = np.random.default_rng(0)
x = rng.uniform(-1.75, 1.75)
steps = [x]
for _ in range(15):
new_x = x + rng.uniform(-0.3, 0.3)
if f_hill(new_x) > f_hill(x): # accept only improvements
x = new_x
steps.append(x)
xs = np.linspace(-2, 2, 400)
plt.plot(xs, f_hill(xs), label="f(x)")
plt.plot(steps, [f_hill(s) for s in steps], "ro-", label="hill-climbing steps")
plt.title("Hill climbing accepts only uphill moves"); plt.legend(); plt.show()
# --- scipy downhill simplex on a multi-modal function -----------------------
def f(x):
return (x % 521) * np.sin(x / 47) * np.exp(-0.002 * x)
x_vals = np.linspace(-1000, 0, 1001)
y_vals = f(x_vals)
for start, kwargs in [(-225, {}), (-550, {}), (-775, {"maxiter": 2})]:
m = optimize.fmin(f, x0=start, disp=False, full_output=True, **kwargs)
print(f"start {start:>5} -> x={m[0][0]:8.2f} f={m[1]:8.2f}")
# --- how often do random restarts find the global minimum? ------------------
global_min = int(min(y_vals))
n_trials, found = 100, 0
for _ in range(n_trials):
m = optimize.fmin(f, x0=np.random.randint(-1000, 0), disp=False, full_output=True)
if int(m[1]) == global_min:
found += 1
print(f"After {n_trials} random initializations, the algorithm found the global minimum {found} times")In [ ]:
# runnable — K-Means: elbow, silhouette, PCA visualisation
import numpy as np, matplotlib.pyplot as plt
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.metrics import silhouette_score
from sklearn.datasets import make_blobs
np.random.seed(42)
# In the exam you load it: data = pd.read_csv('blobs_data.csv'); X = data.values
X, _ = make_blobs(n_samples=200, n_features=4, centers=4, random_state=42)
# 1) Elbow method (k = 1..10)
inertia = []
for k in range(1, 11):
inertia.append(KMeans(n_clusters=k, random_state=42, n_init=10).fit(X).inertia_)
plt.figure(figsize=(8, 5))
plt.plot(range(1, 11), inertia, marker="o")
plt.title("Elbow Method: Inertia vs. Number of Clusters")
plt.xlabel("Number of Clusters"); plt.ylabel("Inertia (Sum of Squared Distances)")
plt.show()
# 2) Silhouette score (k = 2..10)
sil_scores = []
for k in range(2, 11):
km = KMeans(n_clusters=k, random_state=42, n_init=10).fit(X)
sil_scores.append(silhouette_score(X, km.labels_))
plt.figure(figsize=(8, 5))
plt.plot(range(2, 11), sil_scores, marker="o", color="orange")
plt.title("Silhouette Score vs. Number of Clusters")
plt.xlabel("Number of Clusters"); plt.ylabel("Silhouette Score")
plt.show()
# 3) Final clustering, visualised in 2D via PCA
optimal_k = int(np.argmax(sil_scores)) + 2
print("optimal k =", optimal_k)
kmeans = KMeans(n_clusters=optimal_k, random_state=42, n_init=10).fit(X)
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
centroids_pca = pca.transform(kmeans.cluster_centers_)
plt.scatter(X_pca[:, 0], X_pca[:, 1], c=kmeans.labels_, cmap="viridis", s=50, alpha=0.5)
plt.scatter(centroids_pca[:, 0], centroids_pca[:, 1], c="darkred", s=200,
marker="X", label="Centroids")
plt.title("K-Means Clustering with PCA (Centroids in Red)")
plt.xlabel("PCA Component 1"); plt.ylabel("PCA Component 2")
plt.legend(); plt.show()In [ ]:
# runnable — traverse the mock-exam decision tree programmatically
def classify(x1, x2, x3):
if x1 == 1: # Node 2
if x2 == 1:
return "A"
return "A" if x3 == 1 else "B" # Node 3
if x1 == 2: # Node 4
if x3 == 1:
return "B"
if x3 == 2:
return "A"
return "C"
return "B" # Else at the root
for i, inp in enumerate([(1, 2, 1), (2, 1, 0), (3, 1, 2), (1, 1, 3), (2, 2, 2)], start=1):
print(f"Input {i}: X1={inp[0]}, X2={inp[1]}, X3={inp[2]} -> Class {classify(*inp)}")In [ ]:
# reference — the Titanic preprocessing block (needs 05_titanic_clean.csv)
import pandas as pd, numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
t_df = pd.read_csv("05_titanic_clean.csv")
t_df.drop(["Cabin", "Name", "PassengerId", "Embarked", "Ticket"], axis=1, inplace=True)
display(t_df.describe())
# categorical -> one-hot (each category gets its own binary column)
t_df = pd.get_dummies(t_df, columns=["Sex"], prefix=["Sex"])
t_df = pd.get_dummies(t_df, columns=["Title"])
# numerical -> mean 0, std 1
scaler = StandardScaler()
t_df[["Age", "Fare"]] = scaler.fit_transform(t_df[["Age", "Fare"]])
# features / target; float32 is the standard dtype for PyTorch inputs
X = t_df.drop(["Survived"], axis=1).astype(np.float32).values
y = t_df["Survived"].values
print("input_dim must be", X.shape[1])In [ ]:
# runnable — the four dataset flavours (only the first two run offline)
import torch
from torch.utils.data import TensorDataset, DataLoader, Dataset, Subset
# 1) TensorDataset --------------------------------------------------------
X_train = torch.randn(100, 14); y_train = torch.randint(0, 2, (100,))
X_val = torch.randn(40, 14); y_val = torch.randint(0, 2, (40,))
train_dataset = TensorDataset(X_train, y_train)
val_dataset = TensorDataset(X_val, y_val) # ⚠ NOT (X_train, y_train)!
batch_size = 32
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=batch_size, shuffle=False)
print("batches:", len(train_loader), "| samples:", len(train_loader.dataset))
# 2) Custom Dataset -------------------------------------------------------
class TextDataset(Dataset):
def __init__(self, sequences):
self.sequences = sequences
def __len__(self):
return len(self.sequences)
def __getitem__(self, idx):
return self.sequences[idx]
# 3) Built-in benchmark # needs download
# transform = transforms.Compose([transforms.ToTensor(),
# transforms.Normalize((0.5,), (0.5,))]) # -> [-1, 1]
# train_data = datasets.FashionMNIST(root='./data', train=True, transform=transform, download=True)
# test_data = datasets.FashionMNIST(root='./data', train=False, transform=transform, download=True)
# print(len(train_data), train_data.classes)
# 4) ImageFolder # needs an unzipped folder DATA_PATH/train/{cats,dogs}
# train_dataset = datasets.ImageFolder(os.path.join(DATA_PATH, 'train'))In [ ]:
# reference — RAMDatasetWrapper: cache a whole dataset in memory
from tqdm.auto import tqdm
import torch, PIL
class RAMDatasetWrapper(torch.utils.data.Dataset):
"""Cache an entire dataset in RAM, then apply transforms on access.
Reading many small files from disk becomes the bottleneck when you iterate
over the same images for many epochs. Loading once at the start removes it.
⚠️ Only practical when the dataset fits in memory — otherwise it will
exhaust RAM and crash the session.
"""
def __init__(self, dataset, transform=None):
self.data = [sample for sample in tqdm(dataset)]
self.n = len(self.data)
self.transform = transform
def __getitem__(self, ind):
if self.transform is not None and isinstance(self.data[ind][0], PIL.Image.Image):
return self.transform(self.data[ind][0]), self.data[ind][1]
return self.data[ind]
def set_transform(self, transform):
self.transform = transform
def __len__(self):
return self.nIn [ ]:
# runnable — reproducing the leakage bug and its fix
import torch
from torch.utils.data import TensorDataset
n_train = n_val = 100
X_train = torch.randn(n_train, 1, 28, 28); y_train = torch.randint(0, 10, (n_train,))
X_val = torch.randn(n_val, 1, 28, 28); y_val = torch.randint(0, 10, (n_val,))
leaky = TensorDataset(X_train, y_train) # ❌ what the exam gives you
fixed = TensorDataset(X_val, y_val) # ✅ data leakage fixed!
print("chance accuracy for 10 classes: ~10%")
print("leaky val set is the train set:", torch.equal(leaky.tensors[0], X_train))
print("fixed val set is independent :", not torch.equal(fixed.tensors[0], X_train))In [ ]:
# runnable — the three model-definition styles, all equivalent
import torch, torch.nn as nn, torch.nn.functional as F
# (a) bare nn.Sequential — quickest, used in Assignment 4
model_a = nn.Sequential(
nn.Linear(28 * 28, 128),
nn.ReLU(),
nn.Linear(128, 10),
)
# (b) nn.Module wrapping one Sequential — the exam's preferred style
class MLP(nn.Module):
def __init__(self):
super().__init__()
self.model = nn.Sequential(
nn.Flatten(), # (B,1,28,28) -> (B,784), no manual .view needed
nn.Linear(28 * 28, 128), nn.ReLU(),
nn.Linear(128, 64), nn.ReLU(),
nn.Linear(64, 10),
)
def forward(self, x):
return self.model(x)
# (c) explicit layers + functional forward — when you need branching
class MLPNoSeq(nn.Module):
def __init__(self, input_dim, hidden_dim, output_dim):
super().__init__()
self.fc1 = nn.Linear(input_dim, hidden_dim)
self.fc2 = nn.Linear(hidden_dim, hidden_dim)
self.out = nn.Linear(hidden_dim, output_dim)
def forward(self, x):
x = F.relu(self.fc1(x))
x = F.relu(self.fc2(x))
return self.out(x)
model = MLP()
print(model)
print("\nforward on a batch:", model(torch.randn(4, 1, 28, 28)).shape)In [ ]:
# runnable — the activation zoo, plotted
import torch, torch.nn as nn, matplotlib.pyplot as plt
z = torch.linspace(-4, 4, 400)
acts = {
"ReLU": nn.ReLU(), "LeakyReLU(0.1)": nn.LeakyReLU(0.1), "GELU": nn.GELU(),
"Sigmoid": nn.Sigmoid(), "Tanh": nn.Tanh(),
}
plt.figure(figsize=(9, 5))
for name, fn in acts.items():
plt.plot(z, fn(z), label=name)
plt.axhline(0, color="k", lw=0.5); plt.axvline(0, color="k", lw=0.5)
plt.legend(); plt.title("Activation functions"); plt.show()
print("softmax([2,1,0.1]) =", torch.softmax(torch.tensor([2.0, 1.0, 0.1]), dim=0))In [ ]:
# reference — memorise these four. They are given in every exam.
import numpy as np, torch, matplotlib.pyplot as plt
from tqdm.auto import tqdm
def train_epoch(model, train_dataloader, optimizer, loss_fn):
losses, correct_predictions = [], 0
for features, labels in tqdm(train_dataloader):
features, labels = features.to(device), labels.to(device)
output = model(features) # 1. forward
optimizer.zero_grad() # 2. clear old gradients
loss = loss_fn(output, labels) # 3. loss
loss.backward() # 4. backprop
optimizer.step() # 5. update weights
losses.append(loss.item())
correct_predictions += (output.argmax(dim=1) == labels).sum().item()
accuracy = 100.0 * correct_predictions / len(train_dataloader.dataset)
return np.array(losses).mean(), accuracy
def evaluate(model, dataloader, loss_fn):
losses, correct_predictions = [], 0
with torch.no_grad(): # no gradients -> less memory, faster
for features, labels in dataloader:
features, labels = features.to(device), labels.to(device)
output = model(features)
loss = loss_fn(output, labels)
correct_predictions += (output.argmax(dim=1) == labels).sum().item()
losses.append(loss.item())
accuracy = 100.0 * correct_predictions / len(dataloader.dataset)
return np.array(losses).mean(), accuracy
def train(model, train_dataloader, val_dataloader, optimizer, n_epochs, loss_fn):
train_losses, val_losses, train_accuracies, val_accuracies = [], [], [], []
for epoch in range(n_epochs):
model.train() # enables Dropout / BatchNorm training mode
train_loss, train_accuracy = train_epoch(model, train_dataloader, optimizer, loss_fn)
model.eval() # disables them for a deterministic pass
val_loss, val_accuracy = evaluate(model, val_dataloader, loss_fn)
train_losses.append(train_loss); val_losses.append(val_loss)
train_accuracies.append(train_accuracy); val_accuracies.append(val_accuracy)
print('Epoch {}/{}: train_loss: {:.4f}, train_accuracy: {:.4f}, '
'val_loss: {:.4f}, val_accuracy: {:.4f}'.format(
epoch + 1, n_epochs, train_losses[-1], train_accuracies[-1],
val_losses[-1], val_accuracies[-1]))
return train_losses, val_losses, train_accuracies, val_accuracies
def plot(train_losses, val_losses, train_accuracies, val_accuracies, title):
plt.figure()
plt.plot(np.arange(len(train_losses)), train_losses)
plt.plot(np.arange(len(val_losses)), val_losses)
plt.legend(['train_loss', 'val_loss'])
plt.xlabel('epoch'); plt.ylabel('loss value')
plt.xticks(np.arange(len(train_losses)), np.arange(1, len(train_losses) + 1))
plt.title('{}: Train/val loss'.format(title))
plt.figure()
plt.plot(np.arange(len(train_accuracies)), train_accuracies)
plt.plot(np.arange(len(val_accuracies)), val_accuracies)
plt.legend(['train_acc', 'val_acc'])
plt.xlabel('epoch'); plt.ylabel('accuracy')
plt.xticks(np.arange(len(train_losses)), np.arange(1, len(train_losses) + 1))
plt.title('{}: Train/val accuracy'.format(title))In [ ]:
# runnable — a complete miniature training run on synthetic data
import torch, torch.nn as nn, torch.optim as optim, numpy as np
from torch.utils.data import TensorDataset, DataLoader
set_seed(42)
X_train, y_train = torch.randn(512, 20), torch.randint(0, 3, (512,))
X_val, y_val = torch.randn(128, 20), torch.randint(0, 3, (128,))
train_loader = DataLoader(TensorDataset(X_train, y_train), batch_size=32, shuffle=True)
val_loader = DataLoader(TensorDataset(X_val, y_val), batch_size=32, shuffle=False)
model = nn.Sequential(nn.Linear(20, 32), nn.ReLU(), nn.Linear(32, 3)).to(device)
# --- the four lines that define a training setup ---------------------------
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=0.001)
n_epochs = 5
hist = train(model, train_loader, val_loader, optimizer, n_epochs, criterion)
plot(*hist, title="MLP")
val_loss, val_accuracy = evaluate(model, val_loader, criterion)
print('MLP. Validation loss: {:.2f}, validation accuracy: {:.2f}'.format(val_loss, val_accuracy))In [ ]:
# runnable — one MLP class with switchable regularisation
import torch, torch.nn as nn, torch.optim as optim
class MLP(nn.Module):
"""3 hidden layers, each optionally BatchNorm / LayerNorm / Dropout."""
def __init__(self, input_dim, hidden_dim,
use_batchnorm=False, use_layernorm=False, use_dropout=False):
super().__init__()
def block(in_dim, out_dim):
return [
nn.Linear(in_dim, out_dim),
nn.BatchNorm1d(out_dim) if use_batchnorm else nn.Identity(),
nn.LayerNorm(out_dim) if use_layernorm else nn.Identity(),
nn.ReLU(),
nn.Dropout(0.5) if use_dropout else nn.Identity(),
]
self.model = nn.Sequential(
*block(input_dim, hidden_dim),
*block(hidden_dim, hidden_dim),
*block(hidden_dim, hidden_dim),
nn.Linear(hidden_dim, 2), # binary classification -> 2 logits
)
def forward(self, x):
return self.model(x)
for flags in [{}, {"use_batchnorm": True}, {"use_layernorm": True}, {"use_dropout": True}]:
m = MLP(14, 32, **flags)
name = list(flags)[0].replace("use_", "") if flags else "none"
print(f"{name:>10}: output {tuple(m(torch.randn(8, 14)).shape)}")
# weight decay = L2 regularisation, set on the optimizer, not the model
model = MLP(14, 32)
optimizer_l2 = optim.Adam(model.parameters(), lr=0.001, weight_decay=0.0001)
print("\nAdam with L2:", optimizer_l2.param_groups[0]["weight_decay"])In [ ]:
# runnable — the comparison table and overlaid loss curves (synthetic numbers)
import numpy as np, matplotlib.pyplot as plt
val_losses = list(np.linspace(0.55, 0.72, 30)) # no regularisation: goes UP
val_losses_bn = list(np.linspace(0.58, 0.47, 30))
val_losses_ln = list(np.linspace(0.57, 0.55, 30))
val_losses_drop = list(np.linspace(0.60, 0.45, 30))
val_acc, val_acc_bn, val_acc_ln, val_acc_drop = [78.2], [81.5], [79.3], [82.1]
table_data = [
["Regularization type", "Val Loss", "Val accuracy"],
["No regularization", val_losses[-1], val_acc[-1]],
["Batch Norm", val_losses_bn[-1], val_acc_bn[-1]],
["Layer Norm", val_losses_ln[-1], val_acc_ln[-1]],
["Dropout", val_losses_drop[-1], val_acc_drop[-1]],
]
print("{: >20}| {: >20}| {: >20}".format(*table_data[0]))
print("-".join("" for _ in range(65)))
for row in table_data[1:]:
print("{: >20}| {:20.4f}| {: >20}".format(*row))
loss_curves = [val_losses, val_losses_bn, val_losses_ln, val_losses_drop]
reg_types = ["no regularization", "batch norm", "layer norm", "dropout"]
plt.figure()
for curve in loss_curves:
plt.plot(np.arange(len(curve)), curve)
plt.title("Validation loss curves for different regularization")
plt.legend(reg_types); plt.show()In [ ]:
# reference — EarlyStopping + the full train_model / test_model / plot_training_history suite
import copy, numpy as np, torch, matplotlib.pyplot as plt, seaborn as sns
from tqdm.auto import tqdm
from sklearn.metrics import confusion_matrix, classification_report
class EarlyStopping:
"""Stops training when validation loss has not improved for `patience` epochs."""
def __init__(self, patience=10, min_delta=0.0):
self.patience = patience
self.min_delta = min_delta
self.best_score = None
self.counter = 0
self.should_stop = False
def step(self, current_score):
if self.best_score is None: # first epoch always "improves"
self.best_score, self.counter, self.should_stop = current_score, 0, False
return True
if current_score < self.best_score - self.min_delta: # for loss, LOWER is better
self.best_score, self.counter, self.should_stop = current_score, 0, False
return True
self.counter += 1
if self.counter >= self.patience:
self.should_stop = True
return False
def train_model(model, train_loader, val_loader, criterion, optimizer,
num_epochs=50, name="model", patience=10,
use_early_stopping=False, verbose=True, min_delta=0.001):
"""Train with validation monitoring, LR scheduling, early stopping and
automatic restoration of the best weights.
Returns
-------
model : the model with the BEST validation weights restored
history : dict with train_loss / train_acc / val_loss / val_acc / lr
"""
if len(train_loader) == 0 or len(val_loader) == 0:
raise ValueError("Empty DataLoader.")
best_val_loss = float("inf")
best_model_wts = copy.deepcopy(model.state_dict())
early_stopper = EarlyStopping(patience=patience, min_delta=min_delta)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, mode="min", factor=0.5, patience=4)
history = {"train_loss": [], "train_acc": [], "val_loss": [], "val_acc": [], "lr": []}
for epoch in tqdm(range(num_epochs), desc="Training Progress", unit="epoch",
disable=not verbose):
# ---- train ---------------------------------------------------------
model.train()
running_loss = correct_train = total_train = 0
for images, labels in train_loader:
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
outputs = model(images)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
bs = labels.size(0)
running_loss += loss.item() * bs # weighted: batch sizes may vary
total_train += bs
correct_train += (outputs.argmax(1) == labels).sum().item()
train_loss = running_loss / total_train
train_acc = 100.0 * correct_train / total_train
# ---- validate ------------------------------------------------------
model.eval()
val_running_loss = correct_val = total_val = 0
with torch.no_grad():
for images, labels in val_loader:
images, labels = images.to(device), labels.to(device)
outputs = model(images)
loss = criterion(outputs, labels)
bs = labels.size(0)
val_running_loss += loss.item() * bs
total_val += bs
correct_val += (outputs.argmax(1) == labels).sum().item()
val_loss = val_running_loss / total_val
val_acc = 100.0 * correct_val / total_val
scheduler.step(val_loss) # LR halves on plateau
current_lr = optimizer.param_groups[0]["lr"]
for k, v in zip(history, [train_loss, train_acc, val_loss, val_acc, current_lr]):
history[k].append(v)
if verbose:
print(f"\nEpoch [{epoch+1}/{num_epochs}] | LR: {current_lr:.6f}")
print(f"Train Loss: {train_loss:.4f}, Train Accuracy: {train_acc:.2f}%")
print(f"Val Loss: {val_loss:.4f}, Val Accuracy: {val_acc:.2f}%")
if val_loss < best_val_loss: # ---- checkpoint -------
best_val_loss = val_loss
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(model.state_dict(), f"best_{name}.pth")
if use_early_stopping: # ---- early stopping ---
early_stopper.step(val_loss)
if early_stopper.should_stop:
if verbose:
print(f" Early stopping triggered after epoch {epoch+1}.")
break
model.load_state_dict(best_model_wts) # restore the BEST weights
return model, history
def plot_training_history(history, plot_lr=False):
epochs = range(1, len(history["train_loss"]) + 1)
for keys, ylabel, title in [
(("train_loss", "val_loss"), "Loss", "Training vs Validation Loss"),
(("train_acc", "val_acc"), "Accuracy (%)", "Training vs Validation Accuracy"),
]:
plt.figure(figsize=(8, 5))
for k in keys:
plt.plot(epochs, history[k], label=k)
plt.title(title); plt.xlabel("Epoch"); plt.ylabel(ylabel)
plt.legend(); plt.grid(True); plt.tight_layout(); plt.show()
if plot_lr:
plt.figure(figsize=(8, 5))
plt.plot(epochs, history["lr"], label="Learning Rate")
plt.title("Learning Rate Over Epochs"); plt.xlabel("Epoch")
plt.ylabel("Learning Rate"); plt.legend(); plt.grid(True); plt.show()In [ ]:
# runnable — shape calculators + automatic verification
import math, torch, torch.nn as nn
def conv_out(width, kernel_size, stride, padding, verbose=False):
out = math.floor((width + 2 * padding - kernel_size) / stride) + 1
if verbose:
print(out)
return out
def maxpool_out(width, kernel_size, stride=None, verbose=False):
if stride is None:
stride = kernel_size # PyTorch default
out = math.floor((width - kernel_size) / stride) + 1
if verbose:
print(out)
return out
x = conv_out(28, kernel_size=3, stride=1, padding=0, verbose=True) # 26
x = maxpool_out(x, kernel_size=2, verbose=True) # 13
x = conv_out(x, kernel_size=3, stride=1, padding=1, verbose=True) # 13
x = maxpool_out(x, kernel_size=3, stride=2, verbose=True) # 6
print("flatten:", 16 * 6 * 6) # 576
# --- always double-check by running a dummy tensor through the layers -------
net = nn.Sequential(
nn.Conv2d(1, 8, kernel_size=3, stride=1), nn.ReLU(), nn.MaxPool2d(kernel_size=2),
nn.Conv2d(8, 16, kernel_size=3, stride=1, padding=1), nn.ReLU(),
nn.MaxPool2d(kernel_size=3, stride=2),
)
t = torch.randn(1, 1, 28, 28)
for layer in net:
t = layer(t)
print(f"{layer.__class__.__name__:<12} -> {tuple(t.shape)}")
print("flattened in_features =", t.numel())In [ ]:
# runnable — verify the by-hand convolution with F.conv2d
import torch, torch.nn.functional as F
X = torch.tensor([[1., 1., 1., 1.],
[4., 4., 4., 4.],
[1., 1., 1., 1.],
[1., 1., 1., 1.]]).view(1, 1, 4, 4)
W = torch.empty(2, 1, 3, 3)
W[0, 0] = torch.tensor([[-1., -1., -1.], [2., 2., 2.], [-1., -1., -1.]]) # horizontal
W[1, 0] = torch.tensor([[-1., 2., -1.], [-1., 2., -1.], [-1., 2., -1.]]) # vertical
out = F.conv2d(X, W) # no padding, stride 1
print("conv output shape:", tuple(out.shape), "-> 2 x 2 x 2 volume")
print("f1 =\n", out[0, 0])
print("f2 =\n", out[0, 1])
pooled = F.max_pool2d(out, kernel_size=2)
print("after 2x2 max pool:", tuple(pooled.shape), "->", pooled.flatten().tolist())In [ ]:
# runnable — the five classic kernels on a synthetic digit-like image
import torch, torch.nn.functional as F, matplotlib.pyplot as plt
# A synthetic "digit": a bright cross on a dark field (replace with mnist_train.data[12])
img = torch.zeros(28, 28)
img[10:18, 4:24] = 200.0
img[4:24, 12:16] = 255.0
x = img.view(1, 1, 28, 28)
weight = torch.empty(5, 1, 3, 3)
weight[0, 0] = torch.tensor([[0., 0., 0.], [0., 1., 0.], [0., 0., 0.]]) # identity
weight[1, 0] = torch.tensor([[1., 1., 1.], [1., 1., 1.], [1., 1., 1.]]) # box blur
weight[2, 0] = torch.tensor([[-1., 0., 1.], [-1., 0., 1.], [-1., 0., 1.]]) # vertical edge
weight[3, 0] = torch.tensor([[-1., -1., -1.], [0., 0., 0.], [1., 1., 1.]]) # horizontal edge
weight[4, 0] = torch.tensor([[0., -1., 0.], [-1., 4., -1.], [0., -1., 0.]]) # laplacian
y = F.conv2d(x, weight)
fig, axes = plt.subplots(2, 3, figsize=(10, 6))
axes = axes.flatten()
axes[0].imshow(x[0, 0].numpy(), cmap="gray"); axes[0].set_title("Original Image"); axes[0].axis("off")
for i, name in enumerate(["Identity", "Box Blur", "Vertical Edge", "Horizontal Edge", "Sharpening"]):
axes[i + 1].imshow(y[0, i].detach().numpy(), cmap="gray")
axes[i + 1].set_title(name); axes[i + 1].axis("off")
plt.tight_layout(); plt.show()In [ ]:
# runnable — the three CNNs from Assignment 6, smallest to largest
import torch, torch.nn as nn
class SimpleCNN(nn.Module):
"""Lecture baseline (Softmax REMOVED - CrossEntropyLoss wants logits)."""
def __init__(self, num_channels=1, num_classes=10):
super().__init__()
self.model = nn.Sequential(
nn.Conv2d(num_channels, 32, kernel_size=3, stride=1), nn.ReLU(),
nn.MaxPool2d(kernel_size=2),
nn.Conv2d(32, 32, kernel_size=3, stride=1), nn.ReLU(),
nn.MaxPool2d(kernel_size=2),
nn.Flatten(1),
nn.Dropout(),
nn.LazyLinear(num_classes), # infers in_features on first forward
)
def forward(self, x):
return self.model(x)
class ExpCNN(nn.Module):
"""Simplified: one conv block. Nearly the same accuracy, far fewer params."""
def __init__(self, num_channels=1, num_classes=10):
super().__init__()
self.model = nn.Sequential(
nn.Conv2d(num_channels, 32, kernel_size=3, stride=1), nn.ReLU(),
nn.MaxPool2d(kernel_size=2),
nn.Flatten(1), nn.Dropout(0.3), nn.LazyLinear(num_classes),
)
def forward(self, x):
return self.model(x)
class ExpCNN2(nn.Module):
"""Deeper + BatchNorm + graduated Dropout: >99% on MNIST after 5 epochs."""
def __init__(self, num_channels=1, num_classes=10):
super().__init__()
self.model = nn.Sequential(
nn.Conv2d(num_channels, 32, kernel_size=3, padding=1), nn.BatchNorm2d(32), nn.ReLU(),
nn.Conv2d(32, 32, kernel_size=3, padding=1), nn.BatchNorm2d(32), nn.ReLU(),
nn.MaxPool2d(2), nn.Dropout(0.1),
nn.Conv2d(32, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(),
nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.BatchNorm2d(64), nn.ReLU(),
nn.MaxPool2d(2), nn.Dropout(0.15),
nn.Conv2d(64, 128, kernel_size=3, padding=1), nn.BatchNorm2d(128), nn.ReLU(),
nn.Flatten(), nn.LazyLinear(256), nn.ReLU(), nn.Dropout(0.3),
nn.Linear(256, num_classes),
)
def forward(self, x):
return self.model(x)
dummy = torch.randn(2, 1, 28, 28)
for cls in (SimpleCNN, ExpCNN, ExpCNN2):
m = cls()
out = m(dummy) # first pass materialises LazyLinear
n_params = sum(p.numel() for p in m.parameters())
print(f"{cls.__name__:<10} out {tuple(out.shape)} params {n_params:,}")In [ ]:
# runnable — the exam's features/classifier two-block CNN (Exams AS25 & SS25 3.3)
import torch, torch.nn as nn
class CNNModel(nn.Module):
"""SS25 variant: Conv(1->32,k5,p2) -> BN -> ReLU -> MaxPool(2,2)
Conv(32->64,k3,s2) -> BN -> ReLU -> Flatten
then LazyLinear(128) -> ReLU -> Dropout(0.5) -> Linear(128,10)."""
def __init__(self):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(1, 32, kernel_size=5, padding=2),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size=3, stride=2),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.Flatten(),
)
self.classifier = nn.Sequential(
nn.LazyLinear(128),
nn.ReLU(),
nn.Dropout(p=0.5),
nn.Linear(128, 10),
)
def forward(self, x):
x = self.features(x)
x = self.classifier(x)
return x
model = CNNModel()
print(model(torch.randn(4, 1, 16, 16)).shape, " # USPS is 16x16 -> (batch, 64, 3, 3) -> 576")
print(model)
# ⚠️ Backup model, only if yours does not work — it still earns the training points
class BackupCNN(nn.Module):
def __init__(self):
super().__init__()
self.conv = nn.Conv2d(1, 8, kernel_size=3)
self.relu = nn.ReLU()
self.fc = nn.Linear(8 * 14 * 14, 10)
def forward(self, x):
x = self.conv(x)
x = self.relu(x)
x = x.view(x.size(0), -1)
return self.fc(x)In [ ]:
# runnable — basic vs bad vs good augmentation pipelines
import torch
from torchvision.transforms import v2
transforms_basic = v2.Compose([
v2.ToImage(),
v2.Resize((28, 28)),
v2.ToDtype(torch.float32, scale=True),
v2.Normalize(mean=(0.5,), std=(0.5,)),
])
# ❌ too aggressive for MNIST: flips turn 6 into 9, blur destroys thin strokes
transforms_aug = v2.Compose([
v2.ToImage(),
v2.Resize((28, 28)),
v2.RandomAffine(degrees=20, translate=(0.15, 0.15), scale=(0.9, 1.1), shear=15, fill=0),
v2.RandomVerticalFlip(0.75),
v2.RandomHorizontalFlip(0.75),
v2.GaussianBlur(kernel_size=(5, 9), sigma=(0.5, 1.2)),
v2.ToDtype(torch.float32, scale=True),
v2.Normalize(mean=(0.5,), std=(0.5,)),
])
# ✅ mild and label-preserving: models handwriting variation, nothing more
transforms_aug2 = v2.Compose([
v2.ToImage(),
v2.RandomRotation(degrees=10, fill=0), # writing-style variation
v2.GaussianBlur(kernel_size=(3, 5), sigma=(0.1, 1)), # mild, prevents overfitting
v2.ToDtype(torch.float32, scale=True),
v2.Normalize(mean=(0.5,), std=(0.5,)),
])
img = torch.rand(1, 28, 28)
for name, t in [("basic", transforms_basic), ("aggressive", transforms_aug), ("mild", transforms_aug2)]:
print(f"{name:>11}: {tuple(t(img).shape)} range [{t(img).min():.2f}, {t(img).max():.2f}]")In [ ]:
# reference — plot_transform: show 9 augmented versions next to the original
import torch, numpy as np, matplotlib.pyplot as plt
def plot_transform(image, transformation_fn, cmap=None):
if isinstance(image, torch.Tensor):
image = image.squeeze(0) # (C,H,W) -> (H,W) for grayscale
image = image * 0.5 + 0.5 # denormalise [-1,1] -> [0,1]
image = image.numpy()
plt.figure(figsize=(12, 6))
plt.subplot(2, 5, 1); plt.imshow(image, cmap=cmap)
plt.axis("off"); plt.title("Original Image")
for i in range(9):
t = transformation_fn(image)
if isinstance(t, torch.Tensor):
t = (t.squeeze(0) * 0.5 + 0.5).numpy()
plt.subplot(2, 5, i + 2); plt.imshow(t, cmap=cmap)
plt.axis("off"); plt.title(f"Augmentation {i+1}")
plt.tight_layout(); plt.show()In [ ]:
# runnable — confusion matrix by hand and with sklearn
import numpy as np
from sklearn.metrics import confusion_matrix, classification_report
y = np.array([+1, -1, +1, -1, +1, -1])
y_hat = np.array([+1, -1, +1, +1, +1, -1])
TP = int(np.sum((y == 1) & (y_hat == 1)))
TN = int(np.sum((y == -1) & (y_hat == -1)))
FP = int(np.sum((y == -1) & (y_hat == 1)))
FN = int(np.sum((y == 1) & (y_hat == -1)))
print(f"TP: {TP}, FP: {FP}, TN: {TN}, FN: {FN}")
precision = TP / (TP + FP)
recall = TP / (TP + FN)
f1 = 2 * precision * recall / (precision + recall)
print(f"accuracy={(TP+TN)/len(y):.3f} precision={precision:.3f} recall={recall:.3f} f1={f1:.3f}")
print("\nsklearn confusion matrix (rows = true, cols = predicted):")
print(confusion_matrix(y, y_hat, labels=[-1, 1]))
print(classification_report(y, y_hat, labels=[-1, 1],
target_names=["negative", "positive"], zero_division=0))In [ ]:
# reference — test_model: accuracy + classification report + confusion-matrix heatmap
import numpy as np, torch, seaborn as sns, matplotlib.pyplot as plt
from tqdm.auto import tqdm
from sklearn.metrics import confusion_matrix, classification_report
def test_model(model, test_loader, categories=None):
"""Evaluate on the test set with a classification report and a confusion matrix.
Returns (test_acc, cm, all_labels, all_preds).
`categories` is the list of class names, e.g. [str(i) for i in range(10)],
so the report and heatmap show meaningful labels instead of indices.
"""
if len(test_loader) == 0:
raise ValueError("test_loader is empty.")
model.eval()
correct = total = 0
all_preds, all_labels = [], []
with torch.no_grad():
for images, labels in tqdm(test_loader, desc="Testing", unit="batch"):
images, labels = images.to(device), labels.to(device)
outputs = model(images)
_, predicted = torch.max(outputs, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
all_preds.extend(predicted.cpu().numpy())
all_labels.extend(labels.cpu().numpy())
test_acc = 100.0 * correct / total
unique_labels = sorted(set(all_labels) | set(all_preds))
cm = confusion_matrix(all_labels, all_preds, labels=unique_labels)
display_names = ([f"Class {i}" for i in unique_labels] if categories is None
else [categories[i] for i in unique_labels])
print(f"\nTest Accuracy: {test_acc:.2f}%\n")
print("Classification Report:\n")
print(classification_report(all_labels, all_preds, labels=unique_labels,
target_names=display_names, zero_division=0))
plt.figure(figsize=(8, 6))
sns.heatmap(cm, annot=True, fmt="d", cmap="Blues",
xticklabels=display_names, yticklabels=display_names)
plt.title("Confusion Matrix"); plt.xlabel("Predicted Label"); plt.ylabel("True Label")
plt.tight_layout(); plt.show()
return test_acc, cm, all_labels, all_predsIn [ ]:
# reference — visualize_predictions: correct on the top row, mistakes on the bottom
import numpy as np, torch, matplotlib.pyplot as plt
def visualize_predictions(model, test_loader, categories, num_images=10):
"""Show correctly (green) and incorrectly (red) classified test images."""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model.to(device); model.eval()
correct_samples, incorrect_samples = [], []
with torch.no_grad():
for images, labels in test_loader:
images, labels = images.to(device), labels.to(device)
predicted = torch.argmax(model(images), dim=1)
for i in range(len(predicted)):
sample = (images[i].detach().cpu(), labels[i].item(), predicted[i].item())
(correct_samples if predicted[i] == labels[i] else incorrect_samples).append(sample)
num_correct = min(num_images // 2, len(correct_samples))
num_incorrect = min(num_images - num_correct, len(incorrect_samples))
num_cols = max(num_correct, num_incorrect)
if num_cols == 0:
print("No images available to display."); return
mean = torch.tensor([0.5, 0.5, 0.5]).view(3, 1, 1)
std = torch.tensor([0.5, 0.5, 0.5]).view(3, 1, 1)
plt.figure(figsize=(4 * num_cols, 8))
for row, (samples, n, colour) in enumerate([(correct_samples, num_correct, "green"),
(incorrect_samples, num_incorrect, "red")]):
for i in range(num_cols):
plt.subplot(2, num_cols, row * num_cols + i + 1)
if i < n:
img_tensor, true_label, pred_label = samples[i]
img_tensor = img_tensor * std + mean # reverse normalisation
img = np.clip(img_tensor.permute(1, 2, 0).numpy(), 0, 1)
plt.imshow(img)
plt.title(f"True: {categories[true_label]}\nPred: {categories[pred_label]}",
color=colour, fontsize=12)
plt.axis("off")
plt.tight_layout(); plt.show()In [ ]:
# reference — ResNet18 transfer learning, then fine-tuning # needs download
import copy, torch, torch.nn as nn, torch.optim as optim
from torchvision import models
# 1-2) load pretrained weights and inspect
resnet_weights = models.ResNet18_Weights.DEFAULT
resnet_model = models.resnet18(weights=resnet_weights)
# print(resnet_model) # -> the last layer is called `fc`
# 3) replace the classifier head for 2 classes (cats vs dogs)
in_features = resnet_model.fc.in_features
resnet_model.fc = nn.Linear(in_features, 2)
resnet_model = resnet_model.to(device)
# 4) freeze everything, then unfreeze ONLY the new head
for param in resnet_model.parameters():
param.requires_grad = False
for param in resnet_model.fc.parameters():
param.requires_grad = True
for name, param in resnet_model.named_parameters(): # always double-check
if param.requires_grad:
print("trainable:", name)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(resnet_model.parameters(), lr=0.001)
n_epochs = 50
# best_model_resnet, history_resnet = train_model(
# model=resnet_model, train_loader=train_dataloader, val_loader=val_dataloader,
# criterion=criterion, optimizer=optimizer, num_epochs=n_epochs,
# name='resnet', use_early_stopping=True, verbose=True, min_delta=0.001)
# plot_training_history(history_resnet, plot_lr=True)
# ---- FINE-TUNING: copy the model, then also unfreeze layer4 ----------------
finetune_model = copy.deepcopy(resnet_model)
for param in finetune_model.parameters(): # safe reset
param.requires_grad = False
for name, layer in finetune_model.named_children(): # named_children -> top-level blocks
if name in ["layer4", "fc"]:
for param in layer.parameters():
param.requires_grad = True
optimizer_finetune = optim.Adam(finetune_model.parameters(), lr=0.001)
# best_model_finetune, history_finetune = train_model(
# model=finetune_model, train_loader=train_dataloader, val_loader=val_dataloader,
# criterion=criterion, optimizer=optimizer_finetune, num_epochs=n_epochs,
# name='resnet finetune', use_early_stopping=True, verbose=True)In [ ]:
# reference — the dogs/cats data pipeline that feeds it
import os, torch
from torchvision import datasets
from torchvision.transforms import v2
from torch.utils.data import DataLoader
# DATA_PATH/
# ├── train/{cats,dogs}
# └── test/{cats,dogs}
DATA_PATH = "/content"
train, test = "train", "test"
categories = os.listdir(os.path.join(DATA_PATH, train))
train_dataset = datasets.ImageFolder(os.path.join(DATA_PATH, train))
val_dataset = datasets.ImageFolder(os.path.join(DATA_PATH, test))
transforms_rgb = v2.Compose([
v2.Resize((100, 100)),
v2.ToImage(),
v2.ToDtype(torch.float32, scale=True),
v2.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
])
train_RAM = RAMDatasetWrapper(train_dataset, transforms_rgb)
val_RAM = RAMDatasetWrapper(val_dataset, transforms_rgb)
train_dataloader = DataLoader(train_RAM, batch_size=32, shuffle=True, num_workers=0)
val_dataloader = DataLoader(val_RAM, batch_size=32, shuffle=False, num_workers=0)
for images, labels in train_dataloader:
print(f"image shape: {images.shape}, labels shape: {labels.shape}")
break
def class_distribution(dataset):
"""Print total sample count and per-class counts for a split."""
path = os.path.join(DATA_PATH, dataset)
counts = [len(os.listdir(os.path.join(path, cat))) for cat in categories]
print(f"Number of samples: {sum(counts)}")
for cat, count in zip(categories, counts):
print(f"{cat}: {count} samples")In [ ]:
# runnable — the full VAE (architecture, loss, training, sampling)
import numpy as np, torch, torch.nn as nn, torch.nn.functional as F
import matplotlib.pyplot as plt
from torchvision.utils import make_grid
class VAE(nn.Module):
def __init__(self, input_dim=784, hidden_dim=64, latent_dim=32):
super().__init__()
self.encoder = nn.Sequential(
nn.Flatten(),
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
)
self.fc_mu = nn.Linear(hidden_dim, latent_dim)
self.fc_logvar = nn.Linear(hidden_dim, latent_dim)
self.decoder = nn.Sequential(
nn.Linear(latent_dim, hidden_dim),
nn.ReLU(),
nn.Linear(hidden_dim, input_dim),
nn.Sigmoid(), # outputs in [0,1] -> matches BCE
)
self.latent_dim = latent_dim
def encode(self, x):
z = self.encoder(x)
return self.fc_mu(z), self.fc_logvar(z)
def decode(self, z):
return self.decoder(z)
def sample(self, mu, logvar):
std = torch.exp(0.5 * logvar) # e^(1/2 * log(std^2)) = std
eps = torch.randn_like(std) # eps ~ N(0, 1)
return mu + eps * std # <- the reparameterization trick
def forward(self, x):
mu, logvar = self.encode(x)
z = self.sample(mu, logvar)
return self.decode(z), mu, logvar
def vae_loss(x_tilde, x, mu, logvar):
x = x.view(x_tilde.size()) # a view, no extra memory
BCE = F.binary_cross_entropy(x_tilde, x, reduction="sum")
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
return BCE + KLD
def train_vae(model, optimizer, train_loader, device):
model.train()
train_loss = 0
for data, _ in train_loader:
x = data.to(device)
x_tilde, mu, logvar = model(x)
loss = vae_loss(x_tilde, x, mu, logvar)
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_loss += loss.item()
return train_loss / len(train_loader.dataset)
# smoke test on random "images" so the cell runs offline
vae = VAE(latent_dim=2).to(device)
x = torch.rand(8, 1, 28, 28).to(device)
x_tilde, mu, logvar = vae(x)
print("reconstruction:", tuple(x_tilde.shape), "| mu:", tuple(mu.shape))
print("loss:", vae_loss(x_tilde, x, mu, logvar).item())In [ ]:
# reference — the three VAE visualisations
import numpy as np, torch, matplotlib.pyplot as plt
from torchvision.utils import make_grid
def visualize_samples(model, device, num_samples=16):
"""Sample random z ~ N(0,1) and decode -> a grid of *new* digits."""
model.eval()
with torch.no_grad():
g = torch.Generator(device=device); g.manual_seed(42)
z = torch.randn(num_samples, model.latent_dim, generator=g, device=device)
samples = model.decode(z).cpu().view(-1, 1, 28, 28)
grid = make_grid(samples, nrow=int(num_samples ** 0.5), padding=0)
plt.figure(figsize=(5, 5))
plt.imshow(np.transpose(grid.numpy(), (1, 2, 0)), cmap="gray")
plt.axis("off"); plt.tight_layout(pad=0); plt.show()
def plot_latent_clusters(model, train_loader, device, num_batches=100):
"""Encode the training set and scatter z, coloured by digit label."""
model.eval()
zs, labels = [], []
with torch.no_grad():
for i, (data, label) in enumerate(train_loader):
if i >= num_batches:
break
data = data.view(-1, 784).to(device) # nn.Linear needs 1-D input
mu, logvar = model.encode(data)
zs.append(model.sample(mu, logvar).cpu())
labels.append(label)
zs, labels = torch.cat(zs), torch.cat(labels)
plt.figure(figsize=(8, 6))
for digit in range(10):
mask = labels == digit
plt.scatter(zs[mask, 0], zs[mask, 1], label=str(digit), alpha=0.4, s=15)
plt.legend(title="Digit", loc="upper right")
plt.title("Latent Space Clusters (z ~ N(mu, σ²))")
plt.xlabel("z1"); plt.ylabel("z2"); plt.grid(True); plt.tight_layout(); plt.show()
def generate_image_from_input_z(model, z, device, digit_size=28):
"""Decode ONE hand-picked latent coordinate."""
model.eval()
z_tensor = torch.tensor([z], dtype=torch.float32).to(device)
with torch.no_grad():
return model.decode(z_tensor).view(digit_size, digit_size).cpu().numpy()
def plot_latent_space(model, device, scale=2.0, n=25, digit_size=28, figsize=8):
"""Decode a full grid of latent coordinates -> the classic VAE manifold."""
assert model.latent_dim == 2, "Latent space must be 2D to plot this grid."
grid_x = np.linspace(-scale, scale, n)
grid_y = np.linspace(scale, -scale, n)
figure = np.zeros((digit_size * n, digit_size * n))
model.eval()
with torch.no_grad():
for i, yi in enumerate(grid_y):
for j, xi in enumerate(grid_x):
z_sample = torch.tensor([[xi, yi]], dtype=torch.float32).to(device)
digit = model.decode(z_sample).view(digit_size, digit_size).cpu().numpy()
figure[i * digit_size:(i + 1) * digit_size,
j * digit_size:(j + 1) * digit_size] = digit
plt.figure(figsize=(figsize, figsize))
ticks = np.arange(digit_size // 2, n * digit_size + digit_size // 2, digit_size)
plt.xticks(ticks, np.round(grid_x, 1)); plt.yticks(ticks, np.round(grid_y, 1))
plt.xlabel("z1"); plt.ylabel("z2")
plt.imshow(figure, cmap="gray")
plt.title("Decoded Digits Across 2D Latent Space")
plt.tight_layout(); plt.show()Warning:
Output truncated. This notebook contains too many cells to display efficiently.