How do you diagnose a neural network that is not training correctly from its loss curves?

Reading loss curves is one of the most important practical skills in deep learning. The shape of the training and validation loss over time reveals the failure mode and guides the fix.

import torch
import torch.nn as nn

# Checking for gradient issues
model = nn.Linear(10, 5)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)

for step, (X, y) in enumerate(loader):
    optimizer.zero_grad()
    loss = criterion(model(X), y)
    loss.backward()

    # Check for NaN/Inf in loss
    if not torch.isfinite(loss):
        print(f'Step {step}: non-finite loss = {loss.item()}')
        break

    # Monitor gradient norms
    total_norm = 0
    for p in model.parameters():
        if p.grad is not None:
            total_norm += p.grad.data.norm(2).item() ** 2
    total_norm = total_norm ** 0.5
    if step % 100 == 0:
        print(f'Step {step}: loss={loss.item():.4f} grad_norm={total_norm:.4f}')

    optimizer.step()

# Check dead ReLU neurons
def count_dead_neurons(model, X):
    activations = []
    def hook(m, inp, out):
        activations.append((out <= 0).float().mean().item())
    handles = [l.register_forward_hook(hook)
               for l in model.modules() if isinstance(l, nn.ReLU)]
    with torch.no_grad(): model(X)
    for h in handles: h.remove()
    return activations  # fraction of dead neurons per layer