Why is learning rate scheduling important and what are the most common strategies?

A fixed learning rate is a poor choice for most training runs: too high early on causes instability; too high late in training prevents fine convergence to a sharp minimum. Learning rate schedulers systematically vary the lr during training to get the best of both worlds — fast progress early, precise convergence later.

import torch
import torch.nn as nn
import torch.optim as optim

model = nn.Linear(10, 1)
optimizer = optim.AdamW(model.parameters(), lr=1e-3)

# CosineAnnealingLR — smooth decay from max to min lr
scheduler_cos = optim.lr_scheduler.CosineAnnealingLR(
    optimizer, T_max=100, eta_min=1e-6
)

# OneCycleLR — requires total_steps at init
n_epochs, steps_per_epoch = 10, 100
scheduler_1cycle = optim.lr_scheduler.OneCycleLR(
    optimizer,
    max_lr=1e-2,
    total_steps=n_epochs * steps_per_epoch,
    pct_start=0.3,   # 30% of steps for warmup
)

# ReduceLROnPlateau — triggered by validation metric
scheduler_plateau = optim.lr_scheduler.ReduceLROnPlateau(
    optimizer, mode='min', factor=0.5, patience=5
)

# Training loop
for epoch in range(100):
    train_one_epoch(model, optimizer, loader)
    val_loss = validate(model, val_loader)

    scheduler_cos.step()             # epoch-based schedulers
    scheduler_plateau.step(val_loss) # metric-based scheduler
    print(f'LR: {optimizer.param_groups[0]["lr"]:.6f}')