What optimizers does PyTorch provide and what is the difference between SGD, Adam, and AdamW?

import torch
import torch.nn as nn
import torch.optim as optim

model = nn.Linear(10, 1)

# ── SGD with momentum
opt_sgd = optim.SGD(
    model.parameters(),
    lr=0.01,
    momentum=0.9,        # accelerates in consistent gradient directions
    weight_decay=1e-4,   # L2 regularisation
)

# ── Adam — adaptive learning rate per parameter
opt_adam = optim.Adam(
    model.parameters(),
    lr=1e-3,
    betas=(0.9, 0.999),  # momentum decay rates
    eps=1e-8,
)

# ── AdamW — decoupled weight decay (recommended for fine-tuning)
opt_adamw = optim.AdamW(
    model.parameters(),
    lr=2e-5,              # typical for fine-tuning pretrained models
    weight_decay=0.01,
)

# ── Standard training step
x, y = torch.randn(16, 10), torch.randn(16, 1)
loss_fn = nn.MSELoss()

opt_adamw.zero_grad()       # 1. clear old gradients
pred = model(x)              # 2. forward pass
loss = loss_fn(pred, y)      # 3. compute loss
loss.backward()              # 4. backpropagate
opt_adamw.step()             # 5. update parameters

# ── Different learning rates per parameter group
optimizer = optim.AdamW([
    {"params": model.weight, "lr": 1e-3},
    {"params": model.bias,   "lr": 1e-4},
])