What evaluation metrics are most commonly used in deep learning tasks and how do you implement them in PyTorch?

The choice of evaluation metric should match the task's real-world objective, not just be the easiest to compute. The training loss and the evaluation metric are often different — models are trained with cross-entropy but evaluated with accuracy, F1, mAP, or BLEU depending on the application.

import torch
from torchmetrics import Accuracy, F1Score, AUROC, MeanSquaredError

# torchmetrics: handles accumulation across batches correctly
n_classes = 5
acc  = Accuracy(task='multiclass', num_classes=n_classes)
f1   = F1Score(task='multiclass', num_classes=n_classes, average='macro')
auroc = AUROC(task='multiclass', num_classes=n_classes)

model.eval()
with torch.no_grad():
    for X, y in val_loader:
        logits = model(X)
        preds  = logits.argmax(dim=1)
        probs  = torch.softmax(logits, dim=1)
        acc.update(preds, y)
        f1.update(preds, y)
        auroc.update(probs, y)

print(f'Val Acc:  {acc.compute():.4f}')
print(f'Val F1:   {f1.compute():.4f}')
print(f'Val AUROC:{auroc.compute():.4f}')

# Manual accuracy (without torchmetrics)
all_preds, all_labels = [], []
with torch.no_grad():
    for X, y in val_loader:
        preds = model(X).argmax(1)
        all_preds.append(preds.cpu())
        all_labels.append(y.cpu())
preds  = torch.cat(all_preds)
labels = torch.cat(all_labels)
accuracy = (preds == labels).float().mean()
print(f'Accuracy: {accuracy:.4f}')