What are the most impactful hyperparameters to tune in deep learning and what is the recommended search order?

Deep learning has many hyperparameters, but they are not equally important. Empirical research and practitioner experience has established a rough hierarchy of impact. Tuning in the wrong order wastes compute — finding the optimal dropout rate is pointless if the learning rate is still wildly off.

import optuna
import torch
import torch.nn as nn

def objective(trial):
    # Optuna suggests hyperparameters — log-uniform search for lr
    lr         = trial.suggest_float('lr', 1e-5, 1e-1, log=True)
    wd         = trial.suggest_float('weight_decay', 1e-5, 1e-1, log=True)
    n_layers   = trial.suggest_int('n_layers', 2, 6)
    hidden_dim = trial.suggest_categorical('hidden_dim', [128, 256, 512])
    dropout    = trial.suggest_float('dropout', 0.0, 0.5)

    layers = []
    in_dim = 784
    for _ in range(n_layers):
        layers += [nn.Linear(in_dim, hidden_dim), nn.ReLU(),
                   nn.Dropout(dropout)]
        in_dim = hidden_dim
    model = nn.Sequential(*layers, nn.Linear(in_dim, 10))

    optimizer = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=wd)
    val_acc = train_and_evaluate(model, optimizer, n_epochs=10)
    return val_acc

study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=50)
print('Best params:', study.best_params)