What is the difference between nn.Parameter and a regular tensor attribute in nn.Module?

nn.Parameter is a special tensor subclass that, when assigned as an attribute of an nn.Module, is automatically registered in the module's parameter list — meaning it appears in model.parameters(), gets moved by .to(device), and is saved in state_dict(). A plain tensor attribute does none of this.

import torch
import torch.nn as nn

class CustomLayer(nn.Module):
    def __init__(self, dim: int):
        super().__init__()

        # nn.Parameter — automatically registered, tracked, trained
        self.weight = nn.Parameter(torch.randn(dim, dim))
        self.bias   = nn.Parameter(torch.zeros(dim))

        # Plain tensor — NOT registered, NOT trained, invisible to optimizer
        self.scale  = torch.tensor(2.0)  # WRONG if meant to be learnable!

        # register_buffer — for non-trainable state that SHOULD move with
        # the model and be saved (e.g. BatchNorm running mean/var)
        self.register_buffer("running_mean", torch.zeros(dim))

    def forward(self, x):
        return x @ self.weight + self.bias

layer = CustomLayer(10)

# Check what appears in parameters()
for name, p in layer.named_parameters():
    print(name, p.shape)
# weight torch.Size([10, 10])
# bias   torch.Size([10])
# scale and running_mean do NOT appear here!

# Check state_dict — includes parameters AND buffers, but not plain tensors
print(layer.state_dict().keys())
# odict_keys(['weight', 'bias', 'running_mean'])

# .to(device) moves Parameters and registered buffers, but NOT plain tensor attrs
layer.to("cuda") if torch.cuda.is_available() else None
# layer.scale would STILL be on CPU — a common silent bug!