What is the computation graph in PyTorch and how does the dynamic graph differ from a static graph?

PyTorch builds a dynamic computation graph (also called eager execution or define-by-run). Every time you run the forward pass, a new graph is constructed on-the-fly based on the actual Python code paths executed. This is in contrast to TensorFlow 1.x's static graph, which is compiled once and then executed repeatedly.

import torch

# Dynamic graph: Python control flow works naturally
def dynamic_model(x, use_relu=True):
    h = x @ torch.randn(4, 4)
    if use_relu:           # real Python if — changes the graph!
        h = torch.relu(h)
    else:
        h = torch.tanh(h)
    return h.sum()

x = torch.randn(2, 4, requires_grad=True)

# Each call may build a DIFFERENT graph depending on use_relu
loss1 = dynamic_model(x, use_relu=True)
loss1.backward()   # graph includes ReLU nodes

x.grad.zero_()
loss2 = dynamic_model(x, use_relu=False)
loss2.backward()   # graph includes Tanh nodes

# The graph is discarded after backward() by default
# retain_graph=True keeps it for multiple backward calls
y = (x ** 2).sum()
y.backward(retain_graph=True)   # graph kept
y.backward()                    # can call again

# Inspecting the graph
z = x ** 3
print(z.grad_fn)                # <PowBackward0>
print(z.grad_fn.next_functions) # upstream functions