What is an encoder-decoder architecture and how is it used for sequence-to-sequence tasks?

Encoder-decoder (seq2seq) architectures handle tasks where the input and output are sequences of potentially different lengths — machine translation, summarisation, speech recognition, image captioning. The encoder processes the full input sequence and produces a context representation; the decoder generates the output sequence token by token, conditioning each prediction on the context and all previously generated tokens.

In transformer-based seq2seq, the encoder uses bidirectional self-attention (each position attends to all input positions), while the decoder uses two attention mechanisms: masked self-attention (each output position can only attend to previous output positions, preserving the autoregressive property) and cross-attention (each decoder position attends to all encoder output positions to draw relevant information from the input).

import torch
import torch.nn as nn

# PyTorch's built-in Transformer (encoder-decoder)
transformer = nn.Transformer(
    d_model=512,
    nhead=8,
    num_encoder_layers=6,
    num_decoder_layers=6,
    dim_feedforward=2048,
    dropout=0.1,
    batch_first=True
)

# Source and target sequences
src = torch.randn(4, 20, 512)   # (batch, src_len, d_model)
tgt = torch.randn(4, 15, 512)   # (batch, tgt_len, d_model)

# Causal mask: prevent decoder from attending to future target tokens
tgt_len = tgt.size(1)
tgt_mask = nn.Transformer.generate_square_subsequent_mask(tgt_len)

out = transformer(src, tgt, tgt_mask=tgt_mask)
print(out.shape)  # (4, 15, 512)

# Teacher forcing: at training time, feed ground-truth previous tokens
# to the decoder (not its own previous predictions)
# At inference: autoregressive — use model's own previous output:
def greedy_decode(model, src, max_len, sos_idx, eos_idx):
    memory = model.encoder(src)
    ys = torch.tensor([[sos_idx]])
    for _ in range(max_len):
        mask = nn.Transformer.generate_square_subsequent_mask(ys.size(1))
        out  = model.decoder(ys.float(), memory, tgt_mask=mask)
        next_token = out[:, -1].argmax()
        ys = torch.cat([ys, next_token.unsqueeze(0).unsqueeze(0)], dim=1)
        if next_token.item() == eos_idx: break
    return ys