How do you use the Hugging Face Datasets library for training and evaluation?

The datasets library provides a unified interface to thousands of NLP and computer vision datasets from the Hub, with built-in streaming, caching, and memory-mapped access via Apache Arrow. It integrates directly with the Transformers Trainer and works well with PyTorch DataLoader.

from datasets import load_dataset, DatasetDict

# ── Load a public dataset
ds = load_dataset('imdb')           # train/test splits
print(ds)                           # DatasetDict with splits
print(ds['train'][0])               # {'text': '...', 'label': 1}
print(ds['train'].features)         # {'text': Value(dtype='string'), 'label': ClassLabel}

# ── Stream large datasets without downloading everything
stream_ds = load_dataset('c4', 'en', split='train', streaming=True)
for sample in stream_ds.take(3):
    print(sample['text'][:100])

# ── Load from local files
local_ds = load_dataset('csv', data_files={'train': 'train.csv', 'test': 'test.csv'})

# ── Preprocessing: map over the dataset
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')

def tokenize(examples):
    return tokenizer(
        examples['text'],
        padding='max_length',
        truncation=True,
        max_length=512,
    )

tokenized = ds.map(
    tokenize,
    batched=True,          # process in batches of 1000 — much faster
    remove_columns=['text'],# remove raw text after tokenising
    num_proc=4,            # parallel processing
)
tokenized.set_format('torch')  # return tensors in PyTorch format

# ── Train/val split
split = ds['train'].train_test_split(test_size=0.1, seed=42)
train_ds, val_ds = split['train'], split['test']

# ── Filter and select
long_reviews = ds['train'].filter(lambda x: len(x['text']) > 500)
small_ds     = ds['train'].select(range(100))  # first 100 examples