How do you use multimodal models (vision-language) with Hugging Face for image understanding tasks?

Multimodal models like LLaVA, PaliGemma, and Idefics combine a vision encoder (typically a CLIP or SigLIP model) with an LLM, enabling reasoning over both images and text. They are used for image captioning, visual question answering (VQA), document understanding, and chart analysis. Loading them follows the same Auto-class pattern, with the addition of a processor that handles both image and text preprocessing.

from transformers import AutoProcessor, AutoModelForVision2Seq
from PIL import Image
import requests
import torch

# Load PaliGemma (Google's vision-language model)
model_id  = 'google/paligemma-3b-pt-224'
processor = AutoProcessor.from_pretrained(model_id)
model     = AutoModelForVision2Seq.from_pretrained(
    model_id, torch_dtype=torch.bfloat16
).to('cuda')

# Load an image
url   = 'https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/bee.jpg'
image = Image.open(requests.get(url, stream=True).raw).convert('RGB')

# Visual question answering
question = 'What insect is shown in this image?'
inputs = processor(
    images=image,
    text=question,
    return_tensors='pt',
).to('cuda')

with torch.no_grad():
    generated_ids = model.generate(**inputs, max_new_tokens=50)
answer = processor.decode(generated_ids[0], skip_special_tokens=True)
print(answer)  # 'A honeybee is shown in this image.'

# ── Using the pipeline API for vision tasks
from transformers import pipeline

vqa_pipe = pipeline(
    'image-text-to-text',
    model='llava-hf/llava-1.5-7b-hf',
    torch_dtype=torch.bfloat16,
    device_map='auto',
)
result = vqa_pipe(
    {'image': image, 'text': 'Describe what you see in detail.'},
    max_new_tokens=200,
)
print(result[0]['generated_text'])