How do you handle documents or conversations that exceed an LLM's context window?

Every LLM has a maximum context window (measured in tokens) — GPT-4o supports 128K tokens, Claude 3.5 Sonnet 200K, Llama 3.1 128K. Inputs exceeding this limit are either truncated (silently losing content) or raise an error. Several strategies handle long documents:

from langchain_openai import ChatOpenAI
from langchain.chains.summarize import load_summarize_chain
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import PyPDFLoader

llm = ChatOpenAI(model='gpt-4o-mini', temperature=0)

# Load a very long document
docs   = PyPDFLoader('long_report.pdf').load()
chunks = RecursiveCharacterTextSplitter(
    chunk_size=4000, chunk_overlap=200
).split_documents(docs)

# ── Map-reduce summarisation
map_reduce_chain = load_summarize_chain(
    llm,
    chain_type='map_reduce',  # 'stuff' | 'map_reduce' | 'refine'
    verbose=True,
)
summary = map_reduce_chain.invoke({'input_documents': chunks})
print(summary['output_text'])

# ── Token counting before API calls (avoid surprises)
import tiktoken

enc = tiktoken.encoding_for_model('gpt-4o')

def count_tokens(text: str, model: str = 'gpt-4o') -> int:
    enc = tiktoken.encoding_for_model(model)
    return len(enc.encode(text))

with open('big_doc.txt') as f:
    content = f.read()
n_tokens = count_tokens(content)
max_ctx  = 128_000  # gpt-4o context window
print(f'{n_tokens} tokens — {"fits" if n_tokens < max_ctx else "exceeds context"}')