How do you manage LLM API costs and implement caching to reduce redundant calls?

LLM API costs can escalate quickly in production. For context, GPT-4o costs $5/1M input tokens and $15/1M output tokens — a system making 10,000 calls/day with 2,000 tokens each consumes $100+/day. Several strategies keep costs manageable: choosing the right model for the task, caching repeated queries, reducing prompt size, and batching calls.

# ── LangChain in-memory caching (same query returns cached response)
from langchain_core.globals import set_llm_cache
from langchain_community.cache import InMemoryCache, RedisCache
from langchain_openai import ChatOpenAI

# Cache in memory (process-level; resets on restart)
set_llm_cache(InMemoryCache())

llm = ChatOpenAI(model='gpt-4o-mini')
result1 = llm.invoke('What is 2+2?')  # hits API
result2 = llm.invoke('What is 2+2?')  # returns cached; zero cost

# ── Redis semantic cache (caches based on query SIMILARITY)
from langchain_community.cache import RedisSemanticCache
from langchain_openai import OpenAIEmbeddings

semantic_cache = RedisSemanticCache(
    redis_url='redis://localhost:6379',
    embedding=OpenAIEmbeddings(model='text-embedding-3-small'),
    score_threshold=0.95,  # cache if query similarity > 95%
)
set_llm_cache(semantic_cache)
# 'What is two plus two?' -> retrieves cached response for 'What is 2+2?'

# ── Cost estimation before calling
import tiktoken

def estimate_cost(prompt: str, model: str = 'gpt-4o') -> float:
    enc = tiktoken.encoding_for_model(model)
    n   = len(enc.encode(prompt))
    cost_per_1M = {'gpt-4o': 5.0, 'gpt-4o-mini': 0.15}
    return n / 1e6 * cost_per_1M.get(model, 5.0)

print(f'Estimated cost: ${estimate_cost("Hello world", "gpt-4o"):.6f}')

# ── Model routing: cheap model first, expensive only if needed
def smart_route(query: str) -> str:
    if len(query.split()) < 50:  # simple short queries
        return ChatOpenAI(model='gpt-4o-mini').invoke(query).content
    return ChatOpenAI(model='gpt-4o').invoke(query).content