How do you monitor and debug LLM applications in production using LangSmith?

LangSmith is LangChain's observability platform for LLM applications. It automatically traces every LLM call, chain step, and tool invocation, providing: full input/output logging, latency and cost breakdowns, error tracking, prompt version comparison, and human feedback collection. In production, this level of visibility is essential for debugging unexpected outputs, identifying expensive call patterns, and iterating on prompt quality.

# Enable LangSmith tracing with environment variables
import os
os.environ['LANGCHAIN_TRACING_V2']  = 'true'
os.environ['LANGCHAIN_API_KEY']     = 'ls__...'    # LangSmith API key
os.environ['LANGCHAIN_PROJECT']     = 'my-rag-app' # project name

# After setting these, ALL LangChain calls are automatically traced
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate

chain = (
    ChatPromptTemplate.from_template('Answer: {question}')
    | ChatOpenAI(model='gpt-4o-mini')
)
result = chain.invoke({'question': 'What is LangSmith?'})
# This call is now visible at smith.langchain.com with full trace

# ── Manual tracing with @traceable decorator
from langsmith import traceable

@traceable(name='my_rag_step', run_type='retriever')
def retrieve_docs(query: str) -> list:
    # Retrieval logic here
    return [{'content': 'relevant doc', 'source': 'wiki'}]

@traceable(name='full_rag_pipeline')
def rag_pipeline(user_query: str) -> str:
    docs    = retrieve_docs(user_query)   # sub-trace automatically nested
    context = '\n'.join(d['content'] for d in docs)
    resp    = chain.invoke({'question': f'Context: {context}\n{user_query}'})
    return resp.content

answer = rag_pipeline('What is transformer attention?')

# ── Adding user feedback
from langsmith import Client

ls_client = Client()
# After showing response to user, collect feedback
# run_id comes from the LangSmith trace
ls_client.create_feedback(
    run_id='some-run-uuid',
    key='correctness',
    score=1.0,
    comment='Perfect answer, well cited',
)