How do you implement distributed tracing and observability in a Go microservice system?

// OpenTelemetry setup
import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/trace"
)

func initTracer(ctx context.Context) (func(), error) {
    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint("otel-collector:4317"),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil { return nil, err }

    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter),
        trace.WithResource(resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceNameKey.String("user-service"),
            semconv.ServiceVersionKey.String("1.0.0"),
        )),
        trace.WithSampler(trace.TraceIDRatioBased(0.1)), // sample 10%
    )
    otel.SetTracerProvider(tp)
    return func() { tp.Shutdown(context.Background()) }, nil
}

// Instrument a function
var tracer = otel.Tracer("user-service")

func (s *userServiceServer) GetUser(
    ctx context.Context, req *pb.GetUserRequest,
) (*pb.User, error) {
    ctx, span := tracer.Start(ctx, "UserService.GetUser")
    defer span.End()

    span.SetAttributes(
        attribute.Int64("user.id", req.Id),
    )

    user, err := s.repo.FindByID(ctx, int(req.Id))
    if err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return nil, status.Errorf(codes.Internal, "internal error")
    }
    return toProto(user), nil
}

// Propagate trace context in gRPC metadata
// Use otelgrpc interceptors to do this automatically:
grpc.NewServer(
    grpc.StatsHandler(otelgrpc.NewServerHandler()),
)