How do you implement safe retries in Go microservices?

// Safe retry with exponential backoff and jitter
type RetryConfig struct {
    MaxAttempts int
    InitialWait time.Duration
    MaxWait     time.Duration
    Multiplier  float64
}

func Retry(ctx context.Context, cfg RetryConfig, fn func() error) error {
    wait := cfg.InitialWait
    for attempt := 1; ; attempt++ {
        err := fn()
        if err == nil { return nil }

        // Don't retry permanent errors
        if !isRetryable(err) { return err }

        if attempt >= cfg.MaxAttempts {
            return fmt.Errorf("after %d attempts: %w", attempt, err)
        }

        // Exponential backoff with jitter
        jitter := time.Duration(rand.Int63n(int64(wait / 2)))
        sleep := wait + jitter
        if sleep > cfg.MaxWait { sleep = cfg.MaxWait }

        select {
        case <-time.After(sleep):
        case <-ctx.Done():
            return fmt.Errorf("retry cancelled: %w", ctx.Err())
        }
        wait = time.Duration(float64(wait) * cfg.Multiplier)
    }
}

func isRetryable(err error) bool {
    // Only retry transient errors
    code := status.Code(err)
    return code == codes.Unavailable ||
        code == codes.DeadlineExceeded ||
        code == codes.ResourceExhausted
}

// Idempotency key for non-idempotent operations
func (s *userServiceServer) CreateUser(
    ctx context.Context, req *pb.CreateUserRequest,
) (*pb.User, error) {
    // Extract idempotency key from metadata
    md, _ := metadata.FromIncomingContext(ctx)
    idempKey := md.Get("idempotency-key")

    // Check if we've seen this key before
    if len(idempKey) > 0 {
        if cached, err := s.cache.Get(ctx, "idemp:"+idempKey[0]); err == nil {
            var existing pb.User
            proto.Unmarshal(cached, &existing)
            return &existing, nil // return cached response
        }
    }
    // ... create user, cache result with idempotency key
}