How do you implement safe retries in Go microservices?

Retries improve resilience but must be done correctly. Retrying non-idempotent operations (POST create) without idempotency keys causes duplicate records. Retrying without backoff causes thundering herd. Retrying infinite times causes cascading failure.

// Safe retry with exponential backoff and jitter
type RetryConfig struct {
    MaxAttempts int
    InitialWait time.Duration
    MaxWait     time.Duration
    Multiplier  float64
}

func Retry(ctx context.Context, cfg RetryConfig, fn func() error) error {
    wait := cfg.InitialWait
    for attempt := 1; ; attempt++ {
        err := fn()
        if err == nil { return nil }

        // Don't retry permanent errors
        if !isRetryable(err) { return err }

        if attempt >= cfg.MaxAttempts {
            return fmt.Errorf("after %d attempts: %w", attempt, err)
        }

        // Exponential backoff with jitter
        jitter := time.Duration(rand.Int63n(int64(wait / 2)))
        sleep := wait + jitter
        if sleep > cfg.MaxWait { sleep = cfg.MaxWait }

        select {
        case <-time.After(sleep):
        case <-ctx.Done():
            return fmt.Errorf("retry cancelled: %w", ctx.Err())
        }
        wait = time.Duration(float64(wait) * cfg.Multiplier)
    }
}

func isRetryable(err error) bool {
    // Only retry transient errors
    code := status.Code(err)
    return code == codes.Unavailable ||
        code == codes.DeadlineExceeded ||
        code == codes.ResourceExhausted
}

// Idempotency key for non-idempotent operations
func (s *userServiceServer) CreateUser(
    ctx context.Context, req *pb.CreateUserRequest,
) (*pb.User, error) {
    // Extract idempotency key from metadata
    md, _ := metadata.FromIncomingContext(ctx)
    idempKey := md.Get("idempotency-key")

    // Check if we've seen this key before
    if len(idempKey) > 0 {
        if cached, err := s.cache.Get(ctx, "idemp:"+idempKey[0]); err == nil {
            var existing pb.User
            proto.Unmarshal(cached, &existing)
            return &existing, nil // return cached response
        }
    }
    // ... create user, cache result with idempotency key
}

Invest now in Acorns!!! 🚀
Join Acorns and get your $5 bonus!

Acorns is a micro-investing app that automatically invests your "spare change" from daily purchases into diversified, expert-built portfolios of ETFs. It is designed for beginners, allowing you to start investing with as little as $5. The service automates saving and investing. Disclosure: I may receive a referral bonus.

Invest now!!! Get Free equity stock (US, UK only)!

Use Robinhood app to invest in stocks. It is safe and secure. Use the Referral link to claim your free stock when you sign up!.

The Robinhood app makes it easy to trade stocks, crypto and more.

Webull! Receive free stock by signing up using the link: Webull signup.