🛡️ Resilience Patterns for Microservices

@Configuration
public class RetryConfig {

    @Bean
    public RetryRegistry retryRegistry() {
        RetryConfig config = RetryConfig.custom()
            .maxAttempts(3)
            .waitDuration(Duration.ofMillis(500))
            .intervalFunction(IntervalFunction.ofExponentialBackoff(500, 2.0))
            .retryOnResult(response -> response.getStatusCode() == 503)
            .retryExceptions(IOException.class, TimeoutException.class)
            .ignoreExceptions(BusinessException.class, ValidationException.class)
            .failAfterMaxAttempts(true)
            .build();

        return RetryRegistry.of(config);
    }
}

@Service
@Slf4j
public class PaymentServiceClient {

    private final RestTemplate restTemplate;

    @Retry(name = "paymentService", fallbackMethod = "paymentFallback")
    public PaymentResponse processPayment(PaymentRequest request) {
        log.info("Attempting payment for order: {}", request.getOrderId());
        return restTemplate.postForObject(
            "http://payment-service/api/payments",
            request,
            PaymentResponse.class
        );
    }

    private PaymentResponse paymentFallback(PaymentRequest request, Exception ex) {
        log.warn("Payment failed after retries for order: {}", request.getOrderId(), ex);
        return PaymentResponse.builder()
            .status(PaymentStatus.PENDING)
            .message("Payment queued for async processing")
            .build();
    }
}

@Configuration
public class CircuitBreakerConfig {

    @Bean
    public CircuitBreakerRegistry circuitBreakerRegistry() {
        CircuitBreakerConfig config = CircuitBreakerConfig.custom()
            // Sliding window
            .slidingWindowType(SlidingWindowType.COUNT_BASED)
            .slidingWindowSize(10)
            .minimumNumberOfCalls(5)
            // Failure thresholds
            .failureRateThreshold(50.0f)
            .slowCallRateThreshold(80.0f)
            .slowCallDurationThreshold(Duration.ofSeconds(2))
            // State transitions
            .waitDurationInOpenState(Duration.ofSeconds(30))
            .permittedNumberOfCallsInHalfOpenState(3)
            .automaticTransitionFromOpenToHalfOpenEnabled(true)
            // What counts as failure
            .recordExceptions(IOException.class, TimeoutException.class)
            .ignoreExceptions(BusinessException.class)
            .build();

        return CircuitBreakerRegistry.of(config);
    }
}

@Service
@Slf4j
public class InventoryServiceClient {

    private final WebClient webClient;

    @CircuitBreaker(name = "inventoryService", fallbackMethod = "inventoryFallback")
    public Mono<InventoryResponse> checkStock(String productId) {
        return webClient.get()
            .uri("/api/inventory/{productId}", productId)
            .retrieve()
            .bodyToMono(InventoryResponse.class);
    }

    private Mono<InventoryResponse> inventoryFallback(String productId, Throwable t) {
        log.warn("Circuit breaker open for inventory check, product: {}", productId);
        // Return cached or optimistic response
        return Mono.just(InventoryResponse.builder()
            .productId(productId)
            .available(true)  // Optimistic — verify at checkout
            .source("CACHE_FALLBACK")
            .build());
    }
}

@Service
@Slf4j
public class OrderService {

    @Bulkhead(name = "paymentBulkhead", type = Bulkhead.Type.THREADPOOL,
              fallbackMethod = "paymentBulkheadFallback")
    public CompletableFuture<PaymentResponse> processPayment(PaymentRequest request) {
        return CompletableFuture.supplyAsync(() -> {
            return paymentClient.charge(request);
        });
    }

    private CompletableFuture<PaymentResponse> paymentBulkheadFallback(
            PaymentRequest request, BulkheadFullException ex) {
        log.warn("Payment bulkhead full, queuing order: {}", request.getOrderId());
        return CompletableFuture.completedFuture(
            PaymentResponse.queued(request.getOrderId())
        );
    }
}

@Service
public class NotificationService {

    @Bulkhead(name = "emailBulkhead", type = Bulkhead.Type.SEMAPHORE)
    public void sendEmail(EmailRequest request) {
        // At most N concurrent calls allowed
        emailClient.send(request);
    }
}

@RestController
@RequestMapping("/api/orders")
@Slf4j
public class OrderController {

    private final OrderService orderService;

    @RateLimiter(name = "orderApi", fallbackMethod = "rateLimitFallback")
    @PostMapping
    public ResponseEntity<OrderResponse> createOrder(@RequestBody OrderRequest request) {
        return ResponseEntity.ok(orderService.createOrder(request));
    }

    private ResponseEntity<OrderResponse> rateLimitFallback(
            OrderRequest request, RequestNotPermitted ex) {
        log.warn("Rate limit exceeded for order creation");
        return ResponseEntity.status(HttpStatus.TOO_MANY_REQUESTS)
            .header("Retry-After", "1")
            .body(OrderResponse.rateLimited());
    }
}

@Service
@Slf4j
public class ProductCatalogService {

    @TimeLimiter(name = "catalogService", fallbackMethod = "catalogTimeout")
    @CircuitBreaker(name = "catalogService")
    public CompletableFuture<ProductDetails> getProductDetails(String productId) {
        return CompletableFuture.supplyAsync(() -> {
            return catalogClient.fetchProduct(productId);
        });
    }

    private CompletableFuture<ProductDetails> catalogTimeout(
            String productId, TimeoutException ex) {
        log.warn("Catalog service timeout for product: {}", productId);
        return CompletableFuture.completedFuture(
            cachedProductRepository.findById(productId)
                .orElse(ProductDetails.unavailable(productId))
        );
    }
}

@Service
@Slf4j
public class RecommendationService {

    private final RecommendationClient recommendationClient;
    private final CacheManager cacheManager;
    private final List<String> defaultPopularProducts;

    /**
     * Strategy 1: Return cached data
     */
    @CircuitBreaker(name = "recommendations", fallbackMethod = "cachedRecommendations")
    public List<Product> getPersonalizedRecommendations(String userId) {
        List<Product> recommendations = recommendationClient.getForUser(userId);
        // Update cache on success
        cacheManager.getCache("recommendations").put(userId, recommendations);
        return recommendations;
    }

    private List<Product> cachedRecommendations(String userId, Throwable t) {
        log.warn("Using cached recommendations for user: {}", userId);
        Cache.ValueWrapper cached = cacheManager.getCache("recommendations").get(userId);
        if (cached != null) {
            return (List<Product>) cached.get();
        }
        return popularProductsFallback(userId, t);
    }

    /**
     * Strategy 2: Return default/static data
     */
    private List<Product> popularProductsFallback(String userId, Throwable t) {
        log.warn("Using popular products fallback for user: {}", userId);
        return defaultPopularProducts;
    }

    /**
     * Strategy 3: Return empty/neutral response
     */
    private List<Product> emptyFallback(String userId, Throwable t) {
        log.error("All fallbacks exhausted for user: {}", userId);
        return Collections.emptyList();  // UI shows "No recommendations available"
    }
}

@Service
@Slf4j
public class PaymentGatewayService {

    private final PaymentClient paymentClient;

    /**
     * Combined resilience: Retry (innermost) -> CircuitBreaker -> Bulkhead -> TimeLimiter -> RateLimiter (outermost)
     * 
     * Execution flow:
     * 1. RateLimiter: ensures we don't exceed payment provider's API limits
     * 2. TimeLimiter: cancels if total time exceeds 10s
     * 3. Bulkhead: limits concurrent payment calls to 10
     * 4. CircuitBreaker: opens if failure rate > 50%
     * 5. Retry: retries up to 3 times on transient failures
     */
    @RateLimiter(name = "paymentGateway")
    @TimeLimiter(name = "paymentGateway")
    @Bulkhead(name = "paymentGateway", type = Bulkhead.Type.THREADPOOL)
    @CircuitBreaker(name = "paymentGateway", fallbackMethod = "paymentFallback")
    @Retry(name = "paymentGateway")
    public CompletableFuture<PaymentResult> chargeCustomer(ChargeRequest request) {
        return CompletableFuture.supplyAsync(() -> {
            log.info("Charging customer: {} amount: {}", 
                request.getCustomerId(), request.getAmount());
            return paymentClient.charge(request);
        });
    }

    private CompletableFuture<PaymentResult> paymentFallback(
            ChargeRequest request, Throwable t) {
        log.error("Payment failed for customer: {}, queuing for retry", 
            request.getCustomerId(), t);
        // Queue to dead letter for manual/async processing
        paymentRetryQueue.enqueue(request);
        return CompletableFuture.completedFuture(
            PaymentResult.builder()
                .status(PaymentStatus.QUEUED)
                .message("Payment will be processed shortly")
                .retryReference(UUID.randomUUID().toString())
                .build()
        );
    }
}

@Configuration
public class ResilienceHealthConfig {

    @Bean
    public HealthIndicator circuitBreakerHealthIndicator(
            CircuitBreakerRegistry registry) {
        return new CircuitBreakersHealthIndicator(registry);
    }
}