📊 Logging, Monitoring & Distributed Tracing

@Component
@Order(Ordered.HIGHEST_PRECEDENCE)
public class CorrelationIdFilter extends OncePerRequestFilter {

    private static final String CORRELATION_ID_HEADER = "X-Correlation-ID";

    @Override
    protected void doFilterInternal(HttpServletRequest request,
                                     HttpServletResponse response,
                                     FilterChain filterChain) throws ServletException, IOException {
        try {
            String correlationId = extractOrGenerateCorrelationId(request);
            MDC.put("correlationId", correlationId);
            MDC.put("userId", extractUserId(request));

            // Propagate correlation ID in response headers
            response.setHeader(CORRELATION_ID_HEADER, correlationId);

            filterChain.doFilter(request, response);
        } finally {
            MDC.clear(); // Prevent memory leaks in thread pools
        }
    }

    private String extractOrGenerateCorrelationId(HttpServletRequest request) {
        String correlationId = request.getHeader(CORRELATION_ID_HEADER);
        return (correlationId != null && !correlationId.isBlank())
                ? correlationId
                : UUID.randomUUID().toString();
    }

    private String extractUserId(HttpServletRequest request) {
        // Extract from JWT or security context
        Authentication auth = SecurityContextHolder.getContext().getAuthentication();
        return auth != null ? auth.getName() : "anonymous";
    }
}

@Configuration
public class RestClientConfig {

    @Bean
    public RestClient restClient(RestClient.Builder builder) {
        return builder
            .requestInterceptor((request, body, execution) -> {
                // Propagate correlation ID from MDC to outgoing requests
                String correlationId = MDC.get("correlationId");
                if (correlationId != null) {
                    request.getHeaders().set("X-Correlation-ID", correlationId);
                }
                return execution.execute(request, body);
            })
            .build();
    }
}

@Service
@RequiredArgsConstructor
public class OrderService {

    private static final Logger log = LoggerFactory.getLogger(OrderService.class);
    private final ObservationRegistry observationRegistry;
    private final PaymentClient paymentClient;
    private final InventoryClient inventoryClient;

    public Order createOrder(CreateOrderRequest request) {
        return Observation.createNotStarted("order.create", observationRegistry)
            .lowCardinalityKeyValue("order.type", request.getType().name())
            .highCardinalityKeyValue("order.id", request.getOrderId())
            .observe(() -> {
                log.info("Creating order for customer={}, items={}",
                    request.getCustomerId(), request.getItems().size());

                // Each call creates a child span automatically
                InventoryResult inventory = inventoryClient.reserve(request.getItems());
                if (!inventory.isAvailable()) {
                    log.warn("Insufficient inventory for order={}", request.getOrderId());
                    throw new InsufficientInventoryException(request.getOrderId());
                }

                PaymentResult payment = paymentClient.charge(request.getPaymentDetails());
                if (!payment.isSuccess()) {
                    log.error("Payment failed for order={}, reason={}",
                        request.getOrderId(), payment.getDeclineReason());
                    inventoryClient.release(request.getItems());
                    throw new PaymentFailedException(payment.getDeclineReason());
                }

                Order order = Order.create(request, inventory, payment);
                log.info("Order created successfully orderId={}, total={}",
                    order.getId(), order.getTotal());
                return order;
            });
    }
}

@Configuration
@EnableAsync
public class AsyncConfig implements AsyncConfigurer {

    @Override
    public Executor getAsyncExecutor() {
        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
        executor.setCorePoolSize(10);
        executor.setMaxPoolSize(50);
        executor.setQueueCapacity(100);
        executor.setThreadNamePrefix("async-");
        executor.initialize();

        // Wrap executor to propagate trace context to async threads
        return ContextExecutorService.wrap(executor.getThreadPoolExecutor());
    }
}

@Service
@RequiredArgsConstructor
public class PaymentService {

    private static final Logger log = LoggerFactory.getLogger(PaymentService.class);
    private final MeterRegistry meterRegistry;
    private final PaymentGateway paymentGateway;

    private final Counter paymentSuccessCounter;
    private final Counter paymentFailureCounter;
    private final Timer paymentProcessingTimer;

    public PaymentService(MeterRegistry meterRegistry, PaymentGateway paymentGateway) {
        this.meterRegistry = meterRegistry;
        this.paymentGateway = paymentGateway;

        this.paymentSuccessCounter = Counter.builder("payments.processed")
            .tag("status", "success")
            .description("Number of successful payments")
            .register(meterRegistry);

        this.paymentFailureCounter = Counter.builder("payments.processed")
            .tag("status", "failure")
            .description("Number of failed payments")
            .register(meterRegistry);

        this.paymentProcessingTimer = Timer.builder("payments.duration")
            .description("Payment processing duration")
            .publishPercentiles(0.5, 0.95, 0.99)
            .register(meterRegistry);
    }

    public PaymentResult processPayment(PaymentRequest request) {
        return paymentProcessingTimer.record(() -> {
            try {
                PaymentResult result = paymentGateway.charge(request);
                paymentSuccessCounter.increment();

                // Track payment amount as distribution summary
                DistributionSummary.builder("payments.amount")
                    .tag("currency", request.getCurrency())
                    .register(meterRegistry)
                    .record(request.getAmount().doubleValue());

                log.info("Payment processed successfully paymentId={}, amount={}",
                    result.getPaymentId(), request.getAmount());
                return result;

            } catch (PaymentGatewayException e) {
                paymentFailureCounter.increment();
                meterRegistry.counter("payments.errors",
                    "type", e.getErrorCode(),
                    "gateway", request.getGateway()
                ).increment();

                log.error("Payment processing failed paymentId={}, error={}",
                    request.getPaymentId(), e.getErrorCode(), e);
                throw e;
            }
        });
    }

    // Gauge for active payment sessions (current state)
    @Scheduled(fixedRate = 30000)
    public void reportActivePayments() {
        Gauge.builder("payments.active_sessions", paymentGateway, PaymentGateway::getActiveSessions)
            .description("Currently active payment sessions")
            .register(meterRegistry);
    }
}

@Component
@RequiredArgsConstructor
public class SystemMetrics {

    private final MeterRegistry registry;
    private final DataSource dataSource;
    private final ThreadPoolTaskExecutor taskExecutor;

    @PostConstruct
    public void registerMetrics() {
        // Connection pool utilization (USE - Utilization)
        Gauge.builder("db.pool.active", dataSource, ds -> {
            HikariDataSource hds = (HikariDataSource) ds;
            return hds.getHikariPoolMXBean().getActiveConnections();
        }).register(registry);

        // Thread pool saturation (USE - Saturation)
        Gauge.builder("threadpool.queue.size", taskExecutor,
            e -> e.getThreadPoolExecutor().getQueue().size())
            .register(registry);

        // Thread pool utilization
        Gauge.builder("threadpool.active", taskExecutor,
            e -> e.getThreadPoolExecutor().getActiveCount())
            .register(registry);

        // JVM memory utilization
        Gauge.builder("jvm.heap.utilization", () ->
            (double) Runtime.getRuntime().totalMemory() / Runtime.getRuntime().maxMemory())
            .register(registry);
    }
}

// BAD: Logging sensitive data
log.info("User login: username={}, password={}", username, password);

// BAD: Logging inside tight loops
for (Item item : items) {
    log.debug("Processing item: {}", item);  // Could generate millions of logs
}

// BAD: String concatenation (evaluates even if level disabled)
log.debug("Order details: " + order.toString());

// GOOD: Parameterized logging (lazy evaluation)
log.debug("Order details: {}", order);

// GOOD: Guard expensive computations
if (log.isDebugEnabled()) {
    log.debug("Full payload: {}", objectMapper.writeValueAsString(largeObject));
}

// GOOD: Structured key-value pairs for searchability
log.info("Order processed orderId={} customerId={} total={} items={}",
    order.getId(), order.getCustomerId(), order.getTotal(), order.getItemCount());

@Component
public class DownstreamHealthIndicator implements HealthIndicator {

    private final RestClient restClient;
    private final MeterRegistry meterRegistry;
    private static final Logger log = LoggerFactory.getLogger(DownstreamHealthIndicator.class);

    @Override
    public Health health() {
        Timer.Sample sample = Timer.start(meterRegistry);
        try {
            ResponseEntity<Void> response = restClient.get()
                .uri("/actuator/health")
                .retrieve()
                .toBodilessEntity();

            sample.stop(Timer.builder("health.check.duration")
                .tag("target", "payment-service")
                .tag("status", "UP")
                .register(meterRegistry));

            return Health.up()
                .withDetail("payment-service", "reachable")
                .build();

        } catch (Exception e) {
            sample.stop(Timer.builder("health.check.duration")
                .tag("target", "payment-service")
                .tag("status", "DOWN")
                .register(meterRegistry));

            log.error("Health check failed for payment-service", e);
            return Health.down()
                .withDetail("payment-service", e.getMessage())
                .build();
        }
    }
}