Spring Boot Actuator

@Component
public class PaymentGatewayHealthIndicator implements HealthIndicator {

    private final PaymentGatewayClient gatewayClient;
    private final CircuitBreaker circuitBreaker;

    public PaymentGatewayHealthIndicator(PaymentGatewayClient gatewayClient,
                                          CircuitBreaker circuitBreaker) {
        this.gatewayClient = gatewayClient;
        this.circuitBreaker = circuitBreaker;
    }

    @Override
    public Health health() {
        // If circuit breaker is open, don't even try — we already know it's down
        if (circuitBreaker.getState() == CircuitBreaker.State.OPEN) {
            return Health.down()
                .withDetail("reason", "Circuit breaker OPEN")
                .withDetail("provider", "Stripe")
                .withDetail("failureCount", circuitBreaker.getMetrics().getNumberOfFailedCalls())
                .withDetail("waitDuration", circuitBreaker.getCircuitBreakerConfig()
                    .getWaitDurationInOpenState().toSeconds() + "s")
                .build();
        }

        try {
            long start = System.nanoTime();
            GatewayStatus status = gatewayClient.healthCheck();
            long latencyMs = (System.nanoTime() - start) / 1_000_000;

            if (status.isHealthy() && latencyMs < 2000) {
                return Health.up()
                    .withDetail("provider", "Stripe")
                    .withDetail("latencyMs", latencyMs)
                    .withDetail("apiVersion", status.getApiVersion())
                    .build();
            } else if (status.isHealthy()) {
                // Healthy but slow — degraded
                return Health.status("DEGRADED")
                    .withDetail("provider", "Stripe")
                    .withDetail("latencyMs", latencyMs)
                    .withDetail("warning", "Response time exceeds 2000ms threshold")
                    .build();
            }
            return Health.down()
                .withDetail("provider", "Stripe")
                .withDetail("error", status.getErrorMessage())
                .build();
        } catch (Exception e) {
            return Health.down(e)
                .withDetail("provider", "Stripe")
                .build();
        }
    }
}

@Component
public class OrderQueueHealthIndicator implements HealthIndicator {

    private final KafkaConsumer<String, String> consumer;
    private static final long LAG_THRESHOLD_WARNING = 10_000;
    private static final long LAG_THRESHOLD_CRITICAL = 50_000;

    @Override
    public Health health() {
        try {
            Map<TopicPartition, Long> endOffsets = consumer.endOffsets(
                consumer.assignment());

            long totalLag = 0;
            for (TopicPartition partition : consumer.assignment()) {
                long currentOffset = consumer.position(partition);
                long endOffset = endOffsets.get(partition);
                totalLag += (endOffset - currentOffset);
            }

            Health.Builder builder = totalLag < LAG_THRESHOLD_WARNING
                ? Health.up()
                : totalLag < LAG_THRESHOLD_CRITICAL
                    ? Health.status("DEGRADED")
                    : Health.down();

            return builder
                .withDetail("consumerGroup", "order-processor")
                .withDetail("totalLag", totalLag)
                .withDetail("partitions", consumer.assignment().size())
                .withDetail("threshold_warning", LAG_THRESHOLD_WARNING)
                .withDetail("threshold_critical", LAG_THRESHOLD_CRITICAL)
                .build();
        } catch (Exception e) {
            return Health.down(e).build();
        }
    }
}

@Service
public class OrderService {

    private final Counter orderCounter;
    private final Counter failedOrderCounter;
    private final Timer orderProcessingTimer;
    private final AtomicInteger activeOrders;
    private final DistributionSummary orderValueSummary;

    public OrderService(MeterRegistry registry) {
        this.orderCounter = Counter.builder("orders.placed.total")
            .description("Total number of orders successfully placed")
            .tag("service", "order-service")
            .register(registry);

        this.failedOrderCounter = Counter.builder("orders.failed.total")
            .description("Total number of failed order attempts")
            .tag("service", "order-service")
            .register(registry);

        this.orderProcessingTimer = Timer.builder("orders.processing.duration")
            .description("Time taken to process an order end-to-end")
            .publishPercentiles(0.5, 0.95, 0.99)   // client-side percentiles
            .publishPercentileHistogram()            // server-side histogram for Prometheus
            .serviceLevelObjectives(                 // SLO buckets
                Duration.ofMillis(100),
                Duration.ofMillis(500),
                Duration.ofSeconds(1),
                Duration.ofSeconds(5)
            )
            .register(registry);

        this.activeOrders = registry.gauge("orders.active.count",
            new AtomicInteger(0));

        this.orderValueSummary = DistributionSummary.builder("orders.value.usd")
            .description("Distribution of order monetary values")
            .baseUnit("usd")
            .publishPercentiles(0.5, 0.9, 0.99)
            .scale(0.01)  // convert cents to dollars
            .register(registry);
    }

    public Order placeOrder(OrderRequest request) {
        activeOrders.incrementAndGet();
        try {
            return orderProcessingTimer.record(() -> {
                Order order = processOrder(request);
                orderCounter.increment();
                orderValueSummary.record(order.getTotalAmountCents());
                return order;
            });
        } catch (Exception e) {
            failedOrderCounter.increment();
            throw e;
        } finally {
            activeOrders.decrementAndGet();
        }
    }
}

@RestController
@RequestMapping("/api/orders")
public class OrderController {

    @Timed(value = "http.orders.create",
           description = "Order creation endpoint",
           percentiles = {0.5, 0.95, 0.99})
    @PostMapping
    public ResponseEntity<Order> createOrder(@RequestBody OrderRequest request) {
        return ResponseEntity.ok(orderService.placeOrder(request));
    }

    @Timed(value = "http.orders.search",
           percentiles = {0.5, 0.95, 0.99})
    @GetMapping("/search")
    public ResponseEntity<Page<Order>> searchOrders(OrderSearchCriteria criteria) {
        return ResponseEntity.ok(orderService.search(criteria));
    }
}

@Configuration
public class MetricsConfig {

    @Bean
    public TimedAspect timedAspect(MeterRegistry registry) {
        return new TimedAspect(registry);
    }
}

@Component
public class AppInfoContributor implements InfoContributor {

    @Override
    public void contribute(Info.Builder builder) {
        builder.withDetail("app", Map.of(
            "name", "Order Service",
            "team", "Commerce Platform",
            "oncall", "commerce-oncall@company.com",
            "runbook", "https://wiki.internal/runbooks/order-service",
            "sla", "99.95% availability, p99 < 500ms"
        ));
    }
}

@Configuration
@EnableWebSecurity
public class ActuatorSecurityConfig {

    @Bean
    @Order(1)  // higher priority than main security config
    public SecurityFilterChain actuatorSecurity(HttpSecurity http) throws Exception {
        return http
            .securityMatcher("/actuator/**")
            .authorizeHttpRequests(auth -> auth
                .requestMatchers("/actuator/health").permitAll()
                .requestMatchers("/actuator/health/**").permitAll()
                .requestMatchers("/actuator/info").permitAll()
                .requestMatchers("/actuator/prometheus").hasRole("MONITORING")
                .requestMatchers("/actuator/loggers/**").hasRole("OPS")
                .requestMatchers("/actuator/env/**").hasRole("ADMIN")
                .requestMatchers("/actuator/heapdump").hasRole("ADMIN")
                .requestMatchers("/actuator/threaddump").hasRole("ADMIN")
                .requestMatchers("/actuator/**").hasRole("ADMIN")
            )
            .httpBasic(Customizer.withDefaults())
            .csrf(csrf -> csrf.disable())  // actuator endpoints don't need CSRF
            .build();
    }
}

@Component
@Endpoint(id = "features")
public class FeatureFlagEndpoint {

    private final FeatureFlagService featureService;

    public FeatureFlagEndpoint(FeatureFlagService featureService) {
        this.featureService = featureService;
    }

    @ReadOperation
    public Map<String, FeatureStatus> getAllFeatures() {
        return featureService.getAllFlags().stream()
            .collect(Collectors.toMap(
                FeatureFlag::getName,
                f -> new FeatureStatus(f.isEnabled(), f.getRolloutPercentage(),
                    f.getLastModified())
            ));
    }

    @ReadOperation
    public FeatureStatus getFeature(@Selector String name) {
        FeatureFlag flag = featureService.getFlag(name)
            .orElseThrow(() -> new IllegalArgumentException("Unknown feature: " + name));
        return new FeatureStatus(flag.isEnabled(), flag.getRolloutPercentage(),
            flag.getLastModified());
    }

    @WriteOperation
    public void toggleFeature(@Selector String name, boolean enabled) {
        featureService.setEnabled(name, enabled);
    }

    @DeleteOperation
    public void removeFeature(@Selector String name) {
        featureService.removeFlag(name);
    }

    record FeatureStatus(boolean enabled, int rolloutPercentage, Instant lastModified) {}
}

@Component
@Endpoint(id = "circuit-breakers")
public class CircuitBreakerEndpoint {

    private final CircuitBreakerRegistry registry;

    public CircuitBreakerEndpoint(CircuitBreakerRegistry registry) {
        this.registry = registry;
    }

    @ReadOperation
    public Map<String, CircuitBreakerStatus> getAll() {
        return registry.getAllCircuitBreakers().stream()
            .collect(Collectors.toMap(
                CircuitBreaker::getName,
                cb -> new CircuitBreakerStatus(
                    cb.getState().name(),
                    cb.getMetrics().getFailureRate(),
                    cb.getMetrics().getNumberOfSuccessfulCalls(),
                    cb.getMetrics().getNumberOfFailedCalls()
                )
            ));
    }

    @WriteOperation
    public void reset(@Selector String name) {
        registry.circuitBreaker(name).reset();
    }

    record CircuitBreakerStatus(String state, float failureRate,
                                 int successCount, int failureCount) {}
}