Prometheus Grafana 监控实战构建可观测性平台别叫我大神叫我 Alex 就好。没有监控的系统就像蒙着眼睛开车你不知道前面是坦途还是悬崖。一、Spring Boot 应用埋点1.1 Micrometer 配置Configuration public class MetricsConfig { Bean MeterRegistryCustomizerMeterRegistry metricsCommonTags() { return registry - registry.config() .commonTags(application, myapp) .commonTags(team, platform) .commonTags(environment, ${ENV:development}); } Bean public TimedAspect timedAspect(MeterRegistry registry) { return new TimedAspect(registry); } Bean public CountedAspect countedAspect(MeterRegistry registry) { return new CountedAspect(registry); } } Service public class OrderService { private final Counter orderCounter; private final Timer orderProcessingTimer; private final DistributionSummary orderAmountSummary; public OrderService(MeterRegistry registry) { this.orderCounter Counter.builder(orders.created) .description(Total number of orders created) .register(registry); this.orderProcessingTimer Timer.builder(orders.processing.time) .description(Order processing time) .publishPercentiles(0.5, 0.95, 0.99) .register(registry); this.orderAmountSummary DistributionSummary.builder(orders.amount) .description(Order amount distribution) .baseUnit(yuan) .publishPercentiles(0.5, 0.95, 0.99) .register(registry); } Timed(value orders.create, description Time taken to create order) public Order createOrder(OrderRequest request) { return orderProcessingTimer.record(() - { // 创建订单逻辑 Order order doCreateOrder(request); // 记录指标 orderCounter.increment(); orderAmountSummary.record(order.getTotalAmount().doubleValue()); return order; }); } Counted(value orders.cancelled, description Total cancelled orders) public void cancelOrder(String orderId) { // 取消订单逻辑 } }1.2 自定义指标Component public class BusinessMetrics { Autowired private MeterRegistry registry; // gauges - 实时值 public void registerUserGauge(UserRepository repository) { Gauge.builder(users.total, repository, UserRepository::count) .description(Total number of users) .register(registry); Gauge.builder(users.active, repository, r - r.countByStatus(ACTIVE)) .description(Number of active users) .register(registry); } // 多维度计数器 public void recordApiCall(String endpoint, String method, int statusCode) { Counter.builder(api.calls) .tag(endpoint, endpoint) .tag(method, method) .tag(status, String.valueOf(statusCode)) .register(registry) .increment(); } // 长任务计时器 public LongTaskTimer createLongTaskTimer(String taskName) { return LongTaskTimer.builder(tasks.long) .tag(task, taskName) .register(registry); } }二、Prometheus 配置2.1 Prometheus 配置文件# prometheus.yml global: scrape_interval: 15s evaluation_interval: 15s external_labels: cluster: production replica: {{.ExternalURL}} alerting: alertmanagers: - static_configs: - targets: - alertmanager:9093 rule_files: - /etc/prometheus/rules/*.yml scrape_configs: # Prometheus 自身监控 - job_name: prometheus static_configs: - targets: [localhost:9090] # Spring Boot 应用 - job_name: spring-boot-apps metrics_path: /actuator/prometheus kubernetes_sd_configs: - role: pod namespaces: names: - default - production relabel_configs: - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] action: keep regex: true - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] action: replace target_label: __metrics_path__ regex: (.) - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] action: replace regex: ([^:])(?::\d)?;(\d) replacement: $1:$2 target_label: __address__ - action: labelmap regex: __meta_kubernetes_pod_label_(.) - source_labels: [__meta_kubernetes_namespace] action: replace target_label: kubernetes_namespace - source_labels: [__meta_kubernetes_pod_name] action: replace target_label: kubernetes_pod_name # Node Exporter - job_name: node-exporter kubernetes_sd_configs: - role: node relabel_configs: - action: labelmap regex: __meta_kubernetes_node_label_(.) # JVM 监控 - job_name: jmx-exporter static_configs: - targets: [app:9090]2.2 告警规则# alert-rules.yml groups: - name: application-alerts rules: # 高错误率告警 - alert: HighErrorRate expr: | ( sum(rate(http_server_requests_seconds_count{status~5..}[5m])) / sum(rate(http_server_requests_seconds_count[5m])) ) 0.05 for: 5m labels: severity: critical annotations: summary: High error rate detected description: Error rate is {{ $value | humanizePercentage }} for the last 5 minutes # 响应时间告警 - alert: HighLatency expr: | histogram_quantile(0.95, sum(rate(http_server_requests_seconds_bucket[5m])) by (le, uri) ) 2 for: 10m labels: severity: warning annotations: summary: High latency on {{ $labels.uri }} description: 95th percentile latency is {{ $value }}s # JVM 内存告警 - alert: JvmMemoryHigh expr: | ( jvm_memory_used_bytes{areaheap} / jvm_memory_max_bytes{areaheap} ) 0.85 for: 5m labels: severity: warning annotations: summary: JVM heap memory usage is high description: Heap memory usage is {{ $value | humanizePercentage }} # 应用宕机告警 - alert: ApplicationDown expr: up{jobspring-boot-apps} 0 for: 1m labels: severity: critical annotations: summary: Application {{ $labels.instance }} is down description: Application has been down for more than 1 minute # 业务指标告警 - alert: OrderProcessingSlow expr: | rate(orders_processing_time_seconds_sum[5m]) / rate(orders_processing_time_seconds_count[5m]) 5 for: 10m labels: severity: warning annotations: summary: Order processing is slow description: Average processing time is {{ $value }}s三、Grafana 仪表盘3.1 JVM 监控仪表盘{ dashboard: { title: JVM Metrics, panels: [ { title: Heap Memory Usage, type: timeseries, targets: [ { expr: jvm_memory_used_bytes{area\heap\}, legendFormat: Used }, { expr: jvm_memory_max_bytes{area\heap\}, legendFormat: Max } ], fieldConfig: { defaults: { unit: bytes, min: 0 } } }, { title: GC Duration, type: timeseries, targets: [ { expr: rate(jvm_gc_pause_seconds_sum[5m]), legendFormat: {{action}} ({{cause}}) } ], fieldConfig: { defaults: { unit: s } } }, { title: Thread Count, type: stat, targets: [ { expr: jvm_threads_live_threads, legendFormat: Live Threads }, { expr: jvm_threads_daemon_threads, legendFormat: Daemon Threads } ] }, { title: Class Loading, type: timeseries, targets: [ { expr: jvm_classes_loaded_classes, legendFormat: Loaded }, { expr: rate(jvm_classes_unloaded_classes_total[5m]), legendFormat: Unloaded/sec } ] } ] } }3.2 业务指标仪表盘{ dashboard: { title: Business Metrics, panels: [ { title: Order Rate, type: timeseries, targets: [ { expr: rate(orders_created_total[5m]), legendFormat: Orders/sec } ] }, { title: Order Processing Time, type: heatmap, targets: [ { expr: rate(orders_processing_time_seconds_bucket[5m]), format: heatmap } ] }, { title: Revenue, type: stat, targets: [ { expr: sum(increase(orders_amount_sum[1h])), legendFormat: Last Hour }, { expr: sum(increase(orders_amount_sum[24h])), legendFormat: Last 24h } ], fieldConfig: { defaults: { unit: currencyCNY } } }, { title: API Error Rate, type: timeseries, targets: [ { expr: | sum(rate(http_server_requests_seconds_count{status~5..}[5m])) / sum(rate(http_server_requests_seconds_count[5m])) , legendFormat: Error Rate } ], fieldConfig: { defaults: { unit: percentunit, max: 1 } } } ] } }四、分布式追踪4.1 OpenTelemetry 配置Configuration public class TracingConfig { Bean public OpenTelemetry openTelemetry() { Resource resource Resource.getDefault() .merge(Resource.create(Attributes.of( ResourceAttributes.SERVICE_NAME, myapp, ResourceAttributes.SERVICE_VERSION, 1.0.0, ResourceAttributes.DEPLOYMENT_ENVIRONMENT, production ))); // OTLP Exporter OtlpGrpcSpanExporter spanExporter OtlpGrpcSpanExporter.builder() .setEndpoint(http://otel-collector:4317) .setTimeout(30, TimeUnit.SECONDS) .build(); SdkTracerProvider tracerProvider SdkTracerProvider.builder() .addSpanProcessor(BatchSpanProcessor.builder(spanExporter).build()) .setResource(resource) .build(); return OpenTelemetrySdk.builder() .setTracerProvider(tracerProvider) .buildAndRegisterGlobal(); } Bean public Tracer tracer(OpenTelemetry openTelemetry) { return openTelemetry.getTracer(myapp, 1.0.0); } } Service public class TracedOrderService { Autowired private Tracer tracer; public Order createOrder(OrderRequest request) { Span span tracer.spanBuilder(create-order) .setSpanKind(SpanKind.INTERNAL) .startSpan(); try (Scope scope span.makeCurrent()) { span.setAttribute(user.id, request.getUserId()); span.setAttribute(order.amount, request.getTotalAmount().doubleValue()); // 验证库存 validateInventory(request.getItems()); // 创建订单 Order order saveOrder(request); span.setAttribute(order.id, order.getId()); span.setStatus(StatusCode.OK); return order; } catch (Exception e) { span.setStatus(StatusCode.ERROR, e.getMessage()); span.recordException(e); throw e; } finally { span.end(); } } }五、日志聚合5.1 Loki 日志配置# promtail-config.yml server: http_listen_port: 9080 grpc_listen_port: 0 positions: filename: /tmp/positions.yaml clients: - url: http://loki:3100/loki/api/v1/push scrape_configs: - job_name: spring-boot-logs static_configs: - targets: - localhost labels: job: spring-boot __path__: /var/log/myapp/*.log pipeline_stages: - json: expressions: timestamp: timestamp level: level message: message traceId: traceId spanId: spanId - timestamp: source: timestamp format: RFC3339 - labels: level: traceId:5.2 结构化日志Configuration public class LoggingConfig { Bean public Logger logger() { LoggerContext context (LoggerContext) LoggerFactory.getILoggerFactory(); // JSON 格式 EncoderILoggingEvent encoder new LogstashEncoder(); encoder.setContext(context); encoder.start(); // 控制台输出 ConsoleAppenderILoggingEvent consoleAppender new ConsoleAppender(); consoleAppender.setEncoder(encoder); consoleAppender.setContext(context); consoleAppender.start(); Logger rootLogger context.getLogger(Logger.ROOT_LOGGER_NAME); rootLogger.addAppender(consoleAppender); return rootLogger; } } Component public class StructuredLogger { private static final Logger logger LoggerFactory.getLogger(StructuredLogger.class); public void logOrderCreated(Order order, TraceContext traceContext) { logger.info(Order created, StructuredArguments.keyValue(orderId, order.getId()), StructuredArguments.keyValue(userId, order.getUserId()), StructuredArguments.keyValue(amount, order.getTotalAmount()), StructuredArguments.keyValue(traceId, traceContext.traceId()), StructuredArguments.keyValue(spanId, traceContext.spanId()) ); } }六、总结构建可观测性平台的核心要素指标监控业务指标 系统指标日志聚合结构化日志 集中存储分布式追踪请求链路追踪告警管理及时发现问题可视化直观的仪表盘这其实可以更优雅一点。可观测性不是附加功能而是系统设计的核心部分。参考资源Prometheus DocumentationGrafana DocumentationOpenTelemetryMicrometer