Наблюдаемость приложений ASP.NET Core с OpenTelemetry, Prometheus и Grafana

dotnet add package OpenTelemetry.Extensions.Hosting
dotnet add package OpenTelemetry.Instrumentation.AspNetCore
dotnet add package OpenTelemetry.Exporter.Console

var builder = WebApplication.CreateBuilder(args);
 
// Добавляем OpenTelemetry
var openTelemetryBuilder = builder.Services.AddOpenTelemetry();
 
// Настраиваем ресурс (информацию о сервисе)
openTelemetryBuilder.ConfigureResource(resource => resource
    .AddService(builder.Environment.ApplicationName));
 
// Настраиваем метрики
openTelemetryBuilder.WithMetrics(metrics => metrics
    .AddAspNetCoreInstrumentation()
    .AddConsoleExporter());
 
var app = builder.Build();
// ...остальная конфигурация

[Trace("ImportantBusinessOperation")]
public async Task<Result> ProcessOrder(Order order)
{
    // логика обработки заказа
}

// Создаем трейсер
using var activity = _activitySource.StartActivity("ProcessPayment");
 
try
{
    // Добавляем атрибуты к трейсу
    activity?.SetTag("payment.amount", payment.Amount);
    activity?.SetTag("payment.method", payment.Method);
    
    // Выполняем бизнес-операцию
    var result = await _paymentGateway.ProcessAsync(payment);
    
    // Отмечаем успешное завершение
    activity?.SetStatus(ActivityStatusCode.Ok);
    
    return result;
}
catch (Exception ex)
{
    // Фиксируем ошибку в трейсе
    activity?.SetStatus(ActivityStatusCode.Error, ex.Message);
    throw;
}

services.AddOpenTelemetry().WithTracing(builder => builder
    .SetSampler(new TraceIdRatioBasedSampler(0.1)) // собираем только 10% трейсов
    .AddAspNetCoreInstrumentation()
    .AddOtlpExporter());

public class PrioritySampler : Sampler
{
    public override SamplingResult ShouldSample(in SamplingParameters parameters)
    {
        // Анализируем путь запроса или другие атрибуты
        if (parameters.Name.Contains("Payment") || parameters.Name.Contains("Register"))
        {
            return new SamplingResult(SamplingDecision.RecordAndSample);
        }
        
        // Для остальных запросов используем выборочное семплирование
        return new Random().NextDouble() < 0.05 
            ? new SamplingResult(SamplingDecision.RecordAndSample)
            : new SamplingResult(SamplingDecision.Drop);
    }
}

services.AddOpenTelemetry().WithTracing(builder => builder
    .AddAspNetCoreInstrumentation()
    .AddHttpClientInstrumentation()
    .AddOtlpExporter());

// При отправке сообщения
var currentActivity = Activity.Current;
if (currentActivity != null)
{
    // Добавляем контекст в метаданные сообщения
    message.Properties.Add("TraceId", currentActivity.TraceId.ToString());
    message.Properties.Add("SpanId", currentActivity.SpanId.ToString());
}
 
// При получении сообщения
var traceIdStr = message.Properties["TraceId"] as string;
var parentSpanIdStr = message.Properties["SpanId"] as string;
 
if (traceIdStr != null && parentSpanIdStr != null)
{
    var traceId = ActivityTraceId.CreateFromString(traceIdStr);
    var parentSpanId = ActivitySpanId.CreateFromString(parentSpanIdStr);
    
    // Создаем новую активность с родительским контекстом
    using var activity = _activitySource.StartActivity(
        "ProcessMessage",
        ActivityKind.Consumer,
        new ActivityContext(traceId, parentSpanId, ActivityTraceFlags.Recorded));
    
    // Дальнейшая обработка сообщения
}

dotnet add package OpenTelemetry.Exporter.Prometheus.AspNetCore

openTelemetryBuilder.WithMetrics(metrics => metrics
  .AddAspNetCoreInstrumentation()
  .AddPrometheusExporter());
 
// После вызова app.Build()
app.MapPrometheusScrapingEndpoint();

# HELP http_server_request_duration_seconds Duration of HTTP request.
# TYPE http_server_request_duration_seconds histogram
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.005"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.01"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.025"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.05"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.075"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.1"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.25"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.5"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="0.75"} 0
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="1"} 2
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="2.5"} 3
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="5"} 3
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="7.5"} 3
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="10"} 3
http_server_request_duration_seconds_bucket{http_route="api/Products",http_response_status_code="200",le="+Inf"} 3
http_server_request_duration_seconds_sum{http_route="api/Products",http_response_status_code="200"} 2.95
http_server_request_duration_seconds_count{http_route="api/Products",http_response_status_code="200"} 3

public class ProductsMetrics : IProductsMetrics
{
  private const string NAME = "Get.Products";
  private readonly Counter<long> _getProductsErrorCounter;
 
  public ProductsMetrics()
  {
      var getProdutcsErrorMeter = new Meter(NAME);
      _getProductsErrorCounter = getProdutcsErrorMeter.CreateCounter<long>("get.products.error.count");
  }
 
  public static string Name => NAME;
 
  public void RegisterError(GetProductError getProductError)
  {
      var tags = new KeyValuePair<string, object?>(nameof(getProductError), getProductError.ToString());
      _getProductsErrorCounter.Add(1, tags);
  }
}
 
public enum GetProductError
{
  QueryProducts,
  FillPrices,
  FillAvailability,
}

try
{
    // Бизнес-логика
}
catch (Exception)
{
    _productsMetrics.RegisterError(GetProductError.FillPrices);
    throw;
}

var customMetricsNames = new string[] { ProductsMetrics.Name };
 
openTelemetryBuilder.WithMetrics(metrics => metrics
  .AddMeter(customMetricsNames)
  .AddAspNetCoreInstrumentation()
  .AddPrometheusExporter());

monitoring-stack/
  ├── docker-compose.yml
  ├── prometheus/
  │   └── prometheus.yml
  ├── grafana/
  │   ├── datasources/
  │   │   └── prometheus.yml
  │   └── dashboards/
  │       ├── dashboard.yml
  │       └── api-dashboard.json
  └── application/
      └── Dockerfile

dotnet add package Serilog.AspNetCore
dotnet add package Serilog.Sinks.Elasticsearch

var builder = WebApplication.CreateBuilder(args);
 
builder.Host.UseSerilog((context, configuration) => 
    configuration
        .ReadFrom.Configuration(context.Configuration)
        .Enrich.FromLogContext()
        .Enrich.WithProperty("ApplicationName", builder.Environment.ApplicationName)
        .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://elasticsearch:9200"))
        {
            AutoRegisterTemplate = true,
            IndexFormat = $"{builder.Environment.ApplicationName.ToLower()}-{DateTime.UtcNow:yyyy-MM}"
        }));

.Enrich.With<ActivityEnricher>()

public class ActivityEnricher : ILogEventEnricher
{
    public void Enrich(LogEvent logEvent, ILogEventPropertyFactory propertyFactory)
    {
        var activity = Activity.Current;
        
        if (activity != null)
        {
            logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty(
                "TraceId", activity.TraceId.ToString()));
                
            logEvent.AddPropertyIfAbsent(propertyFactory.CreateProperty(
                "SpanId", activity.SpanId.ToString()));
        }
    }
}

// Program.cs для каждого микросервиса
var builder = WebApplication.CreateBuilder(args);
 
// Настройка OpenTelemetry
var serviceName = builder.Environment.ApplicationName;
builder.Services.AddOpenTelemetry()
    .ConfigureResource(r => r.AddService(serviceName))
    .WithTracing(tracing => tracing
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddEntityFrameworkCoreInstrumentation()
        .AddSource(serviceName)
        .AddOtlpExporter(opts => opts.Endpoint = new Uri("http://otel-collector:4317")))
    .WithMetrics(metrics => metrics
        .AddAspNetCoreInstrumentation()
        .AddHttpClientInstrumentation()
        .AddRuntimeInstrumentation()
        .AddOtlpExporter(opts => opts.Endpoint = new Uri("http://otel-collector:4317")));
 
// Настройка Serilog для интеграции с ELK
builder.Host.UseSerilog((ctx, cfg) => cfg
    .ReadFrom.Configuration(ctx.Configuration)
    .Enrich.WithProperty("Service", serviceName)
    .Enrich.With<ActivityEnricher>()
    .WriteTo.Elasticsearch(new ElasticsearchSinkOptions(new Uri("http://elasticsearch:9200")) {
        IndexFormat = $"logs-{serviceName.ToLower()}-{DateTime.UtcNow:yyyy.MM}"
    }));