feat: improve kute tool prometheus metrics

tools/Kute/Nethermind.Tools.Kute.Test/MetricsTests.cs

@@ -11,9 +11,9 @@ namespace Nethermind.Tools.Kute.Test;
 
 public class MetricsTests
 {
-    private static JsonRpc.Request.Single Single(int id)
+    private static JsonRpc.Request.Single Single(int id, string method = "test")
     {
-        var json = $$"""{ "id": {{id}}, "method": "test", "params": [] }""";
+        var json = $$"""{ "id": {{id}}, "method": "{{method}}", "params": [] }""";
         return new JsonRpc.Request.Single(JsonNode.Parse(json)!);
     }
 
@@ -31,7 +31,7 @@ public async Task MemoryMetricsReporter_GeneratesValidReport()
         var totalTimer = new Timer();
         using (totalTimer.Time())
         {
-            var single = Single(42);
+            var single = Single(42, "method1");
             var batch = Batch(Single(43), Single(44), Single(45));
 
             var singleTimer = new Timer();
@@ -57,7 +57,9 @@ public async Task MemoryMetricsReporter_GeneratesValidReport()
         report.TotalTime.Should().BeLessThan(TimeSpan.FromMilliseconds(110));
 
         report.Singles.Should().HaveCount(1);
-        report.Singles.Should().ContainKey("42");
+        report.Singles.Should().ContainKey("method1");
+        report.Singles["method1"].Should().HaveCount(1);
+        report.Singles["method1"].Should().ContainKey("42");
 
         report.Batches.Should().HaveCount(1);
         report.Batches.Should().ContainKey("43:45");

tools/Kute/Nethermind.Tools.Kute/Config.cs

@@ -77,8 +77,43 @@ public static class Config
         HelpName = "url",
     };
 
+    public static Option<string?> PrometheusPushGatewayUser { get; } = new("--gateway-user")
+    {
+        Description = "Prometheus Push Gateway username",
+        HelpName = "username",
+    };
+
+    public static Option<string?> PrometheusPushGatewayPassword { get; } = new("--gateway-pass")
+    {
+        Description = "Prometheus Push Gateway password",
+        HelpName = "password",
+    };
+
     public static Option<bool> UnwrapBatch { get; } = new("--unwrapBatch", "-u")
     {
         Description = "Batch requests will be unwraped to single requests"
     };
+
+    public static Option<Dictionary<string, string>> Labels { get; } = new("--labels", "-l")
+    {
+        DefaultValueFactory = r => new Dictionary<string, string>(),
+        CustomParser = r =>
+        {
+            var labels = new Dictionary<string, string>();
+            foreach (var token in r.Tokens)
+            {
+                foreach (var pair in token.Value.Split(','))
+                {
+                    var parts = pair.Split('=', 2);
+                    if (parts.Length == 2)
+                    {
+                        labels.Add(parts[0], parts[1]);
+                    }
+                }
+            }
+            return labels;
+        },
+        Description = "A comma separated list of tags to be added to the metrics in the format key=value",
+        HelpName = "labels",
+    };
 }

tools/Kute/Nethermind.Tools.Kute/Metrics/MemoryMetricsReporter.cs

@@ -9,7 +9,7 @@ public sealed class MemoryMetricsReporter
     : IMetricsReporter
     , IMetricsReportProvider
 {
-    private readonly ConcurrentDictionary<string, TimeSpan> _singles = new();
+    private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, TimeSpan>> _singles = new();
     private readonly ConcurrentDictionary<string, TimeSpan> _batches = new();
 
     private TimeSpan _totalRunningTime;
@@ -36,10 +36,19 @@ public Task Batch(JsonRpc.Request.Batch batch, TimeSpan elapsed, CancellationTok
     }
     public Task Single(JsonRpc.Request.Single single, TimeSpan elapsed, CancellationToken token = default)
     {
-        var id = single.Id?.ToString();
-        if (id is not null)
+        var id = single.Id;
+        var methodName = single.MethodName;
+        if (id is not null && methodName is not null)
         {
-            _singles[id] = elapsed;
+            var newMethodDict = new ConcurrentDictionary<string, TimeSpan>();
+            newMethodDict[id] = elapsed;
+            _singles.AddOrUpdate(methodName,
+            newMethodDict,
+            (_, dict) =>
+            {
+                dict[id] = elapsed;
+                return dict;
+            });
         }
 
         return Task.CompletedTask;
@@ -60,7 +69,7 @@ public MetricsReport Report()
             Ignored = _ignored,
             Responses = _responses,
             TotalTime = _totalRunningTime,
-            Singles = _singles.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
+            Singles = _singles.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyDictionary<string, TimeSpan>)kvp.Value.ToDictionary(ikvp => ikvp.Key, ikvp => ikvp.Value)),
             Batches = _batches.ToDictionary(kvp => kvp.Key, kvp => kvp.Value),
         };
     }

tools/Kute/Nethermind.Tools.Kute/Metrics/MetricsReport.cs

@@ -54,13 +54,13 @@ public sealed record MetricsReport
     public required long Ignored { get; init; }
     public required long Responses { get; init; }
     public required TimeSpan TotalTime { get; init; }
-    public required IReadOnlyDictionary<string, TimeSpan> Singles { get; init; }
+    public required IReadOnlyDictionary<string, IReadOnlyDictionary<string, TimeSpan>> Singles { get; init; }
     public required IReadOnlyDictionary<string, TimeSpan> Batches { get; init; }
 
     // Computed properties
-    private TimeMetrics? _singlesMetrics;
+    private Dictionary<string, TimeMetrics>? _singlesMetrics;
     private TimeMetrics? _batchesMetrics;
-    public TimeMetrics SinglesMetrics => _singlesMetrics ??= TimeMetrics.From(Singles.Values.ToList());
+    public Dictionary<string, TimeMetrics> SinglesMetrics => _singlesMetrics ??= Singles.ToDictionary(kvp => kvp.Key, kvp => TimeMetrics.From(kvp.Value.Values.ToList()));
     public TimeMetrics BatchesMetrics => _batchesMetrics ??= TimeMetrics.From(Batches.Values.ToList());
 }
 

tools/Kute/Nethermind.Tools.Kute/Metrics/PrettyMetricsReportFormatter.cs

@@ -16,11 +16,18 @@ public async Task WriteAsync(Stream stream, MetricsReport report, CancellationTo
         await writer.WriteLineAsync($"Ignored: {report.Ignored}", token);
         await writer.WriteLineAsync($"Responses: {report.Responses}\n", token);
         await writer.WriteLineAsync("Singles:", token);
-        await writer.WriteLineAsync($"  Count: {report.Singles.Count}", token);
-        await writer.WriteLineAsync($"  Max: {report.SinglesMetrics.Max.TotalMilliseconds} ms", token);
-        await writer.WriteLineAsync($"  Average: {report.SinglesMetrics.Average.TotalMilliseconds} ms", token);
-        await writer.WriteLineAsync($"  Min: {report.SinglesMetrics.Min.TotalMilliseconds} ms", token);
-        await writer.WriteLineAsync($"  Stddev: {report.SinglesMetrics.StandardDeviation.TotalMilliseconds} ms", token);
+        foreach (var single in report.SinglesMetrics)
+        {
+            var methodName = single.Key;
+            var metrics = single.Value;
+            await writer.WriteLineAsync($"  {methodName}:", token);
+            await writer.WriteLineAsync($"    Count: {report.Singles[methodName].Count}", token);
+            await writer.WriteLineAsync($"    Max: {metrics.Max.TotalMilliseconds} ms", token);
+            await writer.WriteLineAsync($"    Average: {metrics.Average.TotalMilliseconds} ms", token);
+            await writer.WriteLineAsync($"    Min: {metrics.Min.TotalMilliseconds} ms", token);
+            await writer.WriteLineAsync($"    Stddev: {metrics.StandardDeviation.TotalMilliseconds} ms", token);
+        }
+
         await writer.WriteLineAsync("Batches:", token);
         await writer.WriteLineAsync($"  Count: {report.Batches.Count}", token);
         await writer.WriteLineAsync($"  Max: {report.BatchesMetrics.Max.TotalMilliseconds} ms", token);

tools/Kute/Nethermind.Tools.Kute/Metrics/PrometheusPushGatewayMetricsReporter.cs

@@ -1,6 +1,8 @@
 // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Net.Http.Headers;
+using System.Text;
 using Prometheus.Client;
 using Prometheus.Client.Collectors;
 using Prometheus.Client.MetricPusher;
@@ -18,29 +20,43 @@ public sealed class PrometheusPushGatewayMetricsReporter : IMetricsReporter
     private readonly ICounter _failedCounter;
     private readonly ICounter _ignoredCounter;
     private readonly ICounter _responseCounter;
-    private readonly IMetricFamily<IHistogram, ValueTuple<string>> _singleDuration;
-    private readonly IMetricFamily<IHistogram, ValueTuple<string>> _batchDuration;
-
-    public PrometheusPushGatewayMetricsReporter(string endpoint)
+    private readonly IMetricFamily<IHistogram> _singleDuration;
+    private readonly IMetricFamily<IHistogram> _batchDuration;
+
+    public PrometheusPushGatewayMetricsReporter(
+        string endpoint,
+        Dictionary<string, string> labels,
+        string? user,
+        string? password
+    )
     {
         var registry = new CollectorRegistry();
         var factory = new MetricFactory(registry);
 
-        _messageCounter = factory.CreateCounter("messages", "");
-        _succeededCounter = factory.CreateCounter("succeeded", "");
-        _failedCounter = factory.CreateCounter("failed", "");
-        _ignoredCounter = factory.CreateCounter("ignored", "");
-        _responseCounter = factory.CreateCounter("responses", "");
-        _singleDuration = factory.CreateHistogram("single_duration", "", labelName: "jsonrpc_id");
-        _batchDuration = factory.CreateHistogram("batch_duration", "", labelName: "jsonrpc_id");
+        _messageCounter = factory.CreateCounter(GetMetricName("messages_total"), "");
+        _succeededCounter = factory.CreateCounter(GetMetricName("messages_succeeded"), "");
+        _failedCounter = factory.CreateCounter(GetMetricName("messages_failed"), "");
+        _ignoredCounter = factory.CreateCounter(GetMetricName("messages_ignored"), "");
+        _responseCounter = factory.CreateCounter(GetMetricName("responses_total"), "");
+        _singleDuration = factory.CreateHistogram(GetMetricName("single_duration_seconds"), "", labelNames: new[] { "jsonrpc_id", "method" });
+        _batchDuration = factory.CreateHistogram(GetMetricName("batch_duration_seconds"), "", labelNames: new[] { "jsonrpc_id" });
 
         _endpoint = endpoint;
+        string instanceLabel = labels.TryGetValue("instance", out var instance) ? instance : Guid.NewGuid().ToString();
+        labels.Remove("instance");
+        var httpClient = new HttpClient();
+        if (user is not null && password is not null)
+        {
+            httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{password}")));
+        }
         _pusher = new MetricPusher(new MetricPusherOptions
         {
             CollectorRegistry = registry,
             Endpoint = _endpoint,
-            Job = "kute",
-            Instance = $"{Guid.NewGuid()}",
+            Job = JobName,
+            Instance = instanceLabel,
+            AdditionalLabels = labels,
+            HttpClient = httpClient,
         });
 
         _server = new MetricPushServer(_pusher);
@@ -88,7 +104,7 @@ public Task Batch(JsonRpc.Request.Batch batch, TimeSpan elapsed, CancellationTok
     public Task Single(JsonRpc.Request.Single single, TimeSpan elapsed, CancellationToken token = default)
     {
         _singleDuration
-            .WithLabels(single.Id)
+            .WithLabels(single.Id, single.MethodName)
             .Observe(elapsed.TotalSeconds);
         return Task.CompletedTask;
     }
@@ -98,4 +114,12 @@ public async Task Total(TimeSpan elapsed, CancellationToken token = default)
         await _pusher.PushAsync();
         _server.Stop();
     }
+
+    public static string JobName => "kute";
+    public static string GetMetricName(string name)
+    {
+        var lowerName = name.ToLower();
+        var sanitizedName = lowerName.Replace(" ", "_").Replace("-", "_");
+        return $"{JobName}_{sanitizedName}";
+    }
 }

tools/Kute/Nethermind.Tools.Kute/Program.cs

@@ -33,7 +33,10 @@ public static async Task<int> Main(string[] args)
             Config.ConcurrentRequests,
             Config.ShowProgress,
             Config.UnwrapBatch,
-            Config.PrometheusPushGateway
+            Config.PrometheusPushGateway,
+            Config.PrometheusPushGatewayUser,
+            Config.PrometheusPushGatewayPassword,
+            Config.Labels,
         ];
         rootCommand.SetAction(async (parseResult, cancellationToken) =>
         {
@@ -121,9 +124,12 @@ private static IServiceProvider BuildServiceProvider(ParseResult parseResult)
                 ? new ConsoleProgressReporter()
                 : new NullMetricsReporter();
 
+            Dictionary<string, string> labels = parseResult.GetValue(Config.Labels) ?? new();
             string? prometheusGateway = parseResult.GetValue(Config.PrometheusPushGateway);
+            string? prometheusGatewayUser = parseResult.GetValue(Config.PrometheusPushGatewayUser);
+            string? prometheusGatewayPassword = parseResult.GetValue(Config.PrometheusPushGatewayPassword);
             IMetricsReporter prometheusReporter = prometheusGateway is not null
-                ? new PrometheusPushGatewayMetricsReporter(prometheusGateway)
+                ? new PrometheusPushGatewayMetricsReporter(prometheusGateway, labels, prometheusGatewayUser, prometheusGatewayPassword)
                 : new NullMetricsReporter();
 
             return new ComposedMetricsReporter([memoryReporter, progresReporter, consoleReporter, prometheusReporter]);

tools/Kute/README.md

@@ -6,7 +6,7 @@ Kute - /kjuːt/ - is a benchmarking tool developed at Nethermind to simulate an
 
 This is a C# project and as such, it requires the [dotnet 9](https://dotnet.microsoft.com/en-us/download) SDK. Once installed, just run:
 
-```
+```bash
 dotnet build [-c Release]
 ```
 
@@ -24,49 +24,61 @@ Some typical usages are as follows:
 
 ### Connect to a Nethermind Client running at a specific address using a single file
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0
 ```
 
 ### Use all messages in the directory `/rpc-logs`
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc-logs
 ```
 
 ### Use a single messages file and emit results as HTML
 
-```
--a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Html
+```bash
+-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Json
 ```
 
 ### Use a single message file and emit results as JSON, while reporting metrics to a Prometheus Push Gateway (*)
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Json -g http://localhost:9091
 ```
 
-### Use a single messages file and record all responses into a new file
+### Use a single message file and report to a Prometheus Push Gateway with additional metrics labels
 
+```bash
+-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -g http://localhost:9091 -l key1=value1,key2=value2 -l key3=value3
 ```
+
+### Use a single message file and report to a Prometheus Push Gateway with basic auth
+
+```bash
+-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -g http://localhost:9091 --gateway-user user --gateway-pass pass
+```
+
+### Use a single messages file and record all responses into a new file
+
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -r rpc.responses.txt
 ```
 
 ### Use a single message file, using only `engine` and `eth` methods
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f engine,eth
 ```
 
 ### Use a single message file, using only the first 100 methods
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f .*=100
 ```
 
 ### Use a single message file, using only the first 50 `engine_newPayloadV2` or `engine_newPayloadV3` methods
 
-```
+```bash
 -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f engine_newPayloadV[23]=50
 ```