diff --git a/tools/Kute/Nethermind.Tools.Kute.Test/MetricsTests.cs b/tools/Kute/Nethermind.Tools.Kute.Test/MetricsTests.cs index 2b82a9b36fc..3367f3bc27a 100644 --- a/tools/Kute/Nethermind.Tools.Kute.Test/MetricsTests.cs +++ b/tools/Kute/Nethermind.Tools.Kute.Test/MetricsTests.cs @@ -11,9 +11,9 @@ namespace Nethermind.Tools.Kute.Test; public class MetricsTests { - private static JsonRpc.Request.Single Single(int id) + private static JsonRpc.Request.Single Single(int id, string method = "test") { - var json = $$"""{ "id": {{id}}, "method": "test", "params": [] }"""; + var json = $$"""{ "id": {{id}}, "method": "{{method}}", "params": [] }"""; return new JsonRpc.Request.Single(JsonNode.Parse(json)!); } @@ -31,7 +31,7 @@ public async Task MemoryMetricsReporter_GeneratesValidReport() var totalTimer = new Timer(); using (totalTimer.Time()) { - var single = Single(42); + var single = Single(42, "method1"); var batch = Batch(Single(43), Single(44), Single(45)); var singleTimer = new Timer(); @@ -57,7 +57,9 @@ public async Task MemoryMetricsReporter_GeneratesValidReport() report.TotalTime.Should().BeLessThan(TimeSpan.FromMilliseconds(110)); report.Singles.Should().HaveCount(1); - report.Singles.Should().ContainKey("42"); + report.Singles.Should().ContainKey("method1"); + report.Singles["method1"].Should().HaveCount(1); + report.Singles["method1"].Should().ContainKey("42"); report.Batches.Should().HaveCount(1); report.Batches.Should().ContainKey("43:45"); diff --git a/tools/Kute/Nethermind.Tools.Kute/Config.cs b/tools/Kute/Nethermind.Tools.Kute/Config.cs index 365009c5f3a..c73bd147549 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Config.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Config.cs @@ -77,8 +77,43 @@ public static class Config HelpName = "url", }; + public static Option PrometheusPushGatewayUser { get; } = new("--gateway-user") + { + Description = "Prometheus Push Gateway username", + HelpName = "username", + }; + + public static Option PrometheusPushGatewayPassword { get; } = new("--gateway-pass") + { + Description = "Prometheus Push Gateway password", + HelpName = "password", + }; + public static Option UnwrapBatch { get; } = new("--unwrapBatch", "-u") { Description = "Batch requests will be unwraped to single requests" }; + + public static Option> Labels { get; } = new("--labels", "-l") + { + DefaultValueFactory = r => new Dictionary(), + CustomParser = r => + { + var labels = new Dictionary(); + foreach (var token in r.Tokens) + { + foreach (var pair in token.Value.Split(',')) + { + var parts = pair.Split('=', 2); + if (parts.Length == 2) + { + labels.Add(parts[0], parts[1]); + } + } + } + return labels; + }, + Description = "A comma separated list of tags to be added to the metrics in the format key=value", + HelpName = "labels", + }; } diff --git a/tools/Kute/Nethermind.Tools.Kute/Metrics/MemoryMetricsReporter.cs b/tools/Kute/Nethermind.Tools.Kute/Metrics/MemoryMetricsReporter.cs index e63aaf5b0a9..268e08e18ff 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Metrics/MemoryMetricsReporter.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Metrics/MemoryMetricsReporter.cs @@ -9,7 +9,7 @@ public sealed class MemoryMetricsReporter : IMetricsReporter , IMetricsReportProvider { - private readonly ConcurrentDictionary _singles = new(); + private readonly ConcurrentDictionary> _singles = new(); private readonly ConcurrentDictionary _batches = new(); private TimeSpan _totalRunningTime; @@ -36,10 +36,19 @@ public Task Batch(JsonRpc.Request.Batch batch, TimeSpan elapsed, CancellationTok } public Task Single(JsonRpc.Request.Single single, TimeSpan elapsed, CancellationToken token = default) { - var id = single.Id?.ToString(); - if (id is not null) + var id = single.Id; + var methodName = single.MethodName; + if (id is not null && methodName is not null) { - _singles[id] = elapsed; + var newMethodDict = new ConcurrentDictionary(); + newMethodDict[id] = elapsed; + _singles.AddOrUpdate(methodName, + newMethodDict, + (_, dict) => + { + dict[id] = elapsed; + return dict; + }); } return Task.CompletedTask; @@ -60,7 +69,7 @@ public MetricsReport Report() Ignored = _ignored, Responses = _responses, TotalTime = _totalRunningTime, - Singles = _singles.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), + Singles = _singles.ToDictionary(kvp => kvp.Key, kvp => (IReadOnlyDictionary)kvp.Value.ToDictionary(ikvp => ikvp.Key, ikvp => ikvp.Value)), Batches = _batches.ToDictionary(kvp => kvp.Key, kvp => kvp.Value), }; } diff --git a/tools/Kute/Nethermind.Tools.Kute/Metrics/MetricsReport.cs b/tools/Kute/Nethermind.Tools.Kute/Metrics/MetricsReport.cs index e538ecce8d1..bf0fbd177f1 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Metrics/MetricsReport.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Metrics/MetricsReport.cs @@ -54,13 +54,13 @@ public sealed record MetricsReport public required long Ignored { get; init; } public required long Responses { get; init; } public required TimeSpan TotalTime { get; init; } - public required IReadOnlyDictionary Singles { get; init; } + public required IReadOnlyDictionary> Singles { get; init; } public required IReadOnlyDictionary Batches { get; init; } // Computed properties - private TimeMetrics? _singlesMetrics; + private Dictionary? _singlesMetrics; private TimeMetrics? _batchesMetrics; - public TimeMetrics SinglesMetrics => _singlesMetrics ??= TimeMetrics.From(Singles.Values.ToList()); + public Dictionary SinglesMetrics => _singlesMetrics ??= Singles.ToDictionary(kvp => kvp.Key, kvp => TimeMetrics.From(kvp.Value.Values.ToList())); public TimeMetrics BatchesMetrics => _batchesMetrics ??= TimeMetrics.From(Batches.Values.ToList()); } diff --git a/tools/Kute/Nethermind.Tools.Kute/Metrics/PrettyMetricsReportFormatter.cs b/tools/Kute/Nethermind.Tools.Kute/Metrics/PrettyMetricsReportFormatter.cs index e5639e08664..3fb59254950 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Metrics/PrettyMetricsReportFormatter.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Metrics/PrettyMetricsReportFormatter.cs @@ -16,11 +16,18 @@ public async Task WriteAsync(Stream stream, MetricsReport report, CancellationTo await writer.WriteLineAsync($"Ignored: {report.Ignored}", token); await writer.WriteLineAsync($"Responses: {report.Responses}\n", token); await writer.WriteLineAsync("Singles:", token); - await writer.WriteLineAsync($" Count: {report.Singles.Count}", token); - await writer.WriteLineAsync($" Max: {report.SinglesMetrics.Max.TotalMilliseconds} ms", token); - await writer.WriteLineAsync($" Average: {report.SinglesMetrics.Average.TotalMilliseconds} ms", token); - await writer.WriteLineAsync($" Min: {report.SinglesMetrics.Min.TotalMilliseconds} ms", token); - await writer.WriteLineAsync($" Stddev: {report.SinglesMetrics.StandardDeviation.TotalMilliseconds} ms", token); + foreach (var single in report.SinglesMetrics) + { + var methodName = single.Key; + var metrics = single.Value; + await writer.WriteLineAsync($" {methodName}:", token); + await writer.WriteLineAsync($" Count: {report.Singles[methodName].Count}", token); + await writer.WriteLineAsync($" Max: {metrics.Max.TotalMilliseconds} ms", token); + await writer.WriteLineAsync($" Average: {metrics.Average.TotalMilliseconds} ms", token); + await writer.WriteLineAsync($" Min: {metrics.Min.TotalMilliseconds} ms", token); + await writer.WriteLineAsync($" Stddev: {metrics.StandardDeviation.TotalMilliseconds} ms", token); + } + await writer.WriteLineAsync("Batches:", token); await writer.WriteLineAsync($" Count: {report.Batches.Count}", token); await writer.WriteLineAsync($" Max: {report.BatchesMetrics.Max.TotalMilliseconds} ms", token); diff --git a/tools/Kute/Nethermind.Tools.Kute/Metrics/PrometheusPushGatewayMetricsReporter.cs b/tools/Kute/Nethermind.Tools.Kute/Metrics/PrometheusPushGatewayMetricsReporter.cs index 2f690a7a81c..1837a443892 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Metrics/PrometheusPushGatewayMetricsReporter.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Metrics/PrometheusPushGatewayMetricsReporter.cs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Net.Http.Headers; +using System.Text; using Prometheus.Client; using Prometheus.Client.Collectors; using Prometheus.Client.MetricPusher; @@ -18,29 +20,43 @@ public sealed class PrometheusPushGatewayMetricsReporter : IMetricsReporter private readonly ICounter _failedCounter; private readonly ICounter _ignoredCounter; private readonly ICounter _responseCounter; - private readonly IMetricFamily> _singleDuration; - private readonly IMetricFamily> _batchDuration; - - public PrometheusPushGatewayMetricsReporter(string endpoint) + private readonly IMetricFamily _singleDuration; + private readonly IMetricFamily _batchDuration; + + public PrometheusPushGatewayMetricsReporter( + string endpoint, + Dictionary labels, + string? user, + string? password + ) { var registry = new CollectorRegistry(); var factory = new MetricFactory(registry); - _messageCounter = factory.CreateCounter("messages", ""); - _succeededCounter = factory.CreateCounter("succeeded", ""); - _failedCounter = factory.CreateCounter("failed", ""); - _ignoredCounter = factory.CreateCounter("ignored", ""); - _responseCounter = factory.CreateCounter("responses", ""); - _singleDuration = factory.CreateHistogram("single_duration", "", labelName: "jsonrpc_id"); - _batchDuration = factory.CreateHistogram("batch_duration", "", labelName: "jsonrpc_id"); + _messageCounter = factory.CreateCounter(GetMetricName("messages_total"), ""); + _succeededCounter = factory.CreateCounter(GetMetricName("messages_succeeded"), ""); + _failedCounter = factory.CreateCounter(GetMetricName("messages_failed"), ""); + _ignoredCounter = factory.CreateCounter(GetMetricName("messages_ignored"), ""); + _responseCounter = factory.CreateCounter(GetMetricName("responses_total"), ""); + _singleDuration = factory.CreateHistogram(GetMetricName("single_duration_seconds"), "", labelNames: new[] { "jsonrpc_id", "method" }); + _batchDuration = factory.CreateHistogram(GetMetricName("batch_duration_seconds"), "", labelNames: new[] { "jsonrpc_id" }); _endpoint = endpoint; + string instanceLabel = labels.TryGetValue("instance", out var instance) ? instance : Guid.NewGuid().ToString(); + labels.Remove("instance"); + var httpClient = new HttpClient(); + if (user is not null && password is not null) + { + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", Convert.ToBase64String(Encoding.UTF8.GetBytes($"{user}:{password}"))); + } _pusher = new MetricPusher(new MetricPusherOptions { CollectorRegistry = registry, Endpoint = _endpoint, - Job = "kute", - Instance = $"{Guid.NewGuid()}", + Job = JobName, + Instance = instanceLabel, + AdditionalLabels = labels, + HttpClient = httpClient, }); _server = new MetricPushServer(_pusher); @@ -88,7 +104,7 @@ public Task Batch(JsonRpc.Request.Batch batch, TimeSpan elapsed, CancellationTok public Task Single(JsonRpc.Request.Single single, TimeSpan elapsed, CancellationToken token = default) { _singleDuration - .WithLabels(single.Id) + .WithLabels(single.Id, single.MethodName) .Observe(elapsed.TotalSeconds); return Task.CompletedTask; } @@ -98,4 +114,12 @@ public async Task Total(TimeSpan elapsed, CancellationToken token = default) await _pusher.PushAsync(); _server.Stop(); } + + public static string JobName => "kute"; + public static string GetMetricName(string name) + { + var lowerName = name.ToLower(); + var sanitizedName = lowerName.Replace(" ", "_").Replace("-", "_"); + return $"{JobName}_{sanitizedName}"; + } } diff --git a/tools/Kute/Nethermind.Tools.Kute/Program.cs b/tools/Kute/Nethermind.Tools.Kute/Program.cs index 0266ad676a3..031f6121a7a 100644 --- a/tools/Kute/Nethermind.Tools.Kute/Program.cs +++ b/tools/Kute/Nethermind.Tools.Kute/Program.cs @@ -33,7 +33,10 @@ public static async Task Main(string[] args) Config.ConcurrentRequests, Config.ShowProgress, Config.UnwrapBatch, - Config.PrometheusPushGateway + Config.PrometheusPushGateway, + Config.PrometheusPushGatewayUser, + Config.PrometheusPushGatewayPassword, + Config.Labels, ]; rootCommand.SetAction(async (parseResult, cancellationToken) => { @@ -121,9 +124,12 @@ private static IServiceProvider BuildServiceProvider(ParseResult parseResult) ? new ConsoleProgressReporter() : new NullMetricsReporter(); + Dictionary labels = parseResult.GetValue(Config.Labels) ?? new(); string? prometheusGateway = parseResult.GetValue(Config.PrometheusPushGateway); + string? prometheusGatewayUser = parseResult.GetValue(Config.PrometheusPushGatewayUser); + string? prometheusGatewayPassword = parseResult.GetValue(Config.PrometheusPushGatewayPassword); IMetricsReporter prometheusReporter = prometheusGateway is not null - ? new PrometheusPushGatewayMetricsReporter(prometheusGateway) + ? new PrometheusPushGatewayMetricsReporter(prometheusGateway, labels, prometheusGatewayUser, prometheusGatewayPassword) : new NullMetricsReporter(); return new ComposedMetricsReporter([memoryReporter, progresReporter, consoleReporter, prometheusReporter]); diff --git a/tools/Kute/README.md b/tools/Kute/README.md index 7e4c495120f..87697ed9a27 100644 --- a/tools/Kute/README.md +++ b/tools/Kute/README.md @@ -6,7 +6,7 @@ Kute - /kjuːt/ - is a benchmarking tool developed at Nethermind to simulate an This is a C# project and as such, it requires the [dotnet 9](https://dotnet.microsoft.com/en-us/download) SDK. Once installed, just run: -``` +```bash dotnet build [-c Release] ``` @@ -24,49 +24,61 @@ Some typical usages are as follows: ### Connect to a Nethermind Client running at a specific address using a single file -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 ``` ### Use all messages in the directory `/rpc-logs` -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc-logs ``` ### Use a single messages file and emit results as HTML -``` --a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Html +```bash +-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Json ``` ### Use a single message file and emit results as JSON, while reporting metrics to a Prometheus Push Gateway (*) -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -o Json -g http://localhost:9091 ``` -### Use a single messages file and record all responses into a new file +### Use a single message file and report to a Prometheus Push Gateway with additional metrics labels +```bash +-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -g http://localhost:9091 -l key1=value1,key2=value2 -l key3=value3 ``` + +### Use a single message file and report to a Prometheus Push Gateway with basic auth + +```bash +-a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -g http://localhost:9091 --gateway-user user --gateway-pass pass +``` + +### Use a single messages file and record all responses into a new file + +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -r rpc.responses.txt ``` ### Use a single message file, using only `engine` and `eth` methods -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f engine,eth ``` ### Use a single message file, using only the first 100 methods -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f .*=100 ``` ### Use a single message file, using only the first 50 `engine_newPayloadV2` or `engine_newPayloadV3` methods -``` +```bash -a http://localhost:8551 -s /keystore/jwt-secret -i /rpc.0 -f engine_newPayloadV[23]=50 ```