Additional metrics initial work, with working metrics listing, but without actoual metrics mesurements

Author: pawelchcki

app/controllers/projects/environments_controller.rb

@@ -129,6 +129,12 @@ def metrics
     end
   end
 
+  def additional_metrics
+    additional_metrics = environment.additional_metrics || {}
+
+    render json: additional_metrics, status: additional_metrics.any? ? :ok : :no_content
+  end
+
   private
 
   def verify_api_request!

app/controllers/projects/prometheus_controller.rb

@@ -0,0 +1,22 @@
+class Projects::PrometheusController < Projects::ApplicationController
+  before_action :authorize_read_project!
+
+  def active_metrics
+    return render_404 unless has_prometheus_metrics?
+    matched_metrics = prometheus_service.reactive_query(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself)
+
+    if matched_metrics
+      render json: matched_metrics, status: :ok
+    else
+      head :no_content
+    end
+  end
+
+  def prometheus_service
+    project.monitoring_service
+  end
+
+  def has_prometheus_metrics?
+    prometheus_service&.respond_to?(:reactive_query)
+  end
+end

app/models/environment.rb

@@ -149,10 +149,18 @@ def has_metrics?
     project.monitoring_service.present? && available? && last_deployment.present?
   end
 
+  def has_additional_metrics?
+    has_metrics? && project.monitoring_service&.respond_to?(:reactive_query)
+  end
+
   def metrics
     project.monitoring_service.environment_metrics(self) if has_metrics?
   end
 
+  def additional_metrics
+    project.monitoring_service.reactive_query(Gitlab::Prometheus::Queries::AdditionalMetricsQuery, self.id) if has_additional_metrics?
+  end
+
   # An environment name is not necessarily suitable for use in URLs, DNS
   # or other third-party contexts, so provide a slugified version. A slug has
   # the following properties:

app/models/project_services/prometheus_service.rb

@@ -64,23 +64,26 @@ def test(*args)
   end
 
   def environment_metrics(environment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &:itself)
+    with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &method(:rename_data_to_metrics))
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &:itself)
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics))
     metrics&.merge(deployment_time: created_at.to_i) || {}
   end
 
+  def reactive_query(query_class, *args, &block)
+    calculate_reactive_cache(query_class, *args, &block)
+  end
+
   # Cache metrics for specific environment
   def calculate_reactive_cache(query_class_name, *args)
     return unless active? && project && !project.pending_delete?
 
-    metrics = Kernel.const_get(query_class_name).new(client).query(*args)
-
+    data = Kernel.const_get(query_class_name).new(client).query(*args)
     {
       success: true,
-      metrics: metrics,
+      data: data,
       last_update: Time.now.utc
     }
   rescue Gitlab::PrometheusError => err
@@ -90,4 +93,11 @@ def calculate_reactive_cache(query_class_name, *args)
   def client
     @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url)
   end
+
+  private
+
+  def rename_data_to_metrics(metrics)
+    metrics[:metrics] = metrics.delete :data
+    metrics
+  end
 end

config/additional_metrics.yml

@@ -0,0 +1,26 @@
+- group: Kubernetes
+  priority: 1
+  metrics:
+  - title: "Memory usage"
+    detect: container_memory_usage_bytes
+    weight: 1
+    queries:
+    - query_range: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
+      label: Container memory
+      unit: MiB
+  - title: "Current memory usage"
+    detect: container_memory_usage_bytes
+    weight: 1
+    queries:
+    - query: 'avg(container_memory_usage_bytes{%{environment_filter}}) / 2^20'
+      unit: MiB
+  - title: "CPU usage"
+    detect: container_cpu_usage_seconds_total
+    weight: 1
+    queries:
+    - query_range: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'
+  - title: "Current CPU usage"
+    detect: container_cpu_usage_seconds_total
+    weight: 1
+    queries:
+    - query: 'avg(rate(container_cpu_usage_seconds_total{%{environment_filter}}[2m])) * 100'

config/routes/project.rb

@@ -72,6 +72,10 @@
 
       resource :mattermost, only: [:new, :create]
 
+      namespace :prometheus do
+        get :active_metrics
+      end
+
       resources :deploy_keys, constraints: { id: /\d+/ }, only: [:index, :new, :create] do
         member do
           put :enable
@@ -152,6 +156,7 @@
           post :stop
           get :terminal
           get :metrics
+          get :additional_metrics
           get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', constraints: { format: nil }
         end
 

lib/gitlab/prometheus/metric.rb

@@ -0,0 +1,28 @@
+module Gitlab::Prometheus
+  class Metric
+    attr_reader :group, :title, :detect, :weight, :queries
+
+    def initialize(group, title, detect, weight, queries = [])
+      @group = group
+      @title = title
+      @detect = detect
+      @weight = weight
+      @queries = queries
+    end
+
+    def self.metric_from_entry(group, entry)
+      missing_fields = [:title, :detect, :weight, :queries].select { |key| !entry.has_key?(key) }
+      raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty?
+
+      Metric.new(group, entry[:title], entry[:detect], entry[:weight], entry[:queries])
+    end
+
+    def self.metrics_from_list(group, list)
+      list.map { |entry| metric_from_entry(group, entry) }
+    end
+
+    def self.additional_metrics_raw
+      @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys)
+    end
+  end
+end

lib/gitlab/prometheus/metric_group.rb

@@ -0,0 +1,33 @@
+module Gitlab::Prometheus
+  class MetricGroup
+    attr_reader :priority, :name
+    attr_accessor :metrics
+
+    def initialize(name, priority, metrics = [])
+      @name = name
+      @priority = priority
+      @metrics = metrics
+    end
+
+    def self.all
+      load_groups_from_yaml
+    end
+
+    def self.group_from_entry(entry)
+      missing_fields = [:group, :priority, :metrics].select { |key| !entry.has_key?(key) }
+      raise ParsingError.new("entry missing required fields #{missing_fields}") unless missing_fields.empty?
+
+      group = MetricGroup.new(entry[:group], entry[:priority])
+      group.metrics = Metric.metrics_from_list(group, entry[:metrics])
+      group
+    end
+
+    def self.load_groups_from_yaml
+      additional_metrics_raw.map(&method(:group_from_entry))
+    end
+
+    def self.additional_metrics_raw
+      @additional_metrics_raw ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml'))&.map(&:deep_symbolize_keys).freeze
+    end
+  end
+end

lib/gitlab/prometheus/metrics_sources.rb

@@ -0,0 +1,7 @@
+module Gitlab::Prometheus
+  module MetricsSources
+    def self.additional_metrics
+      @additional_metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).deep_symbolize_keys.freeze
+    end
+  end
+end

lib/gitlab/prometheus/parsing_error.rb

@@ -0,0 +1,3 @@
+module Gitlab::Prometheus
+  ParsingError = Class.new(StandardError)
+end

lib/gitlab/prometheus/queries/additional_metrics_query.rb

@@ -0,0 +1,70 @@
+module Gitlab::Prometheus::Queries
+  class AdditionalMetricsQuery < BaseQuery
+    def self.metrics
+      @metrics ||= YAML.load_file(Rails.root.join('config/custom_metrics.yml')).freeze
+    end
+
+    def query(environment_id)
+      environment = Environment.find_by(id: environment_id)
+
+      context = {
+        environment_slug: environment.slug,
+        environment_filter: %{container_name!="POD",environment="#{environment.slug}"}
+      }
+
+      timeframe_start = 8.hours.ago.to_f
+      timeframe_end = Time.now.to_f
+
+      matched_metrics.map do |group|
+        group[:metrics].map! do |metric|
+          metric[:queries].map! do |query|
+            query = query.symbolize_keys
+            query[:result] =
+              if query.has_key?(:query_range)
+                client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end)
+              else
+                client_query(query[:query] % context, time: timeframe_end)
+              end
+            query
+          end
+          metric
+        end
+        group
+      end
+    end
+
+    def process_query(group, query)
+      result = if query.has_key?(:query_range)
+                 client_query_range(query[:query_range] % context, start: timeframe_start, stop: timeframe_end)
+               else
+                 client_query(query[:query] % context, time: timeframe_end)
+               end
+      contains_metrics = result.all? do |item|
+        item&.[](:values)&.any? || item&.[](:value)&.any?
+      end
+    end
+
+    def process_result(query_result)
+      contains_metrics = query_result.all? do |item|
+        item&.[](:values)&.any? || item&.[](:value)&.any?
+      end
+
+      contains_metrics
+    end
+
+    def matched_metrics
+      label_values = client_label_values || []
+
+      result = Gitlab::Prometheus::MetricsSources.additional_metrics.map do |group|
+        group[:metrics].map!(&:symbolize_keys)
+        group[:metrics].select! do |metric|
+          matcher = Regexp.compile(metric[:detect])
+          label_values.any? &matcher.method(:match)
+        end
+        group
+      end
+
+      result.select {|group| !group[:metrics].empty?}
+    end
+  end
+end

lib/gitlab/prometheus/queries/base_query.rb

@@ -3,7 +3,7 @@ module Prometheus
     module Queries
       class BaseQuery
         attr_accessor :client
-        delegate :query_range, :query, to: :client, prefix: true
+        delegate :query_range, :query, :label_values, :series, to: :client, prefix: true
 
         def raw_memory_usage_query(environment_slug)
           %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20}

lib/gitlab/prometheus/queries/matched_metrics_query.rb

@@ -0,0 +1,72 @@
+module Gitlab::Prometheus::Queries
+  class MatchedMetricsQuery < BaseQuery
+    MAX_QUERY_ITEMS = 40.freeze
+
+    def self.metrics
+      @metrics ||= YAML.load_file(Rails.root.join('config/additional_metrics.yml')).map(&:deep_symbolize_keys)
+    end
+
+    def query
+      groups_data.map do |group, data|
+        {
+          group: group.name,
+          priority: group.priority,
+          active_metrics: data[:active_metrics],
+          metrics_missing_requirements: data[:metrics_missing_requirements]
+        }
+      end
+    end
+
+    def groups_data
+      metrics_series = metrics_with_series(Gitlab::Prometheus::MetricGroup.all)
+      lookup = active_series_lookup(metrics_series)
+
+      groups = {}
+
+      metrics_series.each do |metrics, series|
+        groups[metrics.group] ||= { active_metrics: 0, metrics_missing_requirements: 0 }
+        group = groups[metrics.group]
+
+        if series.all?(&lookup.method(:has_key?))
+          group[:active_metrics] += 1
+        else
+          group[:metrics_missing_requirements] += 1
+        end
+        group
+      end
+
+      groups
+    end
+
+    def active_series_lookup(metrics)
+      timeframe_start = 8.hours.ago
+      timeframe_end = Time.now
+
+      series = metrics.flat_map { |metrics, series| series }.uniq
+
+      lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series|
+        client_series(*batched_series, start: timeframe_start, stop: timeframe_end)
+          .select(&method(:has_matching_label))
+          .map { |series_info| [series_info['__name__'], true] }
+      end
+      lookup.to_h
+    end
+
+    def has_matching_label(series_info)
+      series_info.has_key?('environment')
+    end
+
+    def metrics_with_series(metric_groups)
+      label_values = client_label_values || []
+
+      metrics = metric_groups.flat_map do |group|
+        group.metrics.map do |metric|
+          matcher = Regexp.compile(metric.detect)
+          [metric, label_values.select(&matcher.method(:match))]
+        end
+      end
+
+      metrics.select { |metric, labels| labels&.any? }
+    end
+  end
+end

lib/gitlab/prometheus_client.rb

@@ -29,6 +29,14 @@ def query_range(query, start: 8.hours.ago, stop: Time.now)
       end
     end
 
+    def label_values(name='__name__')
+      json_api_get("label/#{name}/values")
+    end
+
+    def series(*matches, start: 8.hours.ago, stop: Time.now)
+      json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
+    end
+
     private
 
     def json_api_get(type, args = {})