From 769f487f7bbdea3825f4e5806d64bfb6d8dec953 Mon Sep 17 00:00:00 2001 From: Neel Patel Date: Mon, 15 Jun 2020 18:05:56 +0530 Subject: [PATCH 1/5] Run the query for specific database version if provided from yml file. By default query will run on all the databases if "runonserver" is not provided. If user want the query to be run on multiple database versions, use below string. runonserver: "9.5, 9.6" Example yml file as below. ( e.g. below query will run only on database version 9.5 ) pg_replication: query: "SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp())) as lag" master: true runonserver: "9.5" metrics: - lag: usage: "GAUGE" description: "Replication lag behind master in seconds" --- cmd/postgres_exporter/postgres_exporter.go | 24 +++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 37795aacf..abc073d0a 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -115,6 +115,7 @@ type UserQuery struct { Metrics []Mapping `yaml:"metrics"` Master bool `yaml:"master"` // Querying only for master database CacheSeconds uint64 `yaml:"cache_seconds"` // Number of seconds to cache the namespace result metrics for. + RunOnServer string `yaml:"runonserver"` // Querying to run on which server version } // nolint: golint @@ -155,6 +156,7 @@ type intermediateMetricMap struct { columnMappings map[string]ColumnMapping master bool cacheSeconds uint64 + runonserver string } // MetricMapNamespace groups metric maps under a shared set of labels. @@ -163,6 +165,7 @@ type MetricMapNamespace struct { columnMappings map[string]MetricMap // Column mappings in this namespace master bool // Call query only for master database cacheSeconds uint64 // Number of seconds this metric namespace can be cached. 0 disables. + runonserver string // Run the query on which server version } // MetricMap stores the prometheus metric description which a given column will @@ -221,6 +224,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_stat_database": { map[string]ColumnMapping{ @@ -246,6 +250,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_stat_database_conflicts": { map[string]ColumnMapping{ @@ -259,6 +264,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_locks": { map[string]ColumnMapping{ @@ -268,6 +274,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_stat_replication": { map[string]ColumnMapping{ @@ -314,6 +321,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_stat_archiver": { map[string]ColumnMapping{ @@ -328,6 +336,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, "pg_stat_activity": { map[string]ColumnMapping{ @@ -338,6 +347,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, + "all", }, } @@ -494,6 +504,7 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str return nil, nil, err } + // Stores the loaded map representation metricMaps := make(map[string]intermediateMetricMap) newQueryOverrides := make(map[string]string) @@ -509,6 +520,7 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str columnMappings: newMetricMap, master: specs.Master, cacheSeconds: specs.CacheSeconds, + runonserver: specs.RunOnServer, } metricMaps[metric] = metricMap } @@ -678,7 +690,7 @@ func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metri } } - metricMap[namespace] = MetricMapNamespace{variableLabels, thisMap, intermediateMappings.master, intermediateMappings.cacheSeconds} + metricMap[namespace] = MetricMapNamespace{variableLabels, thisMap, intermediateMappings.master, intermediateMappings.cacheSeconds, intermediateMappings.runonserver} } return metricMap @@ -835,6 +847,8 @@ type Server struct { db *sql.DB labels prometheus.Labels master bool + runonserver string + serverversion string // Last version used to calculate metric map. If mismatch on scrape, // then maps are recalculated. @@ -880,6 +894,7 @@ func NewServer(dsn string, opts ...ServerOpt) (*Server, error) { s := &Server{ db: db, master: false, + runonserver: "all", labels: prometheus.Labels{ serverLabelName: fingerprint, }, @@ -1327,6 +1342,12 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str continue } + // check if the query is to be run on specific server version or not + if len(mapping.runonserver) > 0 && !strings.Contains(mapping.runonserver, "all") && !strings.Contains(server.serverversion, mapping.runonserver) { + log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.serverversion, mapping.runonserver) + continue + } + scrapeMetric := false // Check if the metric is cached server.cacheMtx.Lock() @@ -1415,6 +1436,7 @@ func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, server *Server) } server.lastMapVersion = semanticVersion + server.serverversion = semanticVersion.String() if e.userQueriesPath != "" { // Clear the metric while a reload is happening From 1e1e4e2c34f05e448fde85a2a7eadbda86334495 Mon Sep 17 00:00:00 2001 From: Neel Patel Date: Tue, 16 Jun 2020 20:12:34 +0530 Subject: [PATCH 2/5] Fixed the below review comments given by Ashesh Vashi Instead of having db version string from yml file, user can define the range of database server version where query is to be executed. If user want to run the query on database version greater than 10.0.0, use below format. runonserver: ">=10.0.0" Below are the example of db version range user can defined in yml file. <=10.1.0 >=12.1.0 =11.0.0 <9.6.0 || >=11.0.0 --- cmd/postgres_exporter/postgres_exporter.go | 36 +++++++++++----------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index abc073d0a..cc9b618eb 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -224,7 +224,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_stat_database": { map[string]ColumnMapping{ @@ -250,7 +250,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_stat_database_conflicts": { map[string]ColumnMapping{ @@ -264,7 +264,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_locks": { map[string]ColumnMapping{ @@ -274,7 +274,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_stat_replication": { map[string]ColumnMapping{ @@ -321,7 +321,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_stat_archiver": { map[string]ColumnMapping{ @@ -336,7 +336,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, "pg_stat_activity": { map[string]ColumnMapping{ @@ -347,7 +347,7 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "all", + "", }, } @@ -504,7 +504,6 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str return nil, nil, err } - // Stores the loaded map representation metricMaps := make(map[string]intermediateMetricMap) newQueryOverrides := make(map[string]string) @@ -844,11 +843,10 @@ type cachedMetrics struct { // Server describes a connection to Postgres. // Also it contains metrics map and query overrides. type Server struct { - db *sql.DB - labels prometheus.Labels - master bool + db *sql.DB + labels prometheus.Labels + master bool runonserver string - serverversion string // Last version used to calculate metric map. If mismatch on scrape, // then maps are recalculated. @@ -894,7 +892,6 @@ func NewServer(dsn string, opts ...ServerOpt) (*Server, error) { s := &Server{ db: db, master: false, - runonserver: "all", labels: prometheus.Labels{ serverLabelName: fingerprint, }, @@ -1342,10 +1339,14 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str continue } - // check if the query is to be run on specific server version or not - if len(mapping.runonserver) > 0 && !strings.Contains(mapping.runonserver, "all") && !strings.Contains(server.serverversion, mapping.runonserver) { - log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.serverversion, mapping.runonserver) - continue + // check if the query is to be run on specific database server version range or not + if len(mapping.runonserver) > 0 { + serVersion, _ := semver.Parse(server.lastMapVersion.String()) + runServerRange, _ := semver.ParseRange(mapping.runonserver) + if !runServerRange(serVersion) { + log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.lastMapVersion.String(), mapping.runonserver) + continue + } } scrapeMetric := false @@ -1436,7 +1437,6 @@ func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, server *Server) } server.lastMapVersion = semanticVersion - server.serverversion = semanticVersion.String() if e.userQueriesPath != "" { // Clear the metric while a reload is happening From dce6fac1ba7737d6f6e7bb03aa17c0a180f83be7 Mon Sep 17 00:00:00 2001 From: Neel Patel Date: Fri, 12 Feb 2021 15:41:21 +0530 Subject: [PATCH 3/5] Remove the call from unused places where 'runOnServer' is not required. Only Server type hold that value. --- cmd/postgres_exporter/postgres_exporter.go | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index e4a251a87..040cef01f 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -158,7 +158,6 @@ type intermediateMetricMap struct { columnMappings map[string]ColumnMapping master bool cacheSeconds uint64 - runonserver string } // MetricMapNamespace groups metric maps under a shared set of labels. @@ -167,7 +166,6 @@ type MetricMapNamespace struct { columnMappings map[string]MetricMap // Column mappings in this namespace master bool // Call query only for master database cacheSeconds uint64 // Number of seconds this metric namespace can be cached. 0 disables. - runonserver string // Run the query on which server version } // MetricMap stores the prometheus metric description which a given column will @@ -227,7 +225,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_stat_database": { map[string]ColumnMapping{ @@ -253,7 +250,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_stat_database_conflicts": { map[string]ColumnMapping{ @@ -267,7 +263,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_locks": { map[string]ColumnMapping{ @@ -277,7 +272,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_stat_replication": { map[string]ColumnMapping{ @@ -324,7 +318,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_replication_slots": { map[string]ColumnMapping{ @@ -349,7 +342,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, "pg_stat_activity": { map[string]ColumnMapping{ @@ -360,7 +352,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ }, true, 0, - "", }, } @@ -543,7 +534,6 @@ func parseUserQueries(content []byte) (map[string]intermediateMetricMap, map[str columnMappings: newMetricMap, master: specs.Master, cacheSeconds: specs.CacheSeconds, - runonserver: specs.RunOnServer, } metricMaps[metric] = metricMap } @@ -736,7 +726,7 @@ func makeDescMap(pgVersion semver.Version, serverLabels prometheus.Labels, metri } } - metricMap[namespace] = MetricMapNamespace{variableLabels, thisMap, intermediateMappings.master, intermediateMappings.cacheSeconds, intermediateMappings.runonserver} + metricMap[namespace] = MetricMapNamespace{variableLabels, thisMap, intermediateMappings.master, intermediateMappings.cacheSeconds} } return metricMap From 4d0e431c4a1423ff82bbef72de6d732f8fbe99c9 Mon Sep 17 00:00:00 2001 From: Neel Patel Date: Fri, 12 Feb 2021 15:54:42 +0530 Subject: [PATCH 4/5] Fix compilation issues. --- cmd/postgres_exporter/postgres_exporter.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 040cef01f..4504e02ff 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -1452,11 +1452,11 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str } // check if the query is to be run on specific database server version range or not - if len(mapping.runonserver) > 0 { + if len(server.runonserver) > 0 { serVersion, _ := semver.Parse(server.lastMapVersion.String()) - runServerRange, _ := semver.ParseRange(mapping.runonserver) + runServerRange, _ := semver.ParseRange(server.runonserver) if !runServerRange(serVersion) { - log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.lastMapVersion.String(), mapping.runonserver) + log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.lastMapVersion.String(), server.runonserver) continue } } From c1b50870297a2564e93491aed8c03f8bdd8d725a Mon Sep 17 00:00:00 2001 From: Neel Patel Date: Fri, 12 Feb 2021 16:07:55 +0530 Subject: [PATCH 5/5] Fix the issue with Debugln to print the database server version --- cmd/postgres_exporter/postgres_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 4504e02ff..c22f5b639 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -1456,7 +1456,7 @@ func queryNamespaceMappings(ch chan<- prometheus.Metric, server *Server) map[str serVersion, _ := semver.Parse(server.lastMapVersion.String()) runServerRange, _ := semver.ParseRange(server.runonserver) if !runServerRange(serVersion) { - log.Debugln("Query skipped for database version: [%s] as it should be run on database server version: [%s]", server.lastMapVersion.String(), server.runonserver) + log.Debugln("Query skipped for database version: ", server.lastMapVersion.String(), " as it should be run on database server version: ", server.runonserver) continue } }