Merge branch 'prometheus-community:master' into master

Author: winfredwz

.github/workflows/golangci-lint.yml

@@ -1,3 +1,5 @@
+---
+# This action is synced from https://github.com/prometheus/prometheus
 name: golangci-lint
 on:
   push:
@@ -27,4 +29,4 @@ jobs:
       - name: Lint
         uses: golangci/[email protected]
         with:
-          version: v1.51.2
+          version: v1.53.3

.golangci.yml

@@ -1,6 +1,7 @@
 ---
 linters:
   enable:
+  - misspell
   - revive
 
 issues:
@@ -14,3 +15,9 @@ linters-settings:
     exclude-functions:
       # Never check for logger errors.
       - (github.com/go-kit/log.Logger).Log
+  revive:
+    rules:
+      # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter
+      - name: unused-parameter
+        severity: warning
+        disabled: true

.yamllint

@@ -20,5 +20,4 @@ rules:
       config/testdata/section_key_dup.bad.yml
   line-length: disable
   truthy:
-    ignore: |
-      .github/workflows/*.yml
+    check-keys: false

CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.13.2 / 2023-07-21
+
+* [BUGFIX] Fix type issues on pg_postmaster metrics #828
+* [BUGFIX] Fix pg_replication collector instantiation #854
+* [BUGFIX] Fix pg_process_idle metrics #855
+
+## 0.13.1 / 2023-06-27
+
+* [BUGFIX] Make collectors not fail on null values #823
+
 ## 0.13.0 / 2023-06-21
 
 BREAKING CHANGES:

Makefile.common

@@ -55,13 +55,13 @@ ifneq ($(shell command -v gotestsum > /dev/null),)
 endif
 endif
 
-PROMU_VERSION ?= 0.14.0
+PROMU_VERSION ?= 0.15.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
 SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.51.2
+GOLANGCI_LINT_VERSION ?= v1.53.3
 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))

README.md

@@ -30,6 +30,26 @@ To use the multi-target functionality, send an http request to the endpoint `/pr
 
 To avoid putting sensitive information like username and password in the URL, preconfigured auth modules are supported via the [auth_modules](#auth_modules) section of the config file. auth_modules for DSNs can be used with the `/probe` endpoint by specifying the `?auth_module=foo` http parameter.
 
+Example Prometheus config:
+```yaml
+scrape_configs:
+  - job_name: 'postgres'
+    static_configs:
+      - targets:
+        - server1:5432
+        - server2:5432
+    metrics_path: /probe
+    params:
+      auth_module: [foo]
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: 127.0.0.1:9116  # The postgres exporter's real hostname:port.
+```
+
 ## Configuration File
 
 The configuration file controls the behavior of the exporter. It can be set using the `--config.file` command line flag and defaults to `postgres_exporter.yml`.
@@ -73,7 +93,10 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra
 
 
 * `[no-]collector.database`
-  Enable the database collector (default: enabled).
+  Enable the `database` collector (default: enabled).
+
+* `[no-]collector.locks`
+  Enable the `locks` collector (default: enabled).
 
 * `[no-]collector.postmaster`
    Enable the `postmaster` collector (default: enabled).

VERSION

@@ -1 +1 @@
-0.13.0
+0.13.1

cmd/postgres_exporter/postgres_exporter.go

@@ -176,15 +176,6 @@ var builtinMetricMaps = map[string]intermediateMetricMap{
 		true,
 		0,
 	},
-	"pg_locks": {
-		map[string]ColumnMapping{
-			"datname": {LABEL, "Name of this database", nil, nil},
-			"mode":    {LABEL, "Type of Lock", nil, nil},
-			"count":   {GAUGE, "Number of locks", nil, nil},
-		},
-		true,
-		0,
-	},
 	"pg_stat_replication": {
 		map[string]ColumnMapping{
 			"procpid":                  {DISCARD, "Process ID of a WAL sender process", nil, semver.MustParseRange("<9.2.0")},

cmd/postgres_exporter/queries.go

@@ -46,31 +46,6 @@ type OverrideQuery struct {
 // Overriding queries for namespaces above.
 // TODO: validate this is a closed set in tests, and there are no overlaps
 var queryOverrides = map[string][]OverrideQuery{
-	"pg_locks": {
-		{
-			semver.MustParseRange(">0.0.0"),
-			`SELECT pg_database.datname,tmp.mode,COALESCE(count,0) as count
-			FROM
-				(
-				  VALUES ('accesssharelock'),
-				         ('rowsharelock'),
-				         ('rowexclusivelock'),
-				         ('shareupdateexclusivelock'),
-				         ('sharelock'),
-				         ('sharerowexclusivelock'),
-				         ('exclusivelock'),
-				         ('accessexclusivelock'),
-					 ('sireadlock')
-				) AS tmp(mode) CROSS JOIN pg_database
-			LEFT JOIN
-			  (SELECT database, lower(mode) AS mode,count(*) AS count
-			  FROM pg_locks WHERE database IS NOT NULL
-			  GROUP BY database, lower(mode)
-			) AS tmp2
-			ON tmp.mode=tmp2.mode and pg_database.oid = tmp2.database ORDER BY 1`,
-		},
-	},
-
 	"pg_stat_replication": {
 		{
 			semver.MustParseRange(">=10.0.0"),

collector/collector_test.go

@@ -49,8 +49,14 @@ func readMetric(m prometheus.Metric) MetricResult {
 func sanitizeQuery(q string) string {
 	q = strings.Join(strings.Fields(q), " ")
 	q = strings.Replace(q, "(", "\\(", -1)
+	q = strings.Replace(q, "?", "\\?", -1)
 	q = strings.Replace(q, ")", "\\)", -1)
+	q = strings.Replace(q, "[", "\\[", -1)
+	q = strings.Replace(q, "]", "\\]", -1)
+	q = strings.Replace(q, "{", "\\{", -1)
+	q = strings.Replace(q, "}", "\\}", -1)
 	q = strings.Replace(q, "*", "\\*", -1)
+	q = strings.Replace(q, "^", "\\^", -1)
 	q = strings.Replace(q, "$", "\\$", -1)
 	return q
 }

collector/pg_database.go

@@ -15,6 +15,7 @@ package collector
 
 import (
 	"context"
+	"database/sql"
 
 	"github.com/go-kit/log"
 	"github.com/prometheus/client_golang/prometheus"
@@ -79,38 +80,42 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 	var databases []string
 
 	for rows.Next() {
-		var datname string
+		var datname sql.NullString
 		if err := rows.Scan(&datname); err != nil {
 			return err
 		}
 
+		if !datname.Valid {
+			continue
+		}
 		// Ignore excluded databases
 		// Filtering is done here instead of in the query to avoid
 		// a complicated NOT IN query with a variable number of parameters
-		if sliceContains(c.excludedDatabases, datname) {
+		if sliceContains(c.excludedDatabases, datname.String) {
 			continue
 		}
 
-		databases = append(databases, datname)
+		databases = append(databases, datname.String)
 	}
 
 	// Query the size of the databases
 	for _, datname := range databases {
-		var size int64
+		var size sql.NullFloat64
 		err = db.QueryRowContext(ctx, pgDatabaseSizeQuery, datname).Scan(&size)
 		if err != nil {
 			return err
 		}
 
+		sizeMetric := 0.0
+		if size.Valid {
+			sizeMetric = size.Float64
+		}
 		ch <- prometheus.MustNewConstMetric(
 			pgDatabaseSizeDesc,
-			prometheus.GaugeValue, float64(size), datname,
+			prometheus.GaugeValue, sizeMetric, datname,
 		)
 	}
-	if err := rows.Err(); err != nil {
-		return err
-	}
-	return nil
+	return rows.Err()
 }
 
 func sliceContains(slice []string, s string) bool {

collector/pg_database_test.go

@@ -59,3 +59,43 @@ func TestPGDatabaseCollector(t *testing.T) {
 		t.Errorf("there were unfulfilled exceptions: %s", err)
 	}
 }
+
+// TODO add a null db test
+
+func TestPGDatabaseCollectorNullMetric(t *testing.T) {
+	db, mock, err := sqlmock.New()
+	if err != nil {
+		t.Fatalf("Error opening a stub db connection: %s", err)
+	}
+	defer db.Close()
+
+	inst := &instance{db: db}
+
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname"}).
+		AddRow("postgres"))
+
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseSizeQuery)).WithArgs("postgres").WillReturnRows(sqlmock.NewRows([]string{"pg_database_size"}).
+		AddRow(nil))
+
+	ch := make(chan prometheus.Metric)
+	go func() {
+		defer close(ch)
+		c := PGDatabaseCollector{}
+		if err := c.Update(context.Background(), inst, ch); err != nil {
+			t.Errorf("Error calling PGDatabaseCollector.Update: %s", err)
+		}
+	}()
+
+	expected := []MetricResult{
+		{labels: labelMap{"datname": "postgres"}, value: 0, metricType: dto.MetricType_GAUGE},
+	}
+	convey.Convey("Metrics comparison", t, func() {
+		for _, expect := range expected {
+			m := readMetric(<-ch)
+			convey.So(expect, convey.ShouldResemble, m)
+		}
+	})
+	if err := mock.ExpectationsWereMet(); err != nil {
+		t.Errorf("there were unfulfilled exceptions: %s", err)
+	}
+}

collector/pg_database_wraparound.go

@@ -0,0 +1,115 @@
+// Copyright 2023 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"context"
+	"database/sql"
+
+	"github.com/go-kit/log"
+	"github.com/go-kit/log/level"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const databaseWraparoundSubsystem = "database_wraparound"
+
+func init() {
+	registerCollector(databaseWraparoundSubsystem, defaultDisabled, NewPGDatabaseWraparoundCollector)
+}
+
+type PGDatabaseWraparoundCollector struct {
+	log log.Logger
+}
+
+func NewPGDatabaseWraparoundCollector(config collectorConfig) (Collector, error) {
+	return &PGDatabaseWraparoundCollector{log: config.logger}, nil
+}
+
+var (
+	databaseWraparoundAgeDatfrozenxid = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datfrozenxid_seconds"),
+		"Age of the oldest transaction ID that has not been frozen.",
+		[]string{"datname"},
+		prometheus.Labels{},
+	)
+	databaseWraparoundAgeDatminmxid = prometheus.NewDesc(
+		prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datminmxid_seconds"),
+		"Age of the oldest multi-transaction ID that has been replaced with a transaction ID.",
+		[]string{"datname"},
+		prometheus.Labels{},
+	)
+
+	databaseWraparoundQuery = `
+	SELECT
+		datname,
+		age(d.datfrozenxid) as age_datfrozenxid,
+		mxid_age(d.datminmxid) as age_datminmxid
+	FROM
+		pg_catalog.pg_database d
+	WHERE
+		d.datallowconn
+	`
+)
+
+func (c *PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error {
+	db := instance.getDB()
+	rows, err := db.QueryContext(ctx,
+		databaseWraparoundQuery)
+
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var datname sql.NullString
+		var ageDatfrozenxid, ageDatminmxid sql.NullFloat64
+
+		if err := rows.Scan(&datname, &ageDatfrozenxid, &ageDatminmxid); err != nil {
+			return err
+		}
+
+		if !datname.Valid {
+			level.Debug(c.log).Log("msg", "Skipping database with NULL name")
+			continue
+		}
+		if !ageDatfrozenxid.Valid {
+			level.Debug(c.log).Log("msg", "Skipping stat emission with NULL age_datfrozenxid")
+			continue
+		}
+		if !ageDatminmxid.Valid {
+			level.Debug(c.log).Log("msg", "Skipping stat emission with NULL age_datminmxid")
+			continue
+		}
+
+		ageDatfrozenxidMetric := ageDatfrozenxid.Float64
+
+		ch <- prometheus.MustNewConstMetric(
+			databaseWraparoundAgeDatfrozenxid,
+			prometheus.GaugeValue,
+			ageDatfrozenxidMetric, datname.String,
+		)
+
+		ageDatminmxidMetric := ageDatminmxid.Float64
+		ch <- prometheus.MustNewConstMetric(
+			databaseWraparoundAgeDatminmxid,
+			prometheus.GaugeValue,
+			ageDatminmxidMetric, datname.String,
+		)
+	}
+	if err := rows.Err(); err != nil {
+		return err
+	}
+	return nil
+}