Merge remote-tracking branch 'upstream/master' into alessio-upstream-sync

Author: alessio-form3

.circleci/config.yml

@@ -0,0 +1,104 @@
+---
+version: 2.1
+
+orbs:
+  prometheus: prometheus/[email protected]
+
+executors:
+  # This must match .promu.yml.
+  golang:
+    docker:
+    - image: cimg/go:1.21
+
+jobs:
+  test:
+    executor: golang
+
+    steps:
+    - prometheus/setup_environment
+    - run: GOHOSTARCH=386 GOARCH=386 make test
+    - run: make
+    - prometheus/store_artifact:
+        file: postgres_exporter
+
+  integration:
+    docker:
+    - image: cimg/go:1.20
+    - image: << parameters.postgres_image >>
+      environment:
+        POSTGRES_DB: circle_test
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: test
+
+    parameters:
+      postgres_image:
+        type: string
+
+    environment:
+      DATA_SOURCE_NAME: 'postgresql://postgres:test@localhost:5432/circle_test?sslmode=disable'
+      GOOPTS: '-v -tags integration'
+
+    steps:
+    - checkout
+    - setup_remote_docker
+    - run: docker version
+    - run: make build
+    - run: make test
+
+workflows:
+  version: 2
+  postgres_exporter:
+    jobs:
+    - test:
+        filters:
+          tags:
+            only: /.*/
+    - integration:
+        matrix:
+          parameters:
+            postgres_image:
+            - circleci/postgres:11
+            - circleci/postgres:12
+            - circleci/postgres:13
+            - cimg/postgres:14.9
+            - cimg/postgres:15.4
+            - cimg/postgres:16.0
+    - prometheus/build:
+        name: build
+        parallelism: 3
+        promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
+        filters:
+          tags:
+            ignore: /^v.*/
+          branches:
+            ignore: /^(main|master|release-.*|.*build-all.*)$/
+    - prometheus/build:
+        name: build_all
+        parallelism: 12
+        filters:
+          branches:
+            only: /^(main|master|release-.*|.*build-all.*)$/
+          tags:
+            only: /^v.*/
+    - prometheus/publish_master:
+        context: org-context
+        docker_hub_organization: prometheuscommunity
+        quay_io_organization: prometheuscommunity
+        requires:
+        - test
+        - build_all
+        filters:
+          branches:
+            only: master
+    - prometheus/publish_release:
+        context: org-context
+        docker_hub_organization: prometheuscommunity
+        quay_io_organization: prometheuscommunity
+        requires:
+        - test
+        - build_all
+        filters:
+          tags:
+            only: /^v.*/
+          branches:
+            ignore: /.*/

.github/workflows/golangci-lint.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: install Go
         uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Install snmp_exporter/generator dependencies
         run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
         if: github.repository == 'prometheus/snmp_exporter'

.promu.yml

@@ -1,13 +1,12 @@
 go:
     # This must match .circle/config.yml.
-    version: 1.20
+    version: 1.21
 repository:
     path: github.com/form3tech-oss/postgres_exporter
 build:
     binaries:
         - name: postgres_exporter
           path: ./cmd/postgres_exporter
-    flags: -a -tags 'netgo static_build'
     ldflags: |
         -X github.com/prometheus/common/version.Version={{.Version}}
         -X github.com/prometheus/common/version.Revision={{.Revision}}

CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.15.0 / 2023-10-27
+
+* [ENHANCEMENT] Add 1kB and 2kB units #915
+* [BUGFIX] Add error log when probe collector creation fails #918
+* [BUGFIX] Fix test build failures on 32-bit arch #919
+* [BUGFIX] Adjust collector to use separate connection per scrape #936
+
 ## 0.14.0 / 2023-09-11
 
 * [CHANGE] Add `state` label to pg_process_idle_seconds #862

README.md

@@ -7,7 +7,7 @@
 
 Prometheus exporter for PostgreSQL server metrics.
 
-CI Tested PostgreSQL versions: `10`, `11`, `12`, `13`, `14`, `15`
+CI Tested PostgreSQL versions: `11`, `12`, `13`, `14`, `15`, `16`
 
 ## Quick Start
 This package is available for Docker:

VERSION

@@ -0,0 +1 @@
+0.15.0

cmd/postgres_exporter/main.go

@@ -93,7 +93,7 @@ func main() {
 	}
 
 	excludedDatabases := strings.Split(*excludeDatabases, ",")
-	logger.Log("msg", "Excluded databases", "databases", fmt.Sprintf("%v", excludedDatabases))
+	level.Info(logger).Log("msg", "Excluded databases", "databases", fmt.Sprintf("%v", excludedDatabases))
 
 	if *queriesPath != "" {
 		level.Warn(logger).Log("msg", "The extended queries.yaml config is DEPRECATED", "file", *queriesPath)

collector/pg_database.go

@@ -53,12 +53,21 @@ var (
 		"Disk space used by the database",
 		[]string{"datname"}, nil,
 	)
+	pgDatabaseConnectionLimitsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			databaseSubsystem,
+			"connection_limit",
+		),
+		"Connection limit set for the database",
+		[]string{"datname"}, nil,
+	)
 
-	pgDatabaseQuery     = "SELECT pg_database.datname FROM pg_database;"
+	pgDatabaseQuery     = "SELECT pg_database.datname, pg_database.datconnlimit FROM pg_database;"
 	pgDatabaseSizeQuery = "SELECT pg_database_size($1)"
 )
 
-// Update implements Collector and exposes database size.
+// Update implements Collector and exposes database size and connection limits.
 // It is called by the Prometheus registry when collecting metrics.
 // The list of databases is retrieved from pg_database and filtered
 // by the excludeDatabase config parameter. The tradeoff here is that
@@ -81,21 +90,32 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 
 	for rows.Next() {
 		var datname sql.NullString
-		if err := rows.Scan(&datname); err != nil {
+		var connLimit sql.NullInt64
+		if err := rows.Scan(&datname, &connLimit); err != nil {
 			return err
 		}
 
 		if !datname.Valid {
 			continue
 		}
+		database := datname.String
 		// Ignore excluded databases
 		// Filtering is done here instead of in the query to avoid
 		// a complicated NOT IN query with a variable number of parameters
-		if sliceContains(c.excludedDatabases, datname.String) {
+		if sliceContains(c.excludedDatabases, database) {
 			continue
 		}
 
-		databases = append(databases, datname.String)
+		databases = append(databases, database)
+
+		connLimitMetric := 0.0
+		if connLimit.Valid {
+			connLimitMetric = float64(connLimit.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			pgDatabaseConnectionLimitsDesc,
+			prometheus.GaugeValue, connLimitMetric, database,
+		)
 	}
 
 	// Query the size of the databases
@@ -114,6 +134,7 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 			pgDatabaseSizeDesc,
 			prometheus.GaugeValue, sizeMetric, datname,
 		)
+
 	}
 	return rows.Err()
 }

collector/pg_database_test.go

@@ -31,8 +31,8 @@ func TestPGDatabaseCollector(t *testing.T) {
 
 	inst := &instance{db: db}
 
-	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname"}).
-		AddRow("postgres"))
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname", "datconnlimit"}).
+		AddRow("postgres", 15))
 
 	mock.ExpectQuery(sanitizeQuery(pgDatabaseSizeQuery)).WithArgs("postgres").WillReturnRows(sqlmock.NewRows([]string{"pg_database_size"}).
 		AddRow(1024))
@@ -47,6 +47,7 @@ func TestPGDatabaseCollector(t *testing.T) {
 	}()
 
 	expected := []MetricResult{
+		{labels: labelMap{"datname": "postgres"}, value: 15, metricType: dto.MetricType_GAUGE},
 		{labels: labelMap{"datname": "postgres"}, value: 1024, metricType: dto.MetricType_GAUGE},
 	}
 	convey.Convey("Metrics comparison", t, func() {
@@ -71,8 +72,8 @@ func TestPGDatabaseCollectorNullMetric(t *testing.T) {
 
 	inst := &instance{db: db}
 
-	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname"}).
-		AddRow("postgres"))
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname", "datconnlimit"}).
+		AddRow("postgres", nil))
 
 	mock.ExpectQuery(sanitizeQuery(pgDatabaseSizeQuery)).WithArgs("postgres").WillReturnRows(sqlmock.NewRows([]string{"pg_database_size"}).
 		AddRow(nil))
@@ -88,6 +89,7 @@ func TestPGDatabaseCollectorNullMetric(t *testing.T) {
 
 	expected := []MetricResult{
 		{labels: labelMap{"datname": "postgres"}, value: 0, metricType: dto.MetricType_GAUGE},
+		{labels: labelMap{"datname": "postgres"}, value: 0, metricType: dto.MetricType_GAUGE},
 	}
 	convey.Convey("Metrics comparison", t, func() {
 		for _, expect := range expected {

collector/pg_replication_slot.go

@@ -43,7 +43,7 @@ var (
 			"slot_current_wal_lsn",
 		),
 		"current wal lsn value",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
 	)
 	pgReplicationSlotCurrentFlushDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(
@@ -52,7 +52,7 @@ var (
 			"slot_confirmed_flush_lsn",
 		),
 		"last lsn confirmed flushed to the replication slot",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
 	)
 	pgReplicationSlotIsActiveDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(
@@ -61,18 +61,39 @@ var (
 			"slot_is_active",
 		),
 		"whether the replication slot is active or not",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
+	)
+	pgReplicationSlotSafeWal = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			replicationSlotSubsystem,
+			"safe_wal_size_bytes",
+		),
+		"number of bytes that can be written to WAL such that this slot is not in danger of getting in state lost",
+		[]string{"slot_name", "slot_type"}, nil,
+	)
+	pgReplicationSlotWalStatus = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			replicationSlotSubsystem,
+			"wal_status",
+		),
+		"availability of WAL files claimed by this slot",
+		[]string{"slot_name", "slot_type", "wal_status"}, nil,
 	)
 
 	pgReplicationSlotQuery = `SELECT
 		slot_name,
-		CASE WHEN pg_is_in_recovery() THEN 
+		slot_type,
+		CASE WHEN pg_is_in_recovery() THEN
 		    pg_last_wal_receive_lsn() - '0/0'
-		ELSE 
-		    pg_current_wal_lsn() - '0/0' 
+		ELSE
+		    pg_current_wal_lsn() - '0/0'
 		END AS current_wal_lsn,
-		COALESCE(confirmed_flush_lsn, '0/0') - '0/0',
-		active
+		COALESCE(confirmed_flush_lsn, '0/0') - '0/0' AS confirmed_flush_lsn,
+		active,
+		safe_wal_size,
+		wal_status
 	FROM pg_replication_slots;`
 )
 
@@ -87,10 +108,13 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 
 	for rows.Next() {
 		var slotName sql.NullString
+		var slotType sql.NullString
 		var walLSN sql.NullFloat64
 		var flushLSN sql.NullFloat64
 		var isActive sql.NullBool
-		if err := rows.Scan(&slotName, &walLSN, &flushLSN, &isActive); err != nil {
+		var safeWalSize sql.NullInt64
+		var walStatus sql.NullString
+		if err := rows.Scan(&slotName, &slotType, &walLSN, &flushLSN, &isActive, &safeWalSize, &walStatus); err != nil {
 			return err
 		}
 
@@ -102,14 +126,18 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 		if slotName.Valid {
 			slotNameLabel = slotName.String
 		}
+		slotTypeLabel := "unknown"
+		if slotType.Valid {
+			slotTypeLabel = slotType.String
+		}
 
 		var walLSNMetric float64
 		if walLSN.Valid {
 			walLSNMetric = walLSN.Float64
 		}
 		ch <- prometheus.MustNewConstMetric(
 			pgReplicationSlotCurrentWalDesc,
-			prometheus.GaugeValue, walLSNMetric, slotNameLabel,
+			prometheus.GaugeValue, walLSNMetric, slotNameLabel, slotTypeLabel,
 		)
 		if isActive.Valid && isActive.Bool {
 			var flushLSNMetric float64
@@ -118,13 +146,27 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 			}
 			ch <- prometheus.MustNewConstMetric(
 				pgReplicationSlotCurrentFlushDesc,
-				prometheus.GaugeValue, flushLSNMetric, slotNameLabel,
+				prometheus.GaugeValue, flushLSNMetric, slotNameLabel, slotTypeLabel,
 			)
 		}
 		ch <- prometheus.MustNewConstMetric(
 			pgReplicationSlotIsActiveDesc,
-			prometheus.GaugeValue, isActiveValue, slotNameLabel,
+			prometheus.GaugeValue, isActiveValue, slotNameLabel, slotTypeLabel,
 		)
+
+		if safeWalSize.Valid {
+			ch <- prometheus.MustNewConstMetric(
+				pgReplicationSlotSafeWal,
+				prometheus.GaugeValue, float64(safeWalSize.Int64), slotNameLabel, slotTypeLabel,
+			)
+		}
+
+		if walStatus.Valid {
+			ch <- prometheus.MustNewConstMetric(
+				pgReplicationSlotWalStatus,
+				prometheus.GaugeValue, 1, slotNameLabel, slotTypeLabel, walStatus.String,
+			)
+		}
 	}
 	return rows.Err()
 }