Merge remote-tracking branch 'grafana/master' into exporter-package-v0.15.0

Author: dehaansa

.github/workflows/golangci-lint.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
       - name: install Go
         uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
       - name: Install snmp_exporter/generator dependencies
         run: sudo apt-get update && sudo apt-get -y install libsnmp-dev
         if: github.repository == 'prometheus/snmp_exporter'

README.md

@@ -17,10 +17,29 @@ docker run --net=host -it --rm -e POSTGRES_PASSWORD=password postgres
 # Connect to it
 docker run \
   --net=host \
-  -e DATA_SOURCE_NAME="postgresql://postgres:password@localhost:5432/postgres?sslmode=disable" \
+  -e DATA_SOURCE_URI="localhost:5432/postgres?sslmode=disable" \
+  -e DATA_SOURCE_USER=postgres \
+  -e DATA_SOURCE_PASS=password \
   quay.io/prometheuscommunity/postgres-exporter
 ```
 
+Test with:
+```bash
+curl "http://localhost:9187/metrics"
+```
+
+Example Prometheus config:
+```yaml
+scrape_configs:
+  - job_name: postgres
+    static_configs:
+      - targets: ["127.0.0.1:9187"] # Replace IP with the hostname of the docker container if you're running the container in a separate network
+```
+
+Now use the DATA_SOURCE_PASS_FILE with a mounted file containing the password to prevent having the password in an environment variable.
+
+The container process runs with uid/gid 65534 (important for file permissions).
+
 ## Multi-Target Support (BETA)
 **This Feature is in beta and may require changes in future releases. Feedback is welcome.**
 
@@ -208,7 +227,7 @@ The following environment variables configure the exporter:
 * `DATA_SOURCE_URI`
    an alternative to `DATA_SOURCE_NAME` which exclusively accepts the hostname
    without a username and password component. For example, `my_pg_hostname` or
-   `my_pg_hostname?sslmode=disable`.
+   `my_pg_hostname:5432/postgres?sslmode=disable`.
 
 * `DATA_SOURCE_URI_FILE`
    The same as above but reads the URI from a file.

cmd/postgres_exporter/main.go

@@ -94,7 +94,7 @@ func main() {
 	}
 
 	excludedDatabases := strings.Split(*excludeDatabases, ",")
-	rootFallbackLogger.Log("msg", "Excluded databases", "databases", fmt.Sprintf("%v", excludedDatabases))
+	level.Info(rootFallbackLogger).Log("msg", "Excluded databases", "databases", fmt.Sprintf("%v", excludedDatabases))
 
 	if *queriesPath != "" {
 		level.Warn(rootFallbackLogger).Log("msg", "The extended queries.yaml config is DEPRECATED", "file", *queriesPath)
@@ -142,6 +142,7 @@ func main() {
 	if err != nil {
 		level.Warn(rootFallbackLogger).Log("msg", "Failed to create PostgresCollector", "err", err.Error())
 	} else {
+		defer pe.Close()
 		prometheus.MustRegister(pe)
 	}
 

cmd/postgres_exporter/server.go

@@ -178,6 +178,7 @@ func (s *Servers) GetServer(dsn string) (*Server, error) {
 			s.servers[dsn] = server
 		}
 		if err = server.Ping(); err != nil {
+			server.Close()
 			delete(s.servers, dsn)
 			time.Sleep(time.Duration(errCount) * time.Second)
 			continue

collector/collector.go

@@ -164,11 +164,11 @@ func (p PostgresCollector) Collect(ch chan<- prometheus.Metric) {
 
 	// Set up the database connection for the collector.
 	err := inst.setup()
+	defer inst.Close()
 	if err != nil {
 		level.Error(p.logger).Log("msg", "Error opening connection to database", "err", err)
 		return
 	}
-	defer inst.Close()
 
 	wg := sync.WaitGroup{}
 	wg.Add(len(p.Collectors))
@@ -181,6 +181,10 @@ func (p PostgresCollector) Collect(ch chan<- prometheus.Metric) {
 	wg.Wait()
 }
 
+func (p *PostgresCollector) Close() error {
+	return p.instance.Close()
+}
+
 func execute(ctx context.Context, name string, c Collector, instance *instance, ch chan<- prometheus.Metric, logger log.Logger) {
 	begin := time.Now()
 	err := c.Update(ctx, instance, ch)

collector/pg_database.go

@@ -53,12 +53,21 @@ var (
 		"Disk space used by the database",
 		[]string{"datname"}, nil,
 	)
+	pgDatabaseConnectionLimitsDesc = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			databaseSubsystem,
+			"connection_limit",
+		),
+		"Connection limit set for the database",
+		[]string{"datname"}, nil,
+	)
 
-	pgDatabaseQuery     = "SELECT pg_database.datname FROM pg_database;"
+	pgDatabaseQuery     = "SELECT pg_database.datname, pg_database.datconnlimit FROM pg_database;"
 	pgDatabaseSizeQuery = "SELECT pg_database_size($1)"
 )
 
-// Update implements Collector and exposes database size.
+// Update implements Collector and exposes database size and connection limits.
 // It is called by the Prometheus registry when collecting metrics.
 // The list of databases is retrieved from pg_database and filtered
 // by the excludeDatabase config parameter. The tradeoff here is that
@@ -81,21 +90,32 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 
 	for rows.Next() {
 		var datname sql.NullString
-		if err := rows.Scan(&datname); err != nil {
+		var connLimit sql.NullInt64
+		if err := rows.Scan(&datname, &connLimit); err != nil {
 			return err
 		}
 
 		if !datname.Valid {
 			continue
 		}
+		database := datname.String
 		// Ignore excluded databases
 		// Filtering is done here instead of in the query to avoid
 		// a complicated NOT IN query with a variable number of parameters
-		if sliceContains(c.excludedDatabases, datname.String) {
+		if sliceContains(c.excludedDatabases, database) {
 			continue
 		}
 
-		databases = append(databases, datname.String)
+		databases = append(databases, database)
+
+		connLimitMetric := 0.0
+		if connLimit.Valid {
+			connLimitMetric = float64(connLimit.Int64)
+		}
+		ch <- prometheus.MustNewConstMetric(
+			pgDatabaseConnectionLimitsDesc,
+			prometheus.GaugeValue, connLimitMetric, database,
+		)
 	}
 
 	// Query the size of the databases
@@ -114,6 +134,7 @@ func (c PGDatabaseCollector) Update(ctx context.Context, instance *instance, ch
 			pgDatabaseSizeDesc,
 			prometheus.GaugeValue, sizeMetric, datname,
 		)
+
 	}
 	return rows.Err()
 }

collector/pg_database_test.go

@@ -31,8 +31,8 @@ func TestPGDatabaseCollector(t *testing.T) {
 
 	inst := &instance{db: db}
 
-	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname"}).
-		AddRow("postgres"))
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname", "datconnlimit"}).
+		AddRow("postgres", 15))
 
 	mock.ExpectQuery(sanitizeQuery(pgDatabaseSizeQuery)).WithArgs("postgres").WillReturnRows(sqlmock.NewRows([]string{"pg_database_size"}).
 		AddRow(1024))
@@ -47,6 +47,7 @@ func TestPGDatabaseCollector(t *testing.T) {
 	}()
 
 	expected := []MetricResult{
+		{labels: labelMap{"datname": "postgres"}, value: 15, metricType: dto.MetricType_GAUGE},
 		{labels: labelMap{"datname": "postgres"}, value: 1024, metricType: dto.MetricType_GAUGE},
 	}
 	convey.Convey("Metrics comparison", t, func() {
@@ -71,8 +72,8 @@ func TestPGDatabaseCollectorNullMetric(t *testing.T) {
 
 	inst := &instance{db: db}
 
-	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname"}).
-		AddRow("postgres"))
+	mock.ExpectQuery(sanitizeQuery(pgDatabaseQuery)).WillReturnRows(sqlmock.NewRows([]string{"datname", "datconnlimit"}).
+		AddRow("postgres", nil))
 
 	mock.ExpectQuery(sanitizeQuery(pgDatabaseSizeQuery)).WithArgs("postgres").WillReturnRows(sqlmock.NewRows([]string{"pg_database_size"}).
 		AddRow(nil))
@@ -88,6 +89,7 @@ func TestPGDatabaseCollectorNullMetric(t *testing.T) {
 
 	expected := []MetricResult{
 		{labels: labelMap{"datname": "postgres"}, value: 0, metricType: dto.MetricType_GAUGE},
+		{labels: labelMap{"datname": "postgres"}, value: 0, metricType: dto.MetricType_GAUGE},
 	}
 	convey.Convey("Metrics comparison", t, func() {
 		for _, expect := range expected {

collector/pg_replication_slot.go

@@ -43,7 +43,7 @@ var (
 			"slot_current_wal_lsn",
 		),
 		"current wal lsn value",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
 	)
 	pgReplicationSlotCurrentFlushDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(
@@ -52,7 +52,7 @@ var (
 			"slot_confirmed_flush_lsn",
 		),
 		"last lsn confirmed flushed to the replication slot",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
 	)
 	pgReplicationSlotIsActiveDesc = prometheus.NewDesc(
 		prometheus.BuildFQName(
@@ -61,18 +61,39 @@ var (
 			"slot_is_active",
 		),
 		"whether the replication slot is active or not",
-		[]string{"slot_name"}, nil,
+		[]string{"slot_name", "slot_type"}, nil,
+	)
+	pgReplicationSlotSafeWal = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			replicationSlotSubsystem,
+			"safe_wal_size_bytes",
+		),
+		"number of bytes that can be written to WAL such that this slot is not in danger of getting in state lost",
+		[]string{"slot_name", "slot_type"}, nil,
+	)
+	pgReplicationSlotWalStatus = prometheus.NewDesc(
+		prometheus.BuildFQName(
+			namespace,
+			replicationSlotSubsystem,
+			"wal_status",
+		),
+		"availability of WAL files claimed by this slot",
+		[]string{"slot_name", "slot_type", "wal_status"}, nil,
 	)
 
 	pgReplicationSlotQuery = `SELECT
 		slot_name,
-		CASE WHEN pg_is_in_recovery() THEN 
+		slot_type,
+		CASE WHEN pg_is_in_recovery() THEN
 		    pg_last_wal_receive_lsn() - '0/0'
-		ELSE 
-		    pg_current_wal_lsn() - '0/0' 
+		ELSE
+		    pg_current_wal_lsn() - '0/0'
 		END AS current_wal_lsn,
-		COALESCE(confirmed_flush_lsn, '0/0') - '0/0',
-		active
+		COALESCE(confirmed_flush_lsn, '0/0') - '0/0' AS confirmed_flush_lsn,
+		active,
+		safe_wal_size,
+		wal_status
 	FROM pg_replication_slots;`
 )
 
@@ -87,10 +108,13 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 
 	for rows.Next() {
 		var slotName sql.NullString
+		var slotType sql.NullString
 		var walLSN sql.NullFloat64
 		var flushLSN sql.NullFloat64
 		var isActive sql.NullBool
-		if err := rows.Scan(&slotName, &walLSN, &flushLSN, &isActive); err != nil {
+		var safeWalSize sql.NullInt64
+		var walStatus sql.NullString
+		if err := rows.Scan(&slotName, &slotType, &walLSN, &flushLSN, &isActive, &safeWalSize, &walStatus); err != nil {
 			return err
 		}
 
@@ -102,14 +126,18 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 		if slotName.Valid {
 			slotNameLabel = slotName.String
 		}
+		slotTypeLabel := "unknown"
+		if slotType.Valid {
+			slotTypeLabel = slotType.String
+		}
 
 		var walLSNMetric float64
 		if walLSN.Valid {
 			walLSNMetric = walLSN.Float64
 		}
 		ch <- prometheus.MustNewConstMetric(
 			pgReplicationSlotCurrentWalDesc,
-			prometheus.GaugeValue, walLSNMetric, slotNameLabel,
+			prometheus.GaugeValue, walLSNMetric, slotNameLabel, slotTypeLabel,
 		)
 		if isActive.Valid && isActive.Bool {
 			var flushLSNMetric float64
@@ -118,13 +146,27 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance
 			}
 			ch <- prometheus.MustNewConstMetric(
 				pgReplicationSlotCurrentFlushDesc,
-				prometheus.GaugeValue, flushLSNMetric, slotNameLabel,
+				prometheus.GaugeValue, flushLSNMetric, slotNameLabel, slotTypeLabel,
 			)
 		}
 		ch <- prometheus.MustNewConstMetric(
 			pgReplicationSlotIsActiveDesc,
-			prometheus.GaugeValue, isActiveValue, slotNameLabel,
+			prometheus.GaugeValue, isActiveValue, slotNameLabel, slotTypeLabel,
 		)
+
+		if safeWalSize.Valid {
+			ch <- prometheus.MustNewConstMetric(
+				pgReplicationSlotSafeWal,
+				prometheus.GaugeValue, float64(safeWalSize.Int64), slotNameLabel, slotTypeLabel,
+			)
+		}
+
+		if walStatus.Valid {
+			ch <- prometheus.MustNewConstMetric(
+				pgReplicationSlotWalStatus,
+				prometheus.GaugeValue, 1, slotNameLabel, slotTypeLabel, walStatus.String,
+			)
+		}
 	}
 	return rows.Err()
 }