From 6323907f1165342b81b6585ff126361360258e2f Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 16 Jun 2023 12:12:00 -0700 Subject: [PATCH 01/39] Add wal receiver Signed-off-by: Felix Yuan --- collector/pg_stat_walreceiver.go | 212 +++++++++++++++++++++++++++++++ 1 file changed, 212 insertions(+) create mode 100644 collector/pg_stat_walreceiver.go diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go new file mode 100644 index 000000000..56b03fdf9 --- /dev/null +++ b/collector/pg_stat_walreceiver.go @@ -0,0 +1,212 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) +} + +type PGStatWalReceiverCollector struct { + log log.Logger +} + +const statWalReceiverSubsystem = "stat_wal_receiver" + +func NewPGStatWalReceiverCollector(collectorConfig) (Collector, error) { + return &PGStatWalReceiverCollector{}, nil +} + +var ( + statWalReceiverStatus = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "status"), + "Activity status of the WAL receiver process", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverReceiveStartLsn = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "receive_start_lsn"), + "First write-ahead log location used when WAL receiver is started represented as a decimal", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverReceiveStartTli = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "receive_start_tli"), + "First timeline number used when WAL receiver is started", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverFlushedLSN = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "flushed_lsn"), + "Last write-ahead log location already received and flushed to disk, the initial value of this field being the first log location used when WAL receiver is started represented as a decimal", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverReceivedTli = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "received_tli"), + "Timeline number of last write-ahead log location received and flushed to disk", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverLastMsgSendTime = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "last_msg_send_time"), + "Send time of last message received from origin WAL sender", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverLastMsgReceiptTime = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "last_msg_receipt_time"), + "Send time of last message received from origin WAL sender", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverLatestEndLsn = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "latest_end_lsn"), + "Last write-ahead log location reported to origin WAL sender as integer", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverLatestEndTime = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "latest_end_time"), + "Time of last write-ahead log location reported to origin WAL sender", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + statWalReceiverUpstreamNode = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "upstream_node"), + "Node ID of the upstream node", + []string{"upstream_connection", "slot_name"}, + prometheus.Labels{}, + ) + + pgStatWalReceiverQuery = ` + SELECT + trim(both '''' from substring(conninfo from 'host=([^ ]*)')) as upstream_host, + slot_name, + case status + when 'stopped' then 0 + when 'starting' then 1 + when 'streaming' then 2 + when 'waiting' then 3 + when 'restarting' then 4 + when 'stopping' then 5 else -1 + end as status, + (receive_start_lsn- '0/0') % (2^52)::bigint as receive_start_lsn, + receive_start_tli, + (flushed_lsn- '0/0') % (2^52)::bigint as flushed_lsn, + received_tli, + extract(epoch from last_msg_send_time) as last_msg_send_time, + extract(epoch from last_msg_receipt_time) as last_msg_receipt_time, + (latest_end_lsn - '0/0') % (2^52)::bigint as latest_end_lsn, + extract(epoch from latest_end_time) as latest_end_time, + substring(slot_name from 'repmgr_slot_([0-9]*)') as upstream_node + FROM pg_catalog.pg_stat_wal_receiver + ` +) + +func (c *PGStatWalReceiverCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.QueryContext(ctx, pgStatWalReceiverQuery) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var upstreamHost string + var slotName string + var status int + var receiveStartLsn int64 + var receiveStartTli int64 + var flushedLsn int64 + var receivedTli int64 + var lastMsgSendTime float64 + var lastMsgReceiptTime float64 + var latestEndLsn int64 + var latestEndTime float64 + var upstreamNode int + + if err := rows.Scan(&upstreamHost, &slotName, &status, &receiveStartLsn, &receiveStartTli, &flushedLsn, &receivedTli, &lastMsgSendTime, &lastMsgReceiptTime, &latestEndLsn, &latestEndTime, &upstreamNode); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statWalReceiverStatus, + prometheus.GaugeValue, + float64(status), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverReceiveStartLsn, + prometheus.CounterValue, + float64(receiveStartLsn), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverReceiveStartTli, + prometheus.GaugeValue, + float64(receiveStartTli), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverFlushedLSN, + prometheus.CounterValue, + float64(flushedLsn), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverReceivedTli, + prometheus.GaugeValue, + float64(receivedTli), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverLastMsgSendTime, + prometheus.CounterValue, + float64(lastMsgSendTime), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverLastMsgReceiptTime, + prometheus.CounterValue, + float64(lastMsgReceiptTime), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverLatestEndLsn, + prometheus.CounterValue, + float64(latestEndLsn), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverLatestEndTime, + prometheus.CounterValue, + float64(latestEndTime), + upstreamHost, slotName) + + ch <- prometheus.MustNewConstMetric( + statWalReceiverUpstreamNode, + prometheus.GaugeValue, + float64(upstreamNode), + upstreamHost, slotName) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} From 8ee74db738c20a3a939c62dfe901bfbaa879bc69 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Tue, 20 Jun 2023 14:56:00 -0700 Subject: [PATCH 02/39] Finish pg_stat_walreceiver Signed-off-by: Felix Yuan --- collector/collector_test.go | 3 + collector/pg_stat_walreceiver.go | 91 ++++++++++--- collector/pg_stat_walreceiver_test.go | 181 ++++++++++++++++++++++++++ 3 files changed, 256 insertions(+), 19 deletions(-) create mode 100644 collector/pg_stat_walreceiver_test.go diff --git a/collector/collector_test.go b/collector/collector_test.go index 061de8895..5ac7c27be 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -50,7 +50,10 @@ func sanitizeQuery(q string) string { q = strings.Join(strings.Fields(q), " ") q = strings.Replace(q, "(", "\\(", -1) q = strings.Replace(q, ")", "\\)", -1) + q = strings.Replace(q, "[", "\\[", -1) + q = strings.Replace(q, "]", "\\]", -1) q = strings.Replace(q, "*", "\\*", -1) + q = strings.Replace(q, "^", "\\^", -1) q = strings.Replace(q, "$", "\\$", -1) return q } diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index 56b03fdf9..bcebe72bd 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -38,65 +38,97 @@ var ( statWalReceiverStatus = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "status"), "Activity status of the WAL receiver process", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverReceiveStartLsn = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "receive_start_lsn"), "First write-ahead log location used when WAL receiver is started represented as a decimal", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverReceiveStartTli = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "receive_start_tli"), "First timeline number used when WAL receiver is started", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverFlushedLSN = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "flushed_lsn"), "Last write-ahead log location already received and flushed to disk, the initial value of this field being the first log location used when WAL receiver is started represented as a decimal", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverReceivedTli = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "received_tli"), "Timeline number of last write-ahead log location received and flushed to disk", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverLastMsgSendTime = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "last_msg_send_time"), "Send time of last message received from origin WAL sender", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverLastMsgReceiptTime = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "last_msg_receipt_time"), "Send time of last message received from origin WAL sender", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverLatestEndLsn = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "latest_end_lsn"), "Last write-ahead log location reported to origin WAL sender as integer", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverLatestEndTime = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "latest_end_time"), "Time of last write-ahead log location reported to origin WAL sender", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) statWalReceiverUpstreamNode = prometheus.NewDesc( prometheus.BuildFQName(namespace, statWalReceiverSubsystem, "upstream_node"), "Node ID of the upstream node", - []string{"upstream_connection", "slot_name"}, + []string{"upstream_host", "slot_name"}, prometheus.Labels{}, ) - pgStatWalReceiverQuery = ` + pgStatWalColumnQuery = ` + SELECT + column_name + FROM information_schema.columns + WHERE + table_name = 'pg_stat_wal_receiver' and + column_name = 'flushed_lsn' + ` + + pgStatWalReceiverQueryWithNoFlushedLSN = ` + SELECT + trim(both '''' from substring(conninfo from 'host=([^ ]*)')) as upstream_host, + slot_name, + case status + when 'stopped' then 0 + when 'starting' then 1 + when 'streaming' then 2 + when 'waiting' then 3 + when 'restarting' then 4 + when 'stopping' then 5 else -1 + end as status, + (receive_start_lsn- '0/0') % (2^52)::bigint as receive_start_lsn, + receive_start_tli, + received_tli, + extract(epoch from last_msg_send_time) as last_msg_send_time, + extract(epoch from last_msg_receipt_time) as last_msg_receipt_time, + (latest_end_lsn - '0/0') % (2^52)::bigint as latest_end_lsn, + extract(epoch from latest_end_time) as latest_end_time, + substring(slot_name from 'repmgr_slot_([0-9]*)') as upstream_node + FROM pg_catalog.pg_stat_wal_receiver + ` + + pgStatWalReceiverQueryWithFlushedLSN = ` SELECT trim(both '''' from substring(conninfo from 'host=([^ ]*)')) as upstream_host, slot_name, @@ -122,7 +154,20 @@ var ( ) func (c *PGStatWalReceiverCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { - rows, err := db.QueryContext(ctx, pgStatWalReceiverQuery) + hasFlushedLSNRows, err := db.QueryContext(ctx, pgStatWalColumnQuery) + if err != nil { + return err + } + + defer hasFlushedLSNRows.Close() + hasFlushedLSN := hasFlushedLSNRows.Next() + var query string + if hasFlushedLSN { + query = pgStatWalReceiverQueryWithFlushedLSN + } else { + query = pgStatWalReceiverQueryWithNoFlushedLSN + } + rows, err := db.QueryContext(ctx, query) if err != nil { return err } @@ -141,8 +186,14 @@ func (c *PGStatWalReceiverCollector) Update(ctx context.Context, db *sql.DB, ch var latestEndTime float64 var upstreamNode int - if err := rows.Scan(&upstreamHost, &slotName, &status, &receiveStartLsn, &receiveStartTli, &flushedLsn, &receivedTli, &lastMsgSendTime, &lastMsgReceiptTime, &latestEndLsn, &latestEndTime, &upstreamNode); err != nil { - return err + if hasFlushedLSN { + if err := rows.Scan(&upstreamHost, &slotName, &status, &receiveStartLsn, &receiveStartTli, &flushedLsn, &receivedTli, &lastMsgSendTime, &lastMsgReceiptTime, &latestEndLsn, &latestEndTime, &upstreamNode); err != nil { + return err + } + } else { + if err := rows.Scan(&upstreamHost, &slotName, &status, &receiveStartLsn, &receiveStartTli, &receivedTli, &lastMsgSendTime, &lastMsgReceiptTime, &latestEndLsn, &latestEndTime, &upstreamNode); err != nil { + return err + } } ch <- prometheus.MustNewConstMetric( @@ -163,11 +214,13 @@ func (c *PGStatWalReceiverCollector) Update(ctx context.Context, db *sql.DB, ch float64(receiveStartTli), upstreamHost, slotName) - ch <- prometheus.MustNewConstMetric( - statWalReceiverFlushedLSN, - prometheus.CounterValue, - float64(flushedLsn), - upstreamHost, slotName) + if hasFlushedLSN { + ch <- prometheus.MustNewConstMetric( + statWalReceiverFlushedLSN, + prometheus.CounterValue, + float64(flushedLsn), + upstreamHost, slotName) + } ch <- prometheus.MustNewConstMetric( statWalReceiverReceivedTli, diff --git a/collector/pg_stat_walreceiver_test.go b/collector/pg_stat_walreceiver_test.go new file mode 100644 index 000000000..90411f380 --- /dev/null +++ b/collector/pg_stat_walreceiver_test.go @@ -0,0 +1,181 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStatWalReceiverCollectorWithFlushedLSN(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + infoSchemaColumns := []string{ + "column_name", + } + + infoSchemaRows := sqlmock.NewRows(infoSchemaColumns). + AddRow( + "flushed_lsn", + ) + + mock.ExpectQuery(sanitizeQuery(pgStatWalColumnQuery)).WillReturnRows(infoSchemaRows) + + columns := []string{ + "upstream_host", + "slot_name", + "status", + "receive_start_lsn", + "receive_start_tli", + "flushed_lsn", + "received_tli", + "last_msg_send_time", + "last_msg_receipt_time", + "latest_end_lsn", + "latest_end_time", + "upstream_node", + } + rows := sqlmock.NewRows(columns). + AddRow( + "foo", + "bar", + 2, + 1200668684563608, + 1687321285, + 1200668684563609, + 1687321280, + 1687321275, + 1687321276, + 1200668684563610, + 1687321277, + 5, + ) + mock.ExpectQuery(sanitizeQuery(pgStatWalReceiverQueryWithFlushedLSN)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatWalReceiverCollector{} + + if err := c.Update(context.Background(), db, ch); err != nil { + t.Errorf("Error calling PgStatWalReceiverCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 2, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1200668684563608, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321285, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1200668684563609, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321280, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321275, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321276, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1200668684563610, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321277, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 5, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } + +} + +func TestPGStatWalReceiverCollectorWithNoFlushedLSN(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + infoSchemaColumns := []string{ + "column_name", + } + + infoSchemaRows := sqlmock.NewRows(infoSchemaColumns) + + mock.ExpectQuery(sanitizeQuery(pgStatWalColumnQuery)).WillReturnRows(infoSchemaRows) + + columns := []string{ + "upstream_host", + "slot_name", + "status", + "receive_start_lsn", + "receive_start_tli", + "received_tli", + "last_msg_send_time", + "last_msg_receipt_time", + "latest_end_lsn", + "latest_end_time", + "upstream_node", + } + rows := sqlmock.NewRows(columns). + AddRow( + "foo", + "bar", + 2, + 1200668684563608, + 1687321285, + 1687321280, + 1687321275, + 1687321276, + 1200668684563610, + 1687321277, + 5, + ) + mock.ExpectQuery(sanitizeQuery(pgStatWalReceiverQueryWithNoFlushedLSN)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatWalReceiverCollector{} + + if err := c.Update(context.Background(), db, ch); err != nil { + t.Errorf("Error calling PgStatWalReceiverCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 2, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1200668684563608, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321285, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321280, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321275, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321276, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1200668684563610, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 1687321277, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "foo", "slot_name": "bar"}, value: 5, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } + +} From 01176535c31cff96277e80175b6078e8ca458db5 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Tue, 20 Jun 2023 15:16:57 -0700 Subject: [PATCH 03/39] Add pg_archiver Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 81 +++++++++++++++++++++++++++++++++++ collector/pg_archiver_test.go | 57 ++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 collector/pg_archiver.go create mode 100644 collector/pg_archiver_test.go diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go new file mode 100644 index 000000000..1efe6b9e0 --- /dev/null +++ b/collector/pg_archiver.go @@ -0,0 +1,81 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) +} + +type PGArchiverCollector struct { + log log.Logger +} + +const archiverSubsystem = "archiver_receiver" + +func NewPGArchiverCollector(collectorConfig) (Collector, error) { + return &PGArchiverCollector{}, nil +} + +var ( + pgArchiverPendingWalCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, archiverSubsystem, "pending_wal_count"), + "Number of WAL files waiting to be archived", + []string{}, prometheus.Labels{}, + ) + + pgArchiverQuery = ` + WITH + current_wal_file AS ( + SELECT CASE WHEN NOT pg_is_in_recovery() THEN pg_walfile_name(pg_current_wal_insert_lsn()) ELSE NULL END pg_walfile_name + ), + current_wal AS ( + SELECT + ('x'||substring(pg_walfile_name,9,8))::bit(32)::int log, + ('x'||substring(pg_walfile_name,17,8))::bit(32)::int seg, + pg_walfile_name + FROM current_wal_file + ), + archive_wal AS( + SELECT + ('x'||substring(last_archived_wal,9,8))::bit(32)::int log, + ('x'||substring(last_archived_wal,17,8))::bit(32)::int seg, + last_archived_wal + FROM pg_stat_archiver + ) + SELECT coalesce(((cw.log - aw.log) * 256) + (cw.seg-aw.seg),'NaN'::float) as pending_wal_count FROM current_wal cw, archive_wal aw + ` +) + +func (c *PGArchiverCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { + row := db.QueryRowContext(ctx, + pgArchiverQuery) + var pendingWalCount float64 + err := row.Scan(&pendingWalCount) + if err != nil { + return err + } + ch <- prometheus.MustNewConstMetric( + pgArchiverPendingWalCount, + prometheus.GaugeValue, + pendingWalCount, + ) + return nil +} diff --git a/collector/pg_archiver_test.go b/collector/pg_archiver_test.go new file mode 100644 index 000000000..b777dbf8d --- /dev/null +++ b/collector/pg_archiver_test.go @@ -0,0 +1,57 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgArchiverCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + mock.ExpectQuery(sanitizeQuery(pgArchiverQuery)).WillReturnRows(sqlmock.NewRows([]string{"pending_wal_count"}). + AddRow(5)) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGArchiverCollector{} + + if err := c.Update(context.Background(), db, ch); err != nil { + t.Errorf("Error calling PGArchiverCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{}, value: 5, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From eecb3ba5eba80540300aafe0ffad29c9d46a4f37 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Tue, 20 Jun 2023 15:41:13 -0700 Subject: [PATCH 04/39] Missed a label declaration Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index 1efe6b9e0..108a685f6 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -28,7 +28,7 @@ type PGArchiverCollector struct { log log.Logger } -const archiverSubsystem = "archiver_receiver" +const archiverSubsystem = "archiver" func NewPGArchiverCollector(collectorConfig) (Collector, error) { return &PGArchiverCollector{}, nil From a6998af9bd21f086d47c1be975d7e21e1c379097 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Tue, 20 Jun 2023 16:12:37 -0700 Subject: [PATCH 05/39] Add stat_user_indexes Signed-off-by: Felix Yuan --- collector/pg_stat_user_indexes.go | 108 +++++++++++++++++++++++++ collector/pg_stat_user_indexes_test.go | 67 +++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 collector/pg_stat_user_indexes.go create mode 100644 collector/pg_stat_user_indexes_test.go diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go new file mode 100644 index 000000000..b6157acb1 --- /dev/null +++ b/collector/pg_stat_user_indexes.go @@ -0,0 +1,108 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) +} + +type PGStatUserIndexesCollector struct { + log log.Logger +} + +const statUserIndexesSubsystem = "stat_user_indexes" + +func NewPGStatUserIndexesCollector(collectorConfig) (Collector, error) { + return &PGStatUserIndexesCollector{}, nil +} + +var ( + statUserIndexesIdxScan = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_scan"), + "Number of index scans initiated on this index", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + statUserIndexesIdxTupRead = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_read"), + "Number of index entries returned by scans on this index", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + statUserIndexesIdxTupFetch = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_fetch"), + "Number of live table rows fetched by simple index scans using this index", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + statUserIndexesQuery = ` + SELECT + schemaname, + relname, + indexrelname, + idx_scan, + idx_tup_read, + idx_tup_fetch + FROM pg_stat_user_indexes + ` +) + +func (c *PGStatUserIndexesCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.QueryContext(ctx, + statUserIndexesQuery) + + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var schemaname, relname, indexrelname string + var idxScan, idxTupRead, idxTupFetch float64 + + if err := rows.Scan(&schemaname, &relname, &indexrelname, &idxScan, &idxTupRead, &idxTupFetch); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxScan, + prometheus.CounterValue, + idxScan, + schemaname, relname, indexrelname, + ) + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxTupRead, + prometheus.CounterValue, + idxTupRead, + schemaname, relname, indexrelname, + ) + ch <- prometheus.MustNewConstMetric( + statUserIndexesIdxTupFetch, + prometheus.CounterValue, + idxTupFetch, + schemaname, relname, indexrelname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_user_indexes_test.go b/collector/pg_stat_user_indexes_test.go new file mode 100644 index 000000000..9c20a8f19 --- /dev/null +++ b/collector/pg_stat_user_indexes_test.go @@ -0,0 +1,67 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgUserIndexesCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "idx_scan", + "idx_tup_read", + "idx_tup_fetch", + } + rows := sqlmock.NewRows(columns). + AddRow("public", "pgbench_accounts", "pgbench_accounts_pkey", 5, 6, 7) + + mock.ExpectQuery(sanitizeQuery(statUserIndexesQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatUserIndexesCollector{} + + if err := c.Update(context.Background(), db, ch); err != nil { + t.Errorf("Error calling PGStatUserIndexesCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "public", "relname": "pgbench_accounts", "indexrelname": "pgbench_accounts_pkey"}, value: 5, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "public", "relname": "pgbench_accounts", "indexrelname": "pgbench_accounts_pkey"}, value: 6, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "public", "relname": "pgbench_accounts", "indexrelname": "pgbench_accounts_pkey"}, value: 7, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From dff617a0ce2c0ed33daf45345945b130fc76597a Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 15:10:02 -0700 Subject: [PATCH 06/39] Add pg statio user queries and fix a test name Signed-off-by: Felix Yuan --- collector/pg_stat_user_indexes_test.go | 2 +- collector/pg_statio_user_indexes.go | 95 ++++++++++++++++++++++++ collector/pg_statio_user_indexes_test.go | 65 ++++++++++++++++ 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 collector/pg_statio_user_indexes.go create mode 100644 collector/pg_statio_user_indexes_test.go diff --git a/collector/pg_stat_user_indexes_test.go b/collector/pg_stat_user_indexes_test.go index 9c20a8f19..a17bb86ea 100644 --- a/collector/pg_stat_user_indexes_test.go +++ b/collector/pg_stat_user_indexes_test.go @@ -22,7 +22,7 @@ import ( "github.com/smartystreets/goconvey/convey" ) -func TestPgUserIndexesCollector(t *testing.T) { +func TestPgStatUserIndexesCollector(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go new file mode 100644 index 000000000..728832b7c --- /dev/null +++ b/collector/pg_statio_user_indexes.go @@ -0,0 +1,95 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "database/sql" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +func init() { + registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) +} + +type PGStatioUserIndexesCollector struct { + log log.Logger +} + +const statioUserIndexesSubsystem = "statio_user_indexes" + +func NewPGStatioUserIndexesCollector(collectorConfig) (Collector, error) { + return &PGStatioUserIndexesCollector{}, nil +} + +var ( + statioUserIndexesIdxBlksRead = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statioUserIndexesSubsystem, "idx_blks_read"), + "Number of disk blocks read from this index", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + statioUserIndexesIdxBlksHit = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statioUserIndexesSubsystem, "idx_blks_hit"), + "Number of buffer hits in this index", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + statioUserIndexesQuery = ` + SELECT + schemaname, + relname, + indexrelname, + idx_blks_read, + idx_blks_hit + FROM pg_statio_user_indexes + ` +) + +func (c *PGStatioUserIndexesCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { + rows, err := db.QueryContext(ctx, + statioUserIndexesQuery) + + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var schemaname, relname, indexrelname string + var idxBlksRead, idxBlksHit float64 + + if err := rows.Scan(&schemaname, &relname, &indexrelname, &idxBlksRead, &idxBlksHit); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statioUserIndexesIdxBlksRead, + prometheus.CounterValue, + idxBlksRead, + schemaname, relname, indexrelname, + ) + ch <- prometheus.MustNewConstMetric( + statioUserIndexesIdxBlksHit, + prometheus.CounterValue, + idxBlksHit, + schemaname, relname, indexrelname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_statio_user_indexes_test.go b/collector/pg_statio_user_indexes_test.go new file mode 100644 index 000000000..062fa8380 --- /dev/null +++ b/collector/pg_statio_user_indexes_test.go @@ -0,0 +1,65 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgStatioUserIndexesCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "idx_blks_read", + "idx_blks_hit", + } + rows := sqlmock.NewRows(columns). + AddRow("public", "pgtest_accounts", "pgtest_accounts_pkey", 8, 9) + + mock.ExpectQuery(sanitizeQuery(statioUserIndexesQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatioUserIndexesCollector{} + + if err := c.Update(context.Background(), db, ch); err != nil { + t.Errorf("Error calling PGStatioUserIndexesCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "public", "relname": "pgtest_accounts", "indexrelname": "pgtest_accounts_pkey"}, value: 8, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "public", "relname": "pgtest_accounts", "indexrelname": "pgtest_accounts_pkey"}, value: 9, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 4b3e95c26ddacbf6e1e714d1212328ba2aa7f267 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 15:22:36 -0700 Subject: [PATCH 07/39] Migrate from db -> instance Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 4 ++-- collector/pg_archiver_test.go | 3 ++- collector/pg_replication_slot_test.go | 1 + collector/pg_stat_user_indexes.go | 4 ++-- collector/pg_stat_user_indexes_test.go | 3 ++- collector/pg_stat_walreceiver.go | 4 ++-- collector/pg_stat_walreceiver_test.go | 6 ++++-- collector/pg_statio_user_indexes.go | 4 ++-- collector/pg_statio_user_indexes_test.go | 3 ++- 9 files changed, 19 insertions(+), 13 deletions(-) diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index 108a685f6..8c50235bd 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -14,7 +14,6 @@ package collector import ( "context" - "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -64,7 +63,8 @@ var ( ` ) -func (c *PGArchiverCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { +func (c *PGArchiverCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() row := db.QueryRowContext(ctx, pgArchiverQuery) var pendingWalCount float64 diff --git a/collector/pg_archiver_test.go b/collector/pg_archiver_test.go index b777dbf8d..a102d2634 100644 --- a/collector/pg_archiver_test.go +++ b/collector/pg_archiver_test.go @@ -29,6 +29,7 @@ func TestPgArchiverCollector(t *testing.T) { } defer db.Close() + inst := &instance{db: db} mock.ExpectQuery(sanitizeQuery(pgArchiverQuery)).WillReturnRows(sqlmock.NewRows([]string{"pending_wal_count"}). AddRow(5)) @@ -37,7 +38,7 @@ func TestPgArchiverCollector(t *testing.T) { defer close(ch) c := PGArchiverCollector{} - if err := c.Update(context.Background(), db, ch); err != nil { + if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PGArchiverCollector.Update: %s", err) } }() diff --git a/collector/pg_replication_slot_test.go b/collector/pg_replication_slot_test.go index 7e91ea261..6e390bd50 100644 --- a/collector/pg_replication_slot_test.go +++ b/collector/pg_replication_slot_test.go @@ -23,6 +23,7 @@ import ( ) func TestPgReplicationSlotCollectorActive(t *testing.T) { + db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index b6157acb1..b13b22744 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -14,7 +14,6 @@ package collector import ( "context" - "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -66,7 +65,8 @@ var ( ` ) -func (c *PGStatUserIndexesCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { +func (c *PGStatUserIndexesCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() rows, err := db.QueryContext(ctx, statUserIndexesQuery) diff --git a/collector/pg_stat_user_indexes_test.go b/collector/pg_stat_user_indexes_test.go index a17bb86ea..ce59355b7 100644 --- a/collector/pg_stat_user_indexes_test.go +++ b/collector/pg_stat_user_indexes_test.go @@ -28,6 +28,7 @@ func TestPgStatUserIndexesCollector(t *testing.T) { t.Fatalf("Error opening a stub db connection: %s", err) } defer db.Close() + inst := &instance{db: db} columns := []string{ "schemaname", "relname", @@ -46,7 +47,7 @@ func TestPgStatUserIndexesCollector(t *testing.T) { defer close(ch) c := PGStatUserIndexesCollector{} - if err := c.Update(context.Background(), db, ch); err != nil { + if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PGStatUserIndexesCollector.Update: %s", err) } }() diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index bcebe72bd..551ee7044 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -14,7 +14,6 @@ package collector import ( "context" - "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -153,7 +152,8 @@ var ( ` ) -func (c *PGStatWalReceiverCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { +func (c *PGStatWalReceiverCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() hasFlushedLSNRows, err := db.QueryContext(ctx, pgStatWalColumnQuery) if err != nil { return err diff --git a/collector/pg_stat_walreceiver_test.go b/collector/pg_stat_walreceiver_test.go index 90411f380..f9de36c47 100644 --- a/collector/pg_stat_walreceiver_test.go +++ b/collector/pg_stat_walreceiver_test.go @@ -29,6 +29,7 @@ func TestPGStatWalReceiverCollectorWithFlushedLSN(t *testing.T) { } defer db.Close() + inst := &instance{db: db} infoSchemaColumns := []string{ "column_name", } @@ -76,7 +77,7 @@ func TestPGStatWalReceiverCollectorWithFlushedLSN(t *testing.T) { defer close(ch) c := PGStatWalReceiverCollector{} - if err := c.Update(context.Background(), db, ch); err != nil { + if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PgStatWalReceiverCollector.Update: %s", err) } }() @@ -111,6 +112,7 @@ func TestPGStatWalReceiverCollectorWithNoFlushedLSN(t *testing.T) { } defer db.Close() + inst := &instance{db: db} infoSchemaColumns := []string{ "column_name", } @@ -153,7 +155,7 @@ func TestPGStatWalReceiverCollectorWithNoFlushedLSN(t *testing.T) { defer close(ch) c := PGStatWalReceiverCollector{} - if err := c.Update(context.Background(), db, ch); err != nil { + if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PgStatWalReceiverCollector.Update: %s", err) } }() diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go index 728832b7c..d4bb5f6db 100644 --- a/collector/pg_statio_user_indexes.go +++ b/collector/pg_statio_user_indexes.go @@ -14,7 +14,6 @@ package collector import ( "context" - "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -59,7 +58,8 @@ var ( ` ) -func (c *PGStatioUserIndexesCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { +func (c *PGStatioUserIndexesCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() rows, err := db.QueryContext(ctx, statioUserIndexesQuery) diff --git a/collector/pg_statio_user_indexes_test.go b/collector/pg_statio_user_indexes_test.go index 062fa8380..7f599c3e7 100644 --- a/collector/pg_statio_user_indexes_test.go +++ b/collector/pg_statio_user_indexes_test.go @@ -28,6 +28,7 @@ func TestPgStatioUserIndexesCollector(t *testing.T) { t.Fatalf("Error opening a stub db connection: %s", err) } defer db.Close() + inst := &instance{db: db} columns := []string{ "schemaname", "relname", @@ -45,7 +46,7 @@ func TestPgStatioUserIndexesCollector(t *testing.T) { defer close(ch) c := PGStatioUserIndexesCollector{} - if err := c.Update(context.Background(), db, ch); err != nil { + if err := c.Update(context.Background(), inst, ch); err != nil { t.Errorf("Error calling PGStatioUserIndexesCollector.Update: %s", err) } }() From 29736ce944d2a86fa1ebd9ecdf2ba1bf52b86639 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 15:35:29 -0700 Subject: [PATCH 08/39] pg_index_size query Signed-off-by: Felix Yuan --- collector/pg_index_size.go | 88 +++++++++++++++++++++++++++++++++ collector/pg_index_size_test.go | 64 ++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 collector/pg_index_size.go create mode 100644 collector/pg_index_size_test.go diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go new file mode 100644 index 000000000..54bc16f8a --- /dev/null +++ b/collector/pg_index_size.go @@ -0,0 +1,88 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const indexSizeSubsystem = "index_size" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGIndexSizeCollector struct { + log log.Logger +} + +func NewPGIndexSizeCollector(config collectorConfig) (Collector, error) { + return &PGIndexSizeCollector{log: config.logger}, nil +} + +var ( + indexSizeDesc = prometheus.NewDesc( + "pg_index_size", + "Size of the index as per pg_table_size function", + []string{"schemaname", "relname", "indexrelname"}, + prometheus.Labels{}, + ) + + indexSizeQuery = ` + SELECT + schemaname, + tablename as relname, + indexname as indexrelname, + pg_class.relpages * 8192::bigint as index_size + FROM + pg_indexes inner join pg_namespace on pg_indexes.schemaname = pg_namespace.nspname + inner join pg_class on pg_class.relnamespace = pg_namespace.oid and pg_class.relname = pg_indexes.indexname + WHERE + pg_indexes.schemaname != 'pg_catalog' + ` +) + +func (PGIndexSizeCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + indexSizeQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var schemaname, relname, indexrelname string + var indexSize float64 + + if err := rows.Scan(&schemaname, &relname, &indexrelname, &indexSize); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + indexSizeDesc, + prometheus.GaugeValue, + indexSize, + schemaname, relname, indexrelname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_index_size_test.go b/collector/pg_index_size_test.go new file mode 100644 index 000000000..b80680260 --- /dev/null +++ b/collector/pg_index_size_test.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgIndexSizeCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "index_size", + } + rows := sqlmock.NewRows(columns). + AddRow("public", "foo", "foo_key", 100) + + mock.ExpectQuery(sanitizeQuery(indexSizeQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGIndexSizeCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGIndexSizeCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "public", "relname": "foo", "indexrelname": "foo_key"}, value: 100, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 9198ec98f6a1aa28bd83e6028ebaa10da14d0128 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 15:47:27 -0700 Subject: [PATCH 09/39] Add total relation size query Signed-off-by: Felix Yuan --- collector/pg_total_relation_size.go | 84 ++++++++++++++++++++++++ collector/pg_total_relation_size_test.go | 63 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 collector/pg_total_relation_size.go create mode 100644 collector/pg_total_relation_size_test.go diff --git a/collector/pg_total_relation_size.go b/collector/pg_total_relation_size.go new file mode 100644 index 000000000..47160ff01 --- /dev/null +++ b/collector/pg_total_relation_size.go @@ -0,0 +1,84 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const totalRelationSizeSubsystem = "total_relation_size" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGTotalRelationSizeCollector struct { + log log.Logger +} + +func NewPGTotalRelationSizeCollector(config collectorConfig) (Collector, error) { + return &PGTotalRelationSizeCollector{log: config.logger}, nil +} + +var ( + totalRelationSizeBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, totalRelationSizeSubsystem, "bytes"), + "total disk space usage for the specified table and associated indexes", + []string{"schemaname", "relname"}, + prometheus.Labels{}, + ) + + totalRelationSizeQuery = ` + SELECT + relnamespace::regnamespace as schemaname, + relname as relname, + pg_total_relation_size(oid) bytes + FROM pg_class + WHERE relkind = 'r'; + ` +) + +func (PGTotalRelationSizeCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + totalRelationSizeQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var schemaname, relname string + var bytes float64 + + if err := rows.Scan(&schemaname, &relname, &bytes); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + totalRelationSizeBytes, + prometheus.GaugeValue, + bytes, + schemaname, relname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_total_relation_size_test.go b/collector/pg_total_relation_size_test.go new file mode 100644 index 000000000..2b835095c --- /dev/null +++ b/collector/pg_total_relation_size_test.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgTotalRelationSizeCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "bytes", + } + rows := sqlmock.NewRows(columns). + AddRow("public", "bar", 200) + + mock.ExpectQuery(sanitizeQuery(totalRelationSizeQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGTotalRelationSizeCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGTotalRelationSizeCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "public", "relname": "bar"}, value: 200, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From c7dad9d17d97668db2d7f3f52af9aa30814879b6 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 18:22:23 -0700 Subject: [PATCH 10/39] Add pg_blocked query Signed-off-by: Felix Yuan --- collector/pg_blocked.go | 91 ++++++++++++++++++++++++++++++++++++ collector/pg_blocked_test.go | 62 ++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 collector/pg_blocked.go create mode 100644 collector/pg_blocked_test.go diff --git a/collector/pg_blocked.go b/collector/pg_blocked.go new file mode 100644 index 000000000..8e7045882 --- /dev/null +++ b/collector/pg_blocked.go @@ -0,0 +1,91 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const blockedSubsystem = "blocked" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGBlockedCollector struct { + log log.Logger +} + +func NewPGBlockedCollector(config collectorConfig) (Collector, error) { + return &PGBlockedCollector{log: config.logger}, nil +} + +var ( + blockedQueries = prometheus.NewDesc( + prometheus.BuildFQName(namespace, blockedSubsystem, "queries"), + "The current number of blocked queries", + []string{"table"}, + prometheus.Labels{}, + ) + + blockedQuery = ` + SELECT + count(blocked.transactionid) AS queries, + '__transaction__' AS table + FROM pg_catalog.pg_locks blocked + WHERE NOT blocked.granted AND locktype = 'transactionid' + GROUP BY locktype + UNION + SELECT + count(blocked.relation) AS queries, + blocked.relation::regclass::text AS table + FROM pg_catalog.pg_locks blocked + WHERE NOT blocked.granted AND locktype != 'transactionid' + GROUP BY relation + ` +) + +func (PGBlockedCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + blockedQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var table string + var queries float64 + + if err := rows.Scan(&queries, &table); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + blockedQueries, + prometheus.GaugeValue, + queries, + table, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_blocked_test.go b/collector/pg_blocked_test.go new file mode 100644 index 000000000..e05376b5b --- /dev/null +++ b/collector/pg_blocked_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgBlockedCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "queries", + "table", + } + rows := sqlmock.NewRows(columns). + AddRow(1000, "pgbouncer") + + mock.ExpectQuery(sanitizeQuery(blockedQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGBlockedCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGBlockedCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"table": "pgbouncer"}, value: 1000, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From fb04e3cb6416f5a218b93860dfb85bfc2da8901f Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 18:29:16 -0700 Subject: [PATCH 11/39] Add pg_blocked queries Signed-off-by: Felix Yuan --- collector/pg_oldest_blocked.go | 83 +++++++++++++++++++++++++++++ collector/pg_oldest_blocked_test.go | 61 +++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 collector/pg_oldest_blocked.go create mode 100644 collector/pg_oldest_blocked_test.go diff --git a/collector/pg_oldest_blocked.go b/collector/pg_oldest_blocked.go new file mode 100644 index 000000000..24f0730b9 --- /dev/null +++ b/collector/pg_oldest_blocked.go @@ -0,0 +1,83 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const oldestBlockedSubsystem = "oldest_blocked" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGOldestBlockedCollector struct { + log log.Logger +} + +func NewPGOldestBlockedCollector(config collectorConfig) (Collector, error) { + return &PGOldestBlockedCollector{log: config.logger}, nil +} + +var ( + oldestBlockedAgeSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, oldestBlockedSubsystem, "age_seconds"), + "Largest number of seconds any transaction is currently waiting on a lock", + []string{}, + prometheus.Labels{}, + ) + + oldestBlockedQuery = ` + SELECT + coalesce(extract('epoch' from max(clock_timestamp() - state_change)), 0) age_seconds + FROM + pg_catalog.pg_stat_activity + WHERE + wait_event_type = 'Lock' + AND state='active' + ` +) + +func (PGOldestBlockedCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + oldestBlockedQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ageSeconds float64 + + if err := rows.Scan(&ageSeconds); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + oldestBlockedAgeSeconds, + prometheus.GaugeValue, + ageSeconds, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_oldest_blocked_test.go b/collector/pg_oldest_blocked_test.go new file mode 100644 index 000000000..9aac3762f --- /dev/null +++ b/collector/pg_oldest_blocked_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgOldestBlockedCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "age_seconds", + } + rows := sqlmock.NewRows(columns). + AddRow(100000) + + mock.ExpectQuery(sanitizeQuery(oldestBlockedQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGOldestBlockedCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGOldestBlockedCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 100000, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From f76c2ce0b3f5b2bff9158caf14dbc4e6fe5ab43d Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 18:34:50 -0700 Subject: [PATCH 12/39] Add pg_slow Signed-off-by: Felix Yuan --- collector/pg_slow.go | 82 +++++++++++++++++++++++++++++++++++++++ collector/pg_slow_test.go | 61 +++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 collector/pg_slow.go create mode 100644 collector/pg_slow_test.go diff --git a/collector/pg_slow.go b/collector/pg_slow.go new file mode 100644 index 000000000..3f26d4784 --- /dev/null +++ b/collector/pg_slow.go @@ -0,0 +1,82 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const slowSubsystem = "slow" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGSlowCollector struct { + log log.Logger +} + +func NewPGSlowCollector(config collectorConfig) (Collector, error) { + return &PGSlowCollector{log: config.logger}, nil +} + +var ( + slowQueries = prometheus.NewDesc( + prometheus.BuildFQName(namespace, slowSubsystem, "queries"), + "Current number of slow queries", + []string{}, + prometheus.Labels{}, + ) + + slowQuery = ` + SELECT + COUNT(*) AS queries + FROM + pg_catalog.pg_stat_activity + WHERE + state = 'active' AND (now() - query_start) > '1 seconds'::interval + ` +) + +func (PGSlowCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + slowQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var queries float64 + + if err := rows.Scan(&queries); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + slowQueries, + prometheus.GaugeValue, + queries, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_slow_test.go b/collector/pg_slow_test.go new file mode 100644 index 000000000..17d52a4ad --- /dev/null +++ b/collector/pg_slow_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgSlowCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "queries", + } + rows := sqlmock.NewRows(columns). + AddRow(25) + + mock.ExpectQuery(sanitizeQuery(slowQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGSlowCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGSlowCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 25, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 5d06f646c53c5cccf89ee1a8918d19fe519fc618 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 18:43:56 -0700 Subject: [PATCH 13/39] Add long running transactions query Signed-off-by: Felix Yuan --- collector/pg_long_running_transactions.go | 93 +++++++++++++++++++ .../pg_long_running_transactions_test.go | 63 +++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 collector/pg_long_running_transactions.go create mode 100644 collector/pg_long_running_transactions_test.go diff --git a/collector/pg_long_running_transactions.go b/collector/pg_long_running_transactions.go new file mode 100644 index 000000000..c6463f016 --- /dev/null +++ b/collector/pg_long_running_transactions.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const longRunningTransactionsSubsystem = "long_running_transactions" + +func init() { + registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) +} + +type PGLongRunningTransactionsCollector struct { + log log.Logger +} + +func NewPGLongRunningTransactionsCollector(config collectorConfig) (Collector, error) { + return &PGLongRunningTransactionsCollector{log: config.logger}, nil +} + +var ( + longRunningTransactionsCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, longRunningTransactionsSubsystem, "count"), + "Current number of long running transactions", + []string{}, + prometheus.Labels{}, + ) + + longRunningTransactionsAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, longRunningTransactionsSubsystem, "age_in_seconds"), + "The current maximum transaction age in seconds", + []string{}, + prometheus.Labels{}, + ) + + longRunningTransactionsQuery = ` + SELECT + COUNT(*) as transactions, + MAX(EXTRACT(EPOCH FROM (clock_timestamp() - xact_start))) AS age_in_seconds + FROM pg_catalog.pg_stat_activity + WHERE state is distinct from 'idle' AND (now() - xact_start) > '1 minutes'::interval AND query not like 'autovacuum:%' + ` +) + +func (PGLongRunningTransactionsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + longRunningTransactionsQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var transactions, ageInSeconds float64 + + if err := rows.Scan(&transactions, &ageInSeconds); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + longRunningTransactionsCount, + prometheus.GaugeValue, + transactions, + ) + ch <- prometheus.MustNewConstMetric( + longRunningTransactionsAgeInSeconds, + prometheus.GaugeValue, + ageInSeconds, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_long_running_transactions_test.go b/collector/pg_long_running_transactions_test.go new file mode 100644 index 000000000..1e63dc346 --- /dev/null +++ b/collector/pg_long_running_transactions_test.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgLongRunningTransactionsCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "transactions", + "age_in_seconds", + } + rows := sqlmock.NewRows(columns). + AddRow(20, 1200) + + mock.ExpectQuery(sanitizeQuery(longRunningTransactionsQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGLongRunningTransactionsCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGLongRunningTransactionsCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 20, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{}, value: 1200, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From eb86b4cff4a49e675816b86aa1ff5dc7f72b3dff Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 18:59:12 -0700 Subject: [PATCH 14/39] Add stuck in transaction query + xid query + fix some init Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 2 +- collector/pg_blocked.go | 2 +- collector/pg_index_size.go | 2 +- collector/pg_long_running_transactions.go | 2 +- collector/pg_oldest_blocked.go | 2 +- collector/pg_slow.go | 2 +- collector/pg_stat_user_indexes.go | 2 +- collector/pg_stat_walreceiver.go | 2 +- collector/pg_statio_user_indexes.go | 2 +- collector/pg_stuck_idle_in_transaction.go | 86 ++++++++++++++ .../pg_stuck_idle_in_transaction_test.go | 61 ++++++++++ collector/pg_total_relation_size.go | 2 +- collector/pg_xid.go | 99 ++++++++++++++++ collector/pg_xid_test.go | 110 ++++++++++++++++++ 14 files changed, 366 insertions(+), 10 deletions(-) create mode 100644 collector/pg_stuck_idle_in_transaction.go create mode 100644 collector/pg_stuck_idle_in_transaction_test.go create mode 100644 collector/pg_xid.go create mode 100644 collector/pg_xid_test.go diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index 8c50235bd..6e9dcad77 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) + registerCollector("archiver", defaultEnabled, NewPGArchiverCollector) } type PGArchiverCollector struct { diff --git a/collector/pg_blocked.go b/collector/pg_blocked.go index 8e7045882..1000c010d 100644 --- a/collector/pg_blocked.go +++ b/collector/pg_blocked.go @@ -23,7 +23,7 @@ import ( const blockedSubsystem = "blocked" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(blockedSubsystem, defaultEnabled, NewPGBlockedCollector) } type PGBlockedCollector struct { diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go index 54bc16f8a..5852f30e9 100644 --- a/collector/pg_index_size.go +++ b/collector/pg_index_size.go @@ -23,7 +23,7 @@ import ( const indexSizeSubsystem = "index_size" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(indexSizeSubsystem, defaultEnabled, NewPGIndexSizeCollector) } type PGIndexSizeCollector struct { diff --git a/collector/pg_long_running_transactions.go b/collector/pg_long_running_transactions.go index c6463f016..5a5425f6f 100644 --- a/collector/pg_long_running_transactions.go +++ b/collector/pg_long_running_transactions.go @@ -23,7 +23,7 @@ import ( const longRunningTransactionsSubsystem = "long_running_transactions" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(longRunningTransactionsSubsystem, defaultEnabled, NewPGLongRunningTransactionsCollector) } type PGLongRunningTransactionsCollector struct { diff --git a/collector/pg_oldest_blocked.go b/collector/pg_oldest_blocked.go index 24f0730b9..3c0c753ba 100644 --- a/collector/pg_oldest_blocked.go +++ b/collector/pg_oldest_blocked.go @@ -23,7 +23,7 @@ import ( const oldestBlockedSubsystem = "oldest_blocked" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(oldestBlockedSubsystem, defaultEnabled, NewPGOldestBlockedCollector) } type PGOldestBlockedCollector struct { diff --git a/collector/pg_slow.go b/collector/pg_slow.go index 3f26d4784..88e550a84 100644 --- a/collector/pg_slow.go +++ b/collector/pg_slow.go @@ -23,7 +23,7 @@ import ( const slowSubsystem = "slow" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(slowSubsystem, defaultEnabled, NewPGSlowCollector) } type PGSlowCollector struct { diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index b13b22744..8ac8a6431 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) + registerCollector(statUserIndexesSubsystem, defaultEnabled, NewPGStatUserIndexesCollector) } type PGStatUserIndexesCollector struct { diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index 551ee7044..75c725942 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) + registerCollector(statWalReceiverSubsystem, defaultEnabled, NewPGStatWalReceiverCollector) } type PGStatWalReceiverCollector struct { diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go index d4bb5f6db..1ccedf0e9 100644 --- a/collector/pg_statio_user_indexes.go +++ b/collector/pg_statio_user_indexes.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector("replication", defaultEnabled, NewPGStatWalReceiverCollector) + registerCollector(statioUserIndexesSubsystem, defaultEnabled, NewPGStatioUserIndexesCollector) } type PGStatioUserIndexesCollector struct { diff --git a/collector/pg_stuck_idle_in_transaction.go b/collector/pg_stuck_idle_in_transaction.go new file mode 100644 index 000000000..cc2ec849a --- /dev/null +++ b/collector/pg_stuck_idle_in_transaction.go @@ -0,0 +1,86 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const stuckIdleInTransactionSubsystem = "stuck_in_transaction" + +func init() { + registerCollector(stuckIdleInTransactionSubsystem, defaultEnabled, NewPGStuckIdleInTransactionCollector) +} + +type PGStuckIdleInTransactionCollector struct { + log log.Logger +} + +func NewPGStuckIdleInTransactionCollector(config collectorConfig) (Collector, error) { + return &PGStuckIdleInTransactionCollector{log: config.logger}, nil +} + +var ( + stuckIdleInTransactionQueries = prometheus.NewDesc( + prometheus.BuildFQName(namespace, longRunningTransactionsSubsystem, "queries"), + "Current number of queries that are stuck being idle in transactions", + []string{}, + prometheus.Labels{}, + ) + + stuckIdleInTransactionQuery = ` + SELECT + COUNT(*) AS queries + FROM pg_catalog.pg_stat_activity + WHERE + state = 'idle in transaction' AND (now() - query_start) > '10 minutes'::interval + ` +) + +func (PGStuckIdleInTransactionCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + stuckIdleInTransactionQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var queries float64 + + if err := rows.Scan(&queries); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + stuckIdleInTransactionQueries, + prometheus.GaugeValue, + queries, + ) + ch <- prometheus.MustNewConstMetric( + longRunningTransactionsAgeInSeconds, + prometheus.GaugeValue, + queries, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stuck_idle_in_transaction_test.go b/collector/pg_stuck_idle_in_transaction_test.go new file mode 100644 index 000000000..1313806a9 --- /dev/null +++ b/collector/pg_stuck_idle_in_transaction_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStuckIdleInTransactionCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "queries", + } + rows := sqlmock.NewRows(columns). + AddRow(30) + + mock.ExpectQuery(sanitizeQuery(stuckIdleInTransactionQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStuckIdleInTransactionCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStuckIdleInTransactionCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 30, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} diff --git a/collector/pg_total_relation_size.go b/collector/pg_total_relation_size.go index 47160ff01..bb4812595 100644 --- a/collector/pg_total_relation_size.go +++ b/collector/pg_total_relation_size.go @@ -23,7 +23,7 @@ import ( const totalRelationSizeSubsystem = "total_relation_size" func init() { - registerCollector(statioUserTableSubsystem, defaultEnabled, NewPGStatIOUserTablesCollector) + registerCollector(totalRelationSizeSubsystem, defaultEnabled, NewPGTotalRelationSizeCollector) } type PGTotalRelationSizeCollector struct { diff --git a/collector/pg_xid.go b/collector/pg_xid.go new file mode 100644 index 000000000..5b7650be4 --- /dev/null +++ b/collector/pg_xid.go @@ -0,0 +1,99 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const xidSubsystem = "xid" + +func init() { + registerCollector(xidSubsystem, defaultEnabled, NewPGXidCollector) +} + +type PGXidCollector struct { + log log.Logger +} + +func NewPGXidCollector(config collectorConfig) (Collector, error) { + return &PGXidCollector{log: config.logger}, nil +} + +var ( + xidCurrent = prometheus.NewDesc( + prometheus.BuildFQName(namespace, xidSubsystem, "current"), + "Current 64-bit transaction id of the query used to collect this metric (truncated to low 52 bits)", + []string{}, prometheus.Labels{}, + ) + xidXmin = prometheus.NewDesc( + prometheus.BuildFQName(namespace, xidSubsystem, "xmin"), + "Oldest transaction id of a transaction still in progress, i.e. not known committed or aborted (truncated to low 52 bits)", + []string{}, prometheus.Labels{}, + ) + xidXminAge = prometheus.NewDesc( + prometheus.BuildFQName(namespace, xidSubsystem, "xmin_age"), + "Age of oldest transaction still not committed or aborted measured in transaction ids", + []string{}, prometheus.Labels{}, + ) + + xidQuery = ` + SELECT + CASE WHEN pg_is_in_recovery() THEN 'NaN'::float ELSE txid_current() % (2^52)::bigint END AS current, + CASE WHEN pg_is_in_recovery() THEN 'NaN'::float ELSE txid_snapshot_xmin(txid_current_snapshot()) % (2^52)::bigint END AS xmin, + CASE WHEN pg_is_in_recovery() THEN 'NaN'::float ELSE txid_current() - txid_snapshot_xmin(txid_current_snapshot()) END AS xmin_age + ` +) + +func (PGXidCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + xidQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var current, xmin, xminAge float64 + + if err := rows.Scan(¤t, &xmin, &xminAge); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + xidCurrent, + prometheus.CounterValue, + current, + ) + ch <- prometheus.MustNewConstMetric( + xidXmin, + prometheus.CounterValue, + xmin, + ) + ch <- prometheus.MustNewConstMetric( + xidXminAge, + prometheus.GaugeValue, + xminAge, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_xid_test.go b/collector/pg_xid_test.go new file mode 100644 index 000000000..7ec3f8b5d --- /dev/null +++ b/collector/pg_xid_test.go @@ -0,0 +1,110 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "math" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPgXidCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "current", + "xmin", + "xmin_age", + } + rows := sqlmock.NewRows(columns). + AddRow(22, 25, 30) + + mock.ExpectQuery(sanitizeQuery(xidQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGXidCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGXidCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 22, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{}, value: 25, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{}, value: 30, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + +// This test is turned off for now +// Because convey cannot compare NaN +func xTestPgNanCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "current", + "xmin", + "xmin_age", + } + rows := sqlmock.NewRows(columns). + AddRow(math.NaN(), math.NaN(), math.NaN()) + + mock.ExpectQuery(sanitizeQuery(xidQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGXidCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGXidCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: math.NaN(), metricType: dto.MetricType_COUNTER}, + {labels: labelMap{}, value: math.NaN(), metricType: dto.MetricType_COUNTER}, + {labels: labelMap{}, value: math.NaN(), metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From b235947d9d1b5385fd08253c50714417d93e9e6b Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 19:33:16 -0700 Subject: [PATCH 15/39] Add database wraparound query Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 96 ++++++++++++++++++++++++ collector/pg_database_wraparound_test.go | 64 ++++++++++++++++ collector/pg_slow_test.go | 2 +- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 collector/pg_database_wraparound.go create mode 100644 collector/pg_database_wraparound_test.go diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go new file mode 100644 index 000000000..8d41666ad --- /dev/null +++ b/collector/pg_database_wraparound.go @@ -0,0 +1,96 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const databaseWraparoundSubsystem = "database_wraparound" + +func init() { + registerCollector(databaseWraparoundSubsystem, defaultEnabled, NewPGDatabaseWraparoundCollector) +} + +type PGDatabaseWraparoundCollector struct { + log log.Logger +} + +func NewPGDatabaseWraparoundCollector(config collectorConfig) (Collector, error) { + return &PGDatabaseWraparoundCollector{log: config.logger}, nil +} + +var ( + databaseWraparoundAgeDatfrozenxid = prometheus.NewDesc( + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datfrozenxid"), + "Age of the oldest transaction ID that has not been frozen.", + []string{"datname"}, + prometheus.Labels{}, + ) + databaseWraparoundAgeDatminmxid = prometheus.NewDesc( + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datminmxid"), + "Age of the oldest multi-transaction ID that has been replaced with a transaction ID.", + []string{"datname"}, + prometheus.Labels{}, + ) + + databaseWraparoundQuery = ` + SELECT + datname, + age(d.datfrozenxid) as age_datfrozenxid, + mxid_age(d.datminmxid) as age_datminmxid + FROM + pg_catalog.pg_database d + WHERE + d.datallowconn + ` +) + +func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + databaseWraparoundQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var datname string + var ageDatfrozenxid, ageDatminmxid float64 + + if err := rows.Scan(&datname, &ageDatfrozenxid, &ageDatminmxid); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + databaseWraparoundAgeDatfrozenxid, + prometheus.GaugeValue, + ageDatfrozenxid, datname, + ) + ch <- prometheus.MustNewConstMetric( + databaseWraparoundAgeDatminmxid, + prometheus.GaugeValue, + ageDatminmxid, datname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_database_wraparound_test.go b/collector/pg_database_wraparound_test.go new file mode 100644 index 000000000..d0a74c362 --- /dev/null +++ b/collector/pg_database_wraparound_test.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGDatabaseWraparoundCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "datname", + "age_datfrozenxid", + "age_datminmxid", + } + rows := sqlmock.NewRows(columns). + AddRow("newreddit", 87126426, 0) + + mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGDatabaseWraparoundCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGDatabaseWraparoundCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"datname": "newreddit"}, value: 87126426, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "newreddit"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} diff --git a/collector/pg_slow_test.go b/collector/pg_slow_test.go index 17d52a4ad..eece77c2a 100644 --- a/collector/pg_slow_test.go +++ b/collector/pg_slow_test.go @@ -22,7 +22,7 @@ import ( "github.com/smartystreets/goconvey/convey" ) -func TestPgSlowCollector(t *testing.T) { +func TestPGSlowCollector(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) From eac7650e62b22968ab74bf42898e7d6ace1c2bcc Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 19:34:16 -0700 Subject: [PATCH 16/39] Fix test name Signed-off-by: Felix Yuan --- collector/pg_long_running_transactions_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_long_running_transactions_test.go b/collector/pg_long_running_transactions_test.go index 1e63dc346..eedda7c65 100644 --- a/collector/pg_long_running_transactions_test.go +++ b/collector/pg_long_running_transactions_test.go @@ -22,7 +22,7 @@ import ( "github.com/smartystreets/goconvey/convey" ) -func TestPgLongRunningTransactionsCollector(t *testing.T) { +func TestPGLongRunningTransactionsCollector(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) From 18ba5bb8214ad6e5234aa8628dbcdaa6e9ea2194 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 19:40:54 -0700 Subject: [PATCH 17/39] xlog location Signed-off-by: Felix Yuan --- collector/pg_xlog_location.go | 81 ++++++++++++++++++++++++++++++ collector/pg_xlog_location_test.go | 61 ++++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 collector/pg_xlog_location.go create mode 100644 collector/pg_xlog_location_test.go diff --git a/collector/pg_xlog_location.go b/collector/pg_xlog_location.go new file mode 100644 index 000000000..42c7adafb --- /dev/null +++ b/collector/pg_xlog_location.go @@ -0,0 +1,81 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const xlogLocationSubsystem = "xlog_location" + +func init() { + registerCollector(xlogLocationSubsystem, defaultEnabled, NewPGXlogLocationCollector) +} + +type PGXlogLocationCollector struct { + log log.Logger +} + +func NewPGXlogLocationCollector(config collectorConfig) (Collector, error) { + return &PGXlogLocationCollector{log: config.logger}, nil +} + +var ( + xlogLocationBytes = prometheus.NewDesc( + prometheus.BuildFQName(namespace, xlogLocationSubsystem, "bytes"), + "Postgres LSN (log sequence number) being generated on primary or replayed on replica (truncated to low 52 bits)", + []string{}, + prometheus.Labels{}, + ) + + xlogLocationQuery = ` + SELECT CASE + WHEN pg_is_in_recovery() + THEN (pg_last_xlog_replay_location() - '0/0') % (2^52)::bigint + ELSE (pg_current_xlog_location() - '0/0') % (2^52)::bigint + END AS bytes + ` +) + +func (PGXlogLocationCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + xlogLocationQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var bytes float64 + + if err := rows.Scan(&bytes); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + xlogLocationBytes, + prometheus.CounterValue, + bytes, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_xlog_location_test.go b/collector/pg_xlog_location_test.go new file mode 100644 index 000000000..9071a2cc1 --- /dev/null +++ b/collector/pg_xlog_location_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGXlogLocationCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "bytes", + } + rows := sqlmock.NewRows(columns). + AddRow(53401) + + mock.ExpectQuery(sanitizeQuery(xlogLocationQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGXlogLocationCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGXlogLocationCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 53401, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From bb5dba8888fa527b2cb5c169be1d0fc2f043809f Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 20:10:24 -0700 Subject: [PATCH 18/39] Add pg_stat_activity_marginalia sampler Signed-off-by: Felix Yuan --- collector/collector_test.go | 1 + collector/pg_stat_activity_marginalia.go | 113 ++++++++++++++++++ collector/pg_stat_activity_marginalia_test.go | 71 +++++++++++ 3 files changed, 185 insertions(+) create mode 100644 collector/pg_stat_activity_marginalia.go create mode 100644 collector/pg_stat_activity_marginalia_test.go diff --git a/collector/collector_test.go b/collector/collector_test.go index 5ac7c27be..40371fde5 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -49,6 +49,7 @@ func readMetric(m prometheus.Metric) MetricResult { func sanitizeQuery(q string) string { q = strings.Join(strings.Fields(q), " ") q = strings.Replace(q, "(", "\\(", -1) + q = strings.Replace(q, "?", "\\?", -1) q = strings.Replace(q, ")", "\\)", -1) q = strings.Replace(q, "[", "\\[", -1) q = strings.Replace(q, "]", "\\]", -1) diff --git a/collector/pg_stat_activity_marginalia.go b/collector/pg_stat_activity_marginalia.go new file mode 100644 index 000000000..4a84e5e40 --- /dev/null +++ b/collector/pg_stat_activity_marginalia.go @@ -0,0 +1,113 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const statActivityMarginaliaSubsystem = "slow" + +func init() { + registerCollector(statActivityMarginaliaSubsystem, defaultEnabled, NewPGStatActivityMarginaliaCollector) +} + +type PGStatActivityMarginaliaCollector struct { + log log.Logger +} + +func NewPGStatActivityMarginaliaCollector(config collectorConfig) (Collector, error) { + return &PGStatActivityMarginaliaCollector{log: config.logger}, nil +} + +var ( + statActivityMarginaliaActiveCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivityMarginaliaSubsystem, "active_count"), + "Number of active queries at time of sample", + []string{"usename", "application", "endpoint", "command", "state", "wait_event", "wait_event_type"}, + prometheus.Labels{}, + ) + statActivityMarginaliaMaxTxAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivityMarginaliaSubsystem, "max_tx_age_in_seconds"), + "Number of active queries at time of sample", + []string{"usename", "application", "endpoint", "command", "state", "wait_event", "wait_event_type"}, + prometheus.Labels{}, + ) + + statActivityMarginaliaQuery = ` + SELECT + usename AS usename, + a.matches[1] AS application, + a.matches[2] AS endpoint, + a.matches[3] AS command, + a.state AS state, + a.wait_event AS wait_event, + a.wait_event_type AS wait_event_type, + COUNT(*) active_count, + MAX(age_in_seconds) AS max_tx_age_in_seconds + FROM ( + SELECT + usename, + regexp_matches(query, '^\s*(?:\/\*(?:application:(\w+),?)?(?:correlation_id:\w+,?)?(?:jid:\w+,?)?(?:endpoint_id:([\w/\-\.:\#\s]+),?)?.*?\*\/)?\s*(\w+)') AS matches, + state, + wait_event, + wait_event_type, + EXTRACT(EPOCH FROM (clock_timestamp() - xact_start)) AS age_in_seconds + FROM + pg_catalog.pg_stat_activity + ) a + GROUP BY usename, application, endpoint, command, state, wait_event, wait_event_type + ORDER BY active_count DESC + ` +) + +func (PGStatActivityMarginaliaCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + statActivityMarginaliaQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var usename, application, endpoint, command, state, wait_event, wait_event_type string + var count, max_tx_age float64 + + if err := rows.Scan(&usename, &application, &endpoint, &command, &state, &wait_event, &wait_event_type, &count, &max_tx_age); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statActivityMarginaliaActiveCount, + prometheus.GaugeValue, + count, + usename, application, endpoint, command, state, wait_event, wait_event_type, + ) + ch <- prometheus.MustNewConstMetric( + statActivityMarginaliaMaxTxAgeInSeconds, + prometheus.GaugeValue, + max_tx_age, + usename, application, endpoint, command, state, wait_event, wait_event_type, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_activity_marginalia_test.go b/collector/pg_stat_activity_marginalia_test.go new file mode 100644 index 000000000..52500ad61 --- /dev/null +++ b/collector/pg_stat_activity_marginalia_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func xTestPGStatActivityMarginaliaCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "usename", + "application", + "endpoint", + "command", + "wait_event", + "state", + "wait_event_type", + "active_count", + "max_tx_age_in_seconds", + } + rows := sqlmock.NewRows(columns). + AddRow("postgres", "service_thing", "", "COMMIT", "ClientRead", "idle", "Client", 25, 0) + + // TODO: Query has a lot of things to escape, figure out how to get this test to work + mock.ExpectQuery(sanitizeQuery(statActivityMarginaliaQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatActivityMarginaliaCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatActivityMarginaliaCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"usename": "postgres", "application": "service_thing", "endpoint": "", "command": "COMMIT", "wait_event": "ClientRead", "state": "idle", "wait_event_type": "Client"}, value: 25, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"usename": "postgres", "application": "service_thing", "endpoint": "", "command": "COMMIT", "wait_event": "ClientRead", "state": "idle", "wait_event_type": "Client"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 8bbd6d42e60a6d755e4b91f649c5a3d24afc9199 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 20:16:19 -0700 Subject: [PATCH 19/39] Add pg stat activity autovacuum Signed-off-by: Felix Yuan --- collector/pg_stat_activity_autovacuum.go | 85 +++++++++++++++++++ collector/pg_stat_activity_autovacuum_test.go | 62 ++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 collector/pg_stat_activity_autovacuum.go create mode 100644 collector/pg_stat_activity_autovacuum_test.go diff --git a/collector/pg_stat_activity_autovacuum.go b/collector/pg_stat_activity_autovacuum.go new file mode 100644 index 000000000..9ea75d3a4 --- /dev/null +++ b/collector/pg_stat_activity_autovacuum.go @@ -0,0 +1,85 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const statActivityAutovacuumSubsystem = "slow" + +func init() { + registerCollector(statActivityAutovacuumSubsystem, defaultEnabled, NewPGStatActivityAutovacuumCollector) +} + +type PGStatActivityAutovacuumCollector struct { + log log.Logger +} + +func NewPGStatActivityAutovacuumCollector(config collectorConfig) (Collector, error) { + return &PGStatActivityAutovacuumCollector{log: config.logger}, nil +} + +var ( + statActivityAutovacuumAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivityAutovacuumSubsystem, "age_in_seconds"), + "The age of the vacuum process in seconds", + []string{"relname"}, + prometheus.Labels{}, + ) + + statActivityAutovacuumQuery = ` + SELECT + SPLIT_PART(query, '.', 2) AS relname, + EXTRACT(EPOCH FROM (clock_timestamp() - xact_start)) AS age_in_seconds + FROM + pg_catalog.pg_stat_activity + WHERE + query like 'autovacuum:%' AND + EXTRACT(EPOCH FROM (clock_timestamp() - xact_start)) > 300 + ` +) + +func (PGStatActivityAutovacuumCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + statActivityAutovacuumQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var relname string + var ageInSeconds float64 + + if err := rows.Scan(&relname, &ageInSeconds); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statActivityAutovacuumAgeInSeconds, + prometheus.GaugeValue, + ageInSeconds, relname, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_activity_autovacuum_test.go b/collector/pg_stat_activity_autovacuum_test.go new file mode 100644 index 000000000..f79be2d2c --- /dev/null +++ b/collector/pg_stat_activity_autovacuum_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStatActivityAutovacuumCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "relname", + "age_in_seconds", + } + rows := sqlmock.NewRows(columns). + AddRow("test", 3600) + + mock.ExpectQuery(sanitizeQuery(statActivityAutovacuumQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatActivityAutovacuumCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatActivityAutovacuumCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"relname": "test"}, value: 3600, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 82a51fecb5135278bd554696e2c59525416b8082 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 20:27:41 -0700 Subject: [PATCH 20/39] Add autovacuum active Signed-off-by: Felix Yuan --- collector/pg_stat_activity_autovacuum.go | 2 +- .../pg_stat_activity_autovacuum_active.go | 90 +++++++++++++++++++ ...pg_stat_activity_autovacuum_active_test.go | 62 +++++++++++++ collector/pg_stat_activity_marginalia.go | 2 +- 4 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 collector/pg_stat_activity_autovacuum_active.go create mode 100644 collector/pg_stat_activity_autovacuum_active_test.go diff --git a/collector/pg_stat_activity_autovacuum.go b/collector/pg_stat_activity_autovacuum.go index 9ea75d3a4..fbd5268cb 100644 --- a/collector/pg_stat_activity_autovacuum.go +++ b/collector/pg_stat_activity_autovacuum.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const statActivityAutovacuumSubsystem = "slow" +const statActivityAutovacuumSubsystem = "stat_activity_autovacuum" func init() { registerCollector(statActivityAutovacuumSubsystem, defaultEnabled, NewPGStatActivityAutovacuumCollector) diff --git a/collector/pg_stat_activity_autovacuum_active.go b/collector/pg_stat_activity_autovacuum_active.go new file mode 100644 index 000000000..28f48e62a --- /dev/null +++ b/collector/pg_stat_activity_autovacuum_active.go @@ -0,0 +1,90 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const statActivityAutovacuumActiveSubsystem = "stat_activity_autovacuum_active" + +func init() { + registerCollector(statActivityAutovacuumActiveSubsystem, defaultEnabled, NewPGStatActivityAutovacuumActiveCollector) +} + +type PGStatActivityAutovacuumActiveCollector struct { + log log.Logger +} + +func NewPGStatActivityAutovacuumActiveCollector(config collectorConfig) (Collector, error) { + return &PGStatActivityAutovacuumActiveCollector{log: config.logger}, nil +} + +var ( + statActivityAutovacuumActiveWorkersCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivityAutovacuumActiveSubsystem, "workers_count"), + "Current number of statActivityAutovacuumActive queries", + []string{"phase", "mode"}, + prometheus.Labels{}, + ) + + statActivityAutovacuumActiveQuery = ` + SELECT + v.phase, + CASE + when a.query ~ '^autovacuum.*to prevent wraparound' then 'wraparound' + when a.query ~* '^vacuum' then 'user' + when a.pid is null then 'idle' + ELSE 'regular' + END as mode, + count(1) as workers_count + FROM pg_stat_progress_vacuum v + LEFT JOIN pg_catalog.pg_stat_activity a using (pid) + GROUP BY 1,2 + ` +) + +func (PGStatActivityAutovacuumActiveCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + statActivityAutovacuumActiveQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var phase, mode string + var workers_count float64 + + if err := rows.Scan(&phase, &mode, &workers_count); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + statActivityAutovacuumActiveWorkersCount, + prometheus.GaugeValue, + workers_count, + phase, mode, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_activity_autovacuum_active_test.go b/collector/pg_stat_activity_autovacuum_active_test.go new file mode 100644 index 000000000..edde1ef90 --- /dev/null +++ b/collector/pg_stat_activity_autovacuum_active_test.go @@ -0,0 +1,62 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStatActivityAutovacuumActiveCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "phase", + "mode", + "workers_count", + } + rows := sqlmock.NewRows(columns). + AddRow("Scanning heap", "regular", 2) + mock.ExpectQuery(sanitizeQuery(statActivityAutovacuumActiveQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatActivityAutovacuumActiveCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatActivityAutovacuumActiveCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"phase": "Scanning heap", "mode": "regular"}, value: 2, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} diff --git a/collector/pg_stat_activity_marginalia.go b/collector/pg_stat_activity_marginalia.go index 4a84e5e40..a31398fcc 100644 --- a/collector/pg_stat_activity_marginalia.go +++ b/collector/pg_stat_activity_marginalia.go @@ -20,7 +20,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const statActivityMarginaliaSubsystem = "slow" +const statActivityMarginaliaSubsystem = "stat_activity_marginalia" func init() { registerCollector(statActivityMarginaliaSubsystem, defaultEnabled, NewPGStatActivityMarginaliaCollector) From c9f83aeb0cf0fce9fe7665ddc20d095960bc0f0e Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Thu, 22 Jun 2023 20:37:34 -0700 Subject: [PATCH 21/39] Long running transactions marginalia Signed-off-by: Felix Yuan --- ...pg_long_running_transactions_marginalia.go | 94 +++++++++++++++++++ ...ng_running_transactions_marginalia_test.go | 64 +++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 collector/pg_long_running_transactions_marginalia.go create mode 100644 collector/pg_long_running_transactions_marginalia_test.go diff --git a/collector/pg_long_running_transactions_marginalia.go b/collector/pg_long_running_transactions_marginalia.go new file mode 100644 index 000000000..ca2465fe9 --- /dev/null +++ b/collector/pg_long_running_transactions_marginalia.go @@ -0,0 +1,94 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +const longRunningTransactionsMarginaliaSubsystem = "long_running_transactions_marginalia" + +func init() { + registerCollector(longRunningTransactionsMarginaliaSubsystem, defaultEnabled, NewPGLongRunningTransactionsMarginaliaCollector) +} + +type PGLongRunningTransactionsMarginaliaCollector struct { + log log.Logger +} + +func NewPGLongRunningTransactionsMarginaliaCollector(config collectorConfig) (Collector, error) { + return &PGLongRunningTransactionsMarginaliaCollector{log: config.logger}, nil +} + +var ( + longRunningTransactionsMarginaliaMaxAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, longRunningTransactionsMarginaliaSubsystem, "max_age_in_seconds"), + "The current maximum transaction age in seconds", + []string{"application", "endpoint"}, + prometheus.Labels{}, + ) + + longRunningTransactionsMarginaliaQuery = ` + SELECT + activity.matches[1] AS application, + activity.matches[2] AS endpoint, + MAX(age_in_seconds) AS max_age_in_seconds + FROM ( + SELECT + regexp_matches(query, '^\s*(?:\/\*(?:application:(\w+),?)?(?:correlation_id:\w+,?)?(?:jid:\w+,?)?(?:endpoint_id:([\w/\-\.:\#\s]+),?)?.*?\*\/)?\s*(\w+)') AS matches, + EXTRACT(EPOCH FROM (clock_timestamp() - xact_start)) AS age_in_seconds + FROM + pg_catalog.pg_stat_activity + WHERE state <> 'idle' + AND (clock_timestamp() - xact_start) > '30 seconds'::interval + AND query NOT LIKE 'autovacuum:%' + ) activity + GROUP BY application, endpoint + ORDER BY max_age_in_seconds DESC + ` +) + +func (PGLongRunningTransactionsMarginaliaCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + longRunningTransactionsMarginaliaQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var application, endpoint string + var max_age_in_seconds float64 + + if err := rows.Scan(&application, &endpoint, &max_age_in_seconds); err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric( + longRunningTransactionsAgeInSeconds, + prometheus.GaugeValue, + max_age_in_seconds, + application, endpoint, + ) + } + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_long_running_transactions_marginalia_test.go b/collector/pg_long_running_transactions_marginalia_test.go new file mode 100644 index 000000000..e6f8b4bf1 --- /dev/null +++ b/collector/pg_long_running_transactions_marginalia_test.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func xTestPGLongRunningTransactionsMarginaliaCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "application", + "endpoint", + "max_age_in_seconds", + } + rows := sqlmock.NewRows(columns). + AddRow("reddit", "GET /r/programming", 32) + + // TODO: Fix the sanitizeQuery escaper to deal better with regex + mock.ExpectQuery(sanitizeQuery(longRunningTransactionsMarginaliaQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGLongRunningTransactionsMarginaliaCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGLongRunningTransactionsMarginaliaCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{}, value: 25, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 9b4d8459538c03300258448dbaef15dd94955320 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 09:55:02 -0700 Subject: [PATCH 22/39] Lint fixes Signed-off-by: Felix Yuan --- collector/pg_long_running_transactions_marginalia.go | 8 ++++---- collector/pg_stat_activity_autovacuum_active.go | 6 +++--- collector/pg_stat_activity_marginalia.go | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/collector/pg_long_running_transactions_marginalia.go b/collector/pg_long_running_transactions_marginalia.go index ca2465fe9..4dfc421f4 100644 --- a/collector/pg_long_running_transactions_marginalia.go +++ b/collector/pg_long_running_transactions_marginalia.go @@ -74,16 +74,16 @@ func (PGLongRunningTransactionsMarginaliaCollector) Update(ctx context.Context, for rows.Next() { var application, endpoint string - var max_age_in_seconds float64 + var maxAgeInSeconds float64 - if err := rows.Scan(&application, &endpoint, &max_age_in_seconds); err != nil { + if err := rows.Scan(&application, &endpoint, &maxAgeInSeconds); err != nil { return err } ch <- prometheus.MustNewConstMetric( - longRunningTransactionsAgeInSeconds, + longRunningTransactionsMarginaliaMaxAgeInSeconds, prometheus.GaugeValue, - max_age_in_seconds, + maxAgeInSeconds, application, endpoint, ) } diff --git a/collector/pg_stat_activity_autovacuum_active.go b/collector/pg_stat_activity_autovacuum_active.go index 28f48e62a..95989b98a 100644 --- a/collector/pg_stat_activity_autovacuum_active.go +++ b/collector/pg_stat_activity_autovacuum_active.go @@ -70,16 +70,16 @@ func (PGStatActivityAutovacuumActiveCollector) Update(ctx context.Context, insta for rows.Next() { var phase, mode string - var workers_count float64 + var workersCount float64 - if err := rows.Scan(&phase, &mode, &workers_count); err != nil { + if err := rows.Scan(&phase, &mode, &workersCount); err != nil { return err } ch <- prometheus.MustNewConstMetric( statActivityAutovacuumActiveWorkersCount, prometheus.GaugeValue, - workers_count, + workersCount, phase, mode, ) } diff --git a/collector/pg_stat_activity_marginalia.go b/collector/pg_stat_activity_marginalia.go index a31398fcc..bb5851c1d 100644 --- a/collector/pg_stat_activity_marginalia.go +++ b/collector/pg_stat_activity_marginalia.go @@ -86,10 +86,10 @@ func (PGStatActivityMarginaliaCollector) Update(ctx context.Context, instance *i defer rows.Close() for rows.Next() { - var usename, application, endpoint, command, state, wait_event, wait_event_type string - var count, max_tx_age float64 + var usename, application, endpoint, command, state, waitEvent, waitEventType string + var count, maxTxAge float64 - if err := rows.Scan(&usename, &application, &endpoint, &command, &state, &wait_event, &wait_event_type, &count, &max_tx_age); err != nil { + if err := rows.Scan(&usename, &application, &endpoint, &command, &state, &waitEvent, &waitEventType, &count, &maxTxAge); err != nil { return err } @@ -97,13 +97,13 @@ func (PGStatActivityMarginaliaCollector) Update(ctx context.Context, instance *i statActivityMarginaliaActiveCount, prometheus.GaugeValue, count, - usename, application, endpoint, command, state, wait_event, wait_event_type, + usename, application, endpoint, command, state, waitEvent, waitEventType, ) ch <- prometheus.MustNewConstMetric( statActivityMarginaliaMaxTxAgeInSeconds, prometheus.GaugeValue, - max_tx_age, - usename, application, endpoint, command, state, wait_event, wait_event_type, + maxTxAge, + usename, application, endpoint, command, state, waitEvent, waitEventType, ) } if err := rows.Err(); err != nil { From 49e1a46f93e8c77fcf486e92a5115485254b1b47 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 10:26:03 -0700 Subject: [PATCH 23/39] Lint Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 4 ++-- collector/pg_stat_user_indexes.go | 4 ++-- collector/pg_stat_walreceiver.go | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index 6e9dcad77..b30a74378 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -29,8 +29,8 @@ type PGArchiverCollector struct { const archiverSubsystem = "archiver" -func NewPGArchiverCollector(collectorConfig) (Collector, error) { - return &PGArchiverCollector{}, nil +func NewPGArchiverCollector(config collectorConfig) (Collector, error) { + return &PGArchiverCollector{log: config.logger}, nil } var ( diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index 8ac8a6431..e4e66476a 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -29,8 +29,8 @@ type PGStatUserIndexesCollector struct { const statUserIndexesSubsystem = "stat_user_indexes" -func NewPGStatUserIndexesCollector(collectorConfig) (Collector, error) { - return &PGStatUserIndexesCollector{}, nil +func NewPGStatUserIndexesCollector(config collectorConfig) (Collector, error) { + return &PGStatUserIndexesCollector{log: config.logger}, nil } var ( diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index 75c725942..df2e8b7ea 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -29,8 +29,8 @@ type PGStatWalReceiverCollector struct { const statWalReceiverSubsystem = "stat_wal_receiver" -func NewPGStatWalReceiverCollector(collectorConfig) (Collector, error) { - return &PGStatWalReceiverCollector{}, nil +func NewPGStatWalReceiverCollector(config collectorConfig) (Collector, error) { + return &PGStatWalReceiverCollector{log: config.logger}, nil } var ( From d822ec475384cf839e01a6969bd4cae51b7889d6 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 10:30:42 -0700 Subject: [PATCH 24/39] Fix the NaN tests Signed-off-by: Felix Yuan --- collector/pg_archiver_test.go | 38 +++++++++++++++++++++++++++++++++++ collector/pg_xid_test.go | 9 +++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/collector/pg_archiver_test.go b/collector/pg_archiver_test.go index a102d2634..e2bd5969d 100644 --- a/collector/pg_archiver_test.go +++ b/collector/pg_archiver_test.go @@ -14,6 +14,7 @@ package collector import ( "context" + "math" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -56,3 +57,40 @@ func TestPgArchiverCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgArchiverNaNCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + mock.ExpectQuery(sanitizeQuery(pgArchiverQuery)).WillReturnRows(sqlmock.NewRows([]string{"pending_wal_count"}). + AddRow(math.NaN())) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGArchiverCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGArchiverCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{}, value: math.NaN(), metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect.labels, convey.ShouldResemble, m.labels) + convey.So(math.IsNaN(m.value), convey.ShouldResemble, math.IsNaN(expect.value)) + convey.So(expect.metricType, convey.ShouldEqual, m.metricType) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} diff --git a/collector/pg_xid_test.go b/collector/pg_xid_test.go index 7ec3f8b5d..6de90fcfa 100644 --- a/collector/pg_xid_test.go +++ b/collector/pg_xid_test.go @@ -65,9 +65,7 @@ func TestPgXidCollector(t *testing.T) { } } -// This test is turned off for now -// Because convey cannot compare NaN -func xTestPgNanCollector(t *testing.T) { +func TestPgNanCollector(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { t.Fatalf("Error opening a stub db connection: %s", err) @@ -101,7 +99,10 @@ func xTestPgNanCollector(t *testing.T) { convey.Convey("Metrics comparison", t, func() { for _, expect := range expected { m := readMetric(<-ch) - convey.So(expect, convey.ShouldResemble, m) + + convey.So(expect.labels, convey.ShouldResemble, m.labels) + convey.So(math.IsNaN(m.value), convey.ShouldResemble, math.IsNaN(expect.value)) + convey.So(expect.metricType, convey.ShouldEqual, m.metricType) } }) if err := mock.ExpectationsWereMet(); err != nil { From 7651eedbcbf5764ecd25d58d356ec40bddb0e634 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 11:25:08 -0700 Subject: [PATCH 25/39] Remove broken tests for now Signed-off-by: Felix Yuan --- ...ng_running_transactions_marginalia_test.go | 64 ----------------- collector/pg_stat_activity_marginalia_test.go | 71 ------------------- 2 files changed, 135 deletions(-) delete mode 100644 collector/pg_long_running_transactions_marginalia_test.go delete mode 100644 collector/pg_stat_activity_marginalia_test.go diff --git a/collector/pg_long_running_transactions_marginalia_test.go b/collector/pg_long_running_transactions_marginalia_test.go deleted file mode 100644 index e6f8b4bf1..000000000 --- a/collector/pg_long_running_transactions_marginalia_test.go +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2023 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package collector - -import ( - "context" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/smartystreets/goconvey/convey" -) - -func xTestPGLongRunningTransactionsMarginaliaCollector(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("Error opening a stub db connection: %s", err) - } - defer db.Close() - inst := &instance{db: db} - columns := []string{ - "application", - "endpoint", - "max_age_in_seconds", - } - rows := sqlmock.NewRows(columns). - AddRow("reddit", "GET /r/programming", 32) - - // TODO: Fix the sanitizeQuery escaper to deal better with regex - mock.ExpectQuery(sanitizeQuery(longRunningTransactionsMarginaliaQuery)).WillReturnRows(rows) - - ch := make(chan prometheus.Metric) - go func() { - defer close(ch) - c := PGLongRunningTransactionsMarginaliaCollector{} - - if err := c.Update(context.Background(), inst, ch); err != nil { - t.Errorf("Error calling PGLongRunningTransactionsMarginaliaCollector.Update: %s", err) - } - }() - expected := []MetricResult{ - {labels: labelMap{}, value: 25, metricType: dto.MetricType_GAUGE}, - } - convey.Convey("Metrics comparison", t, func() { - for _, expect := range expected { - m := readMetric(<-ch) - convey.So(expect, convey.ShouldResemble, m) - } - }) - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled exceptions: %s", err) - } -} diff --git a/collector/pg_stat_activity_marginalia_test.go b/collector/pg_stat_activity_marginalia_test.go deleted file mode 100644 index 52500ad61..000000000 --- a/collector/pg_stat_activity_marginalia_test.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2023 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package collector - -import ( - "context" - "testing" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/smartystreets/goconvey/convey" -) - -func xTestPGStatActivityMarginaliaCollector(t *testing.T) { - db, mock, err := sqlmock.New() - if err != nil { - t.Fatalf("Error opening a stub db connection: %s", err) - } - defer db.Close() - inst := &instance{db: db} - columns := []string{ - "usename", - "application", - "endpoint", - "command", - "wait_event", - "state", - "wait_event_type", - "active_count", - "max_tx_age_in_seconds", - } - rows := sqlmock.NewRows(columns). - AddRow("postgres", "service_thing", "", "COMMIT", "ClientRead", "idle", "Client", 25, 0) - - // TODO: Query has a lot of things to escape, figure out how to get this test to work - mock.ExpectQuery(sanitizeQuery(statActivityMarginaliaQuery)).WillReturnRows(rows) - - ch := make(chan prometheus.Metric) - go func() { - defer close(ch) - c := PGStatActivityMarginaliaCollector{} - - if err := c.Update(context.Background(), inst, ch); err != nil { - t.Errorf("Error calling PGStatActivityMarginaliaCollector.Update: %s", err) - } - }() - expected := []MetricResult{ - {labels: labelMap{"usename": "postgres", "application": "service_thing", "endpoint": "", "command": "COMMIT", "wait_event": "ClientRead", "state": "idle", "wait_event_type": "Client"}, value: 25, metricType: dto.MetricType_GAUGE}, - {labels: labelMap{"usename": "postgres", "application": "service_thing", "endpoint": "", "command": "COMMIT", "wait_event": "ClientRead", "state": "idle", "wait_event_type": "Client"}, value: 0, metricType: dto.MetricType_GAUGE}, - } - convey.Convey("Metrics comparison", t, func() { - for _, expect := range expected { - m := readMetric(<-ch) - convey.So(expect, convey.ShouldResemble, m) - } - }) - if err := mock.ExpectationsWereMet(); err != nil { - t.Errorf("there were unfulfilled exceptions: %s", err) - } -} From 534af41adc517c145adffb6d357f2bb86f34b8ae Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 11:28:42 -0700 Subject: [PATCH 26/39] Lint Signed-off-by: Felix Yuan --- collector/pg_statio_user_indexes.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go index 1ccedf0e9..604315b9e 100644 --- a/collector/pg_statio_user_indexes.go +++ b/collector/pg_statio_user_indexes.go @@ -29,8 +29,8 @@ type PGStatioUserIndexesCollector struct { const statioUserIndexesSubsystem = "statio_user_indexes" -func NewPGStatioUserIndexesCollector(collectorConfig) (Collector, error) { - return &PGStatioUserIndexesCollector{}, nil +func NewPGStatioUserIndexesCollector(config collectorConfig) (Collector, error) { + return &PGStatioUserIndexesCollector{log: config.logger}, nil } var ( From 8d6499fed24d2eee2e679a20c5041a5d68978d1b Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 23 Jun 2023 11:48:38 -0700 Subject: [PATCH 27/39] Disable all new queries by default, rename marginalia -> summary Signed-off-by: Felix Yuan --- collector/pg_archiver.go | 2 +- collector/pg_blocked.go | 2 +- collector/pg_database_wraparound.go | 2 +- collector/pg_index_size.go | 2 +- collector/pg_long_running_transactions.go | 2 +- ...> pg_long_running_transactions_summary.go} | 22 +++++++-------- collector/pg_oldest_blocked.go | 2 +- collector/pg_slow.go | 2 +- collector/pg_stat_activity_autovacuum.go | 2 +- .../pg_stat_activity_autovacuum_active.go | 2 +- ...ginalia.go => pg_stat_activity_summary.go} | 28 +++++++++---------- collector/pg_stat_user_indexes.go | 2 +- collector/pg_stat_walreceiver.go | 2 +- collector/pg_statio_user_indexes.go | 2 +- collector/pg_stuck_idle_in_transaction.go | 2 +- collector/pg_total_relation_size.go | 2 +- collector/pg_xid.go | 2 +- collector/pg_xlog_location.go | 2 +- 18 files changed, 41 insertions(+), 41 deletions(-) rename collector/{pg_long_running_transactions_marginalia.go => pg_long_running_transactions_summary.go} (68%) rename collector/{pg_stat_activity_marginalia.go => pg_stat_activity_summary.go} (73%) diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index b30a74378..74a6de658 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector("archiver", defaultEnabled, NewPGArchiverCollector) + registerCollector("archiver", defaultDisabled, NewPGArchiverCollector) } type PGArchiverCollector struct { diff --git a/collector/pg_blocked.go b/collector/pg_blocked.go index 1000c010d..0d0065a68 100644 --- a/collector/pg_blocked.go +++ b/collector/pg_blocked.go @@ -23,7 +23,7 @@ import ( const blockedSubsystem = "blocked" func init() { - registerCollector(blockedSubsystem, defaultEnabled, NewPGBlockedCollector) + registerCollector(blockedSubsystem, defaultDisabled, NewPGBlockedCollector) } type PGBlockedCollector struct { diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index 8d41666ad..1edb47224 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -23,7 +23,7 @@ import ( const databaseWraparoundSubsystem = "database_wraparound" func init() { - registerCollector(databaseWraparoundSubsystem, defaultEnabled, NewPGDatabaseWraparoundCollector) + registerCollector(databaseWraparoundSubsystem, defaultDisabled, NewPGDatabaseWraparoundCollector) } type PGDatabaseWraparoundCollector struct { diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go index 5852f30e9..3c622cd37 100644 --- a/collector/pg_index_size.go +++ b/collector/pg_index_size.go @@ -23,7 +23,7 @@ import ( const indexSizeSubsystem = "index_size" func init() { - registerCollector(indexSizeSubsystem, defaultEnabled, NewPGIndexSizeCollector) + registerCollector(indexSizeSubsystem, defaultDisabled, NewPGIndexSizeCollector) } type PGIndexSizeCollector struct { diff --git a/collector/pg_long_running_transactions.go b/collector/pg_long_running_transactions.go index 5a5425f6f..417539ab0 100644 --- a/collector/pg_long_running_transactions.go +++ b/collector/pg_long_running_transactions.go @@ -23,7 +23,7 @@ import ( const longRunningTransactionsSubsystem = "long_running_transactions" func init() { - registerCollector(longRunningTransactionsSubsystem, defaultEnabled, NewPGLongRunningTransactionsCollector) + registerCollector(longRunningTransactionsSubsystem, defaultDisabled, NewPGLongRunningTransactionsCollector) } type PGLongRunningTransactionsCollector struct { diff --git a/collector/pg_long_running_transactions_marginalia.go b/collector/pg_long_running_transactions_summary.go similarity index 68% rename from collector/pg_long_running_transactions_marginalia.go rename to collector/pg_long_running_transactions_summary.go index 4dfc421f4..1fe024b4e 100644 --- a/collector/pg_long_running_transactions_marginalia.go +++ b/collector/pg_long_running_transactions_summary.go @@ -20,29 +20,29 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const longRunningTransactionsMarginaliaSubsystem = "long_running_transactions_marginalia" +const longRunningTransactionsSummarySubsystem = "long_running_transactions_summary" func init() { - registerCollector(longRunningTransactionsMarginaliaSubsystem, defaultEnabled, NewPGLongRunningTransactionsMarginaliaCollector) + registerCollector(longRunningTransactionsSummarySubsystem, defaultDisabled, NewPGLongRunningTransactionsSummaryCollector) } -type PGLongRunningTransactionsMarginaliaCollector struct { +type PGLongRunningTransactionsSummaryCollector struct { log log.Logger } -func NewPGLongRunningTransactionsMarginaliaCollector(config collectorConfig) (Collector, error) { - return &PGLongRunningTransactionsMarginaliaCollector{log: config.logger}, nil +func NewPGLongRunningTransactionsSummaryCollector(config collectorConfig) (Collector, error) { + return &PGLongRunningTransactionsSummaryCollector{log: config.logger}, nil } var ( - longRunningTransactionsMarginaliaMaxAgeInSeconds = prometheus.NewDesc( - prometheus.BuildFQName(namespace, longRunningTransactionsMarginaliaSubsystem, "max_age_in_seconds"), + longRunningTransactionsSummaryMaxAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, longRunningTransactionsSummarySubsystem, "max_age_in_seconds"), "The current maximum transaction age in seconds", []string{"application", "endpoint"}, prometheus.Labels{}, ) - longRunningTransactionsMarginaliaQuery = ` + longRunningTransactionsSummaryQuery = ` SELECT activity.matches[1] AS application, activity.matches[2] AS endpoint, @@ -62,10 +62,10 @@ var ( ` ) -func (PGLongRunningTransactionsMarginaliaCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { +func (PGLongRunningTransactionsSummaryCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { db := instance.getDB() rows, err := db.QueryContext(ctx, - longRunningTransactionsMarginaliaQuery) + longRunningTransactionsSummaryQuery) if err != nil { return err @@ -81,7 +81,7 @@ func (PGLongRunningTransactionsMarginaliaCollector) Update(ctx context.Context, } ch <- prometheus.MustNewConstMetric( - longRunningTransactionsMarginaliaMaxAgeInSeconds, + longRunningTransactionsSummaryMaxAgeInSeconds, prometheus.GaugeValue, maxAgeInSeconds, application, endpoint, diff --git a/collector/pg_oldest_blocked.go b/collector/pg_oldest_blocked.go index 3c0c753ba..6bbf52ede 100644 --- a/collector/pg_oldest_blocked.go +++ b/collector/pg_oldest_blocked.go @@ -23,7 +23,7 @@ import ( const oldestBlockedSubsystem = "oldest_blocked" func init() { - registerCollector(oldestBlockedSubsystem, defaultEnabled, NewPGOldestBlockedCollector) + registerCollector(oldestBlockedSubsystem, defaultDisabled, NewPGOldestBlockedCollector) } type PGOldestBlockedCollector struct { diff --git a/collector/pg_slow.go b/collector/pg_slow.go index 88e550a84..c0b44eb45 100644 --- a/collector/pg_slow.go +++ b/collector/pg_slow.go @@ -23,7 +23,7 @@ import ( const slowSubsystem = "slow" func init() { - registerCollector(slowSubsystem, defaultEnabled, NewPGSlowCollector) + registerCollector(slowSubsystem, defaultDisabled, NewPGSlowCollector) } type PGSlowCollector struct { diff --git a/collector/pg_stat_activity_autovacuum.go b/collector/pg_stat_activity_autovacuum.go index fbd5268cb..3ed37ed5b 100644 --- a/collector/pg_stat_activity_autovacuum.go +++ b/collector/pg_stat_activity_autovacuum.go @@ -23,7 +23,7 @@ import ( const statActivityAutovacuumSubsystem = "stat_activity_autovacuum" func init() { - registerCollector(statActivityAutovacuumSubsystem, defaultEnabled, NewPGStatActivityAutovacuumCollector) + registerCollector(statActivityAutovacuumSubsystem, defaultDisabled, NewPGStatActivityAutovacuumCollector) } type PGStatActivityAutovacuumCollector struct { diff --git a/collector/pg_stat_activity_autovacuum_active.go b/collector/pg_stat_activity_autovacuum_active.go index 95989b98a..419ce9259 100644 --- a/collector/pg_stat_activity_autovacuum_active.go +++ b/collector/pg_stat_activity_autovacuum_active.go @@ -23,7 +23,7 @@ import ( const statActivityAutovacuumActiveSubsystem = "stat_activity_autovacuum_active" func init() { - registerCollector(statActivityAutovacuumActiveSubsystem, defaultEnabled, NewPGStatActivityAutovacuumActiveCollector) + registerCollector(statActivityAutovacuumActiveSubsystem, defaultDisabled, NewPGStatActivityAutovacuumActiveCollector) } type PGStatActivityAutovacuumActiveCollector struct { diff --git a/collector/pg_stat_activity_marginalia.go b/collector/pg_stat_activity_summary.go similarity index 73% rename from collector/pg_stat_activity_marginalia.go rename to collector/pg_stat_activity_summary.go index bb5851c1d..0b3290501 100644 --- a/collector/pg_stat_activity_marginalia.go +++ b/collector/pg_stat_activity_summary.go @@ -20,35 +20,35 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const statActivityMarginaliaSubsystem = "stat_activity_marginalia" +const statActivitySummarySubsystem = "stat_activity_summary" func init() { - registerCollector(statActivityMarginaliaSubsystem, defaultEnabled, NewPGStatActivityMarginaliaCollector) + registerCollector(statActivitySummarySubsystem, defaultEnabled, NewPGStatActivitySummaryCollector) } -type PGStatActivityMarginaliaCollector struct { +type PGStatActivitySummaryCollector struct { log log.Logger } -func NewPGStatActivityMarginaliaCollector(config collectorConfig) (Collector, error) { - return &PGStatActivityMarginaliaCollector{log: config.logger}, nil +func NewPGStatActivitySummaryCollector(config collectorConfig) (Collector, error) { + return &PGStatActivitySummaryCollector{log: config.logger}, nil } var ( - statActivityMarginaliaActiveCount = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statActivityMarginaliaSubsystem, "active_count"), + statActivitySummaryActiveCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivitySummarySubsystem, "active_count"), "Number of active queries at time of sample", []string{"usename", "application", "endpoint", "command", "state", "wait_event", "wait_event_type"}, prometheus.Labels{}, ) - statActivityMarginaliaMaxTxAgeInSeconds = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statActivityMarginaliaSubsystem, "max_tx_age_in_seconds"), + statActivitySummaryMaxTxAgeInSeconds = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statActivitySummarySubsystem, "max_tx_age_in_seconds"), "Number of active queries at time of sample", []string{"usename", "application", "endpoint", "command", "state", "wait_event", "wait_event_type"}, prometheus.Labels{}, ) - statActivityMarginaliaQuery = ` + statActivitySummaryQuery = ` SELECT usename AS usename, a.matches[1] AS application, @@ -75,10 +75,10 @@ var ( ` ) -func (PGStatActivityMarginaliaCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { +func (PGStatActivitySummaryCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { db := instance.getDB() rows, err := db.QueryContext(ctx, - statActivityMarginaliaQuery) + statActivitySummaryQuery) if err != nil { return err @@ -94,13 +94,13 @@ func (PGStatActivityMarginaliaCollector) Update(ctx context.Context, instance *i } ch <- prometheus.MustNewConstMetric( - statActivityMarginaliaActiveCount, + statActivitySummaryActiveCount, prometheus.GaugeValue, count, usename, application, endpoint, command, state, waitEvent, waitEventType, ) ch <- prometheus.MustNewConstMetric( - statActivityMarginaliaMaxTxAgeInSeconds, + statActivitySummaryMaxTxAgeInSeconds, prometheus.GaugeValue, maxTxAge, usename, application, endpoint, command, state, waitEvent, waitEventType, diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index e4e66476a..6d69e681d 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector(statUserIndexesSubsystem, defaultEnabled, NewPGStatUserIndexesCollector) + registerCollector(statUserIndexesSubsystem, defaultDisabled, NewPGStatUserIndexesCollector) } type PGStatUserIndexesCollector struct { diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index df2e8b7ea..64e6c4633 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector(statWalReceiverSubsystem, defaultEnabled, NewPGStatWalReceiverCollector) + registerCollector(statWalReceiverSubsystem, defaultDisabled, NewPGStatWalReceiverCollector) } type PGStatWalReceiverCollector struct { diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go index 604315b9e..df9133ff1 100644 --- a/collector/pg_statio_user_indexes.go +++ b/collector/pg_statio_user_indexes.go @@ -20,7 +20,7 @@ import ( ) func init() { - registerCollector(statioUserIndexesSubsystem, defaultEnabled, NewPGStatioUserIndexesCollector) + registerCollector(statioUserIndexesSubsystem, defaultDisabled, NewPGStatioUserIndexesCollector) } type PGStatioUserIndexesCollector struct { diff --git a/collector/pg_stuck_idle_in_transaction.go b/collector/pg_stuck_idle_in_transaction.go index cc2ec849a..e6839d4b3 100644 --- a/collector/pg_stuck_idle_in_transaction.go +++ b/collector/pg_stuck_idle_in_transaction.go @@ -23,7 +23,7 @@ import ( const stuckIdleInTransactionSubsystem = "stuck_in_transaction" func init() { - registerCollector(stuckIdleInTransactionSubsystem, defaultEnabled, NewPGStuckIdleInTransactionCollector) + registerCollector(stuckIdleInTransactionSubsystem, defaultDisabled, NewPGStuckIdleInTransactionCollector) } type PGStuckIdleInTransactionCollector struct { diff --git a/collector/pg_total_relation_size.go b/collector/pg_total_relation_size.go index bb4812595..96b000172 100644 --- a/collector/pg_total_relation_size.go +++ b/collector/pg_total_relation_size.go @@ -23,7 +23,7 @@ import ( const totalRelationSizeSubsystem = "total_relation_size" func init() { - registerCollector(totalRelationSizeSubsystem, defaultEnabled, NewPGTotalRelationSizeCollector) + registerCollector(totalRelationSizeSubsystem, defaultDisabled, NewPGTotalRelationSizeCollector) } type PGTotalRelationSizeCollector struct { diff --git a/collector/pg_xid.go b/collector/pg_xid.go index 5b7650be4..c5db64328 100644 --- a/collector/pg_xid.go +++ b/collector/pg_xid.go @@ -23,7 +23,7 @@ import ( const xidSubsystem = "xid" func init() { - registerCollector(xidSubsystem, defaultEnabled, NewPGXidCollector) + registerCollector(xidSubsystem, defaultDisabled, NewPGXidCollector) } type PGXidCollector struct { diff --git a/collector/pg_xlog_location.go b/collector/pg_xlog_location.go index 42c7adafb..863d4e375 100644 --- a/collector/pg_xlog_location.go +++ b/collector/pg_xlog_location.go @@ -23,7 +23,7 @@ import ( const xlogLocationSubsystem = "xlog_location" func init() { - registerCollector(xlogLocationSubsystem, defaultEnabled, NewPGXlogLocationCollector) + registerCollector(xlogLocationSubsystem, defaultDisabled, NewPGXlogLocationCollector) } type PGXlogLocationCollector struct { From 4891644591ae303723afcadc78886439afcf52c4 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 10:19:12 -0700 Subject: [PATCH 28/39] Update pg_blocked with nulls Signed-off-by: Felix Yuan --- collector/pg_blocked.go | 18 +++++++++++++---- collector/pg_blocked_test.go | 39 ++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/collector/pg_blocked.go b/collector/pg_blocked.go index 0d0065a68..f215ed8c7 100644 --- a/collector/pg_blocked.go +++ b/collector/pg_blocked.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -70,18 +71,27 @@ func (PGBlockedCollector) Update(ctx context.Context, instance *instance, ch cha defer rows.Close() for rows.Next() { - var table string - var queries float64 + var table sql.NullString + var queries sql.NullFloat64 if err := rows.Scan(&queries, &table); err != nil { return err } + tableLabel := "unknown" + if table.Valid { + tableLabel = table.String + } + + queriesMetric := 0.0 + if queries.Valid { + queriesMetric = queries.Float64 + } ch <- prometheus.MustNewConstMetric( blockedQueries, prometheus.GaugeValue, - queries, - table, + queriesMetric, + tableLabel, ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_blocked_test.go b/collector/pg_blocked_test.go index e05376b5b..281edc824 100644 --- a/collector/pg_blocked_test.go +++ b/collector/pg_blocked_test.go @@ -60,3 +60,42 @@ func TestPgBlockedCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgBlockedCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "queries", + "table", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil) + + mock.ExpectQuery(sanitizeQuery(blockedQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGBlockedCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGBlockedCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"table": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 4adddb3a84ecb8fc1c82ab5712fa90b5397a8691 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 10:41:01 -0700 Subject: [PATCH 29/39] Db wraparound test nullable Signed-off-by: Felix Yuan --- collector/pg_database_wraparound.go | 24 +++++++++++--- collector/pg_database_wraparound_test.go | 41 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 4 deletions(-) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index 1edb47224..e9aefa884 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -71,22 +72,37 @@ func (PGDatabaseWraparoundCollector) Update(ctx context.Context, instance *insta defer rows.Close() for rows.Next() { - var datname string - var ageDatfrozenxid, ageDatminmxid float64 + var datname sql.NullString + var ageDatfrozenxid, ageDatminmxid sql.NullFloat64 if err := rows.Scan(&datname, &ageDatfrozenxid, &ageDatminmxid); err != nil { return err } + datnameLabel := "unknown" + if datname.Valid { + datnameLabel = datname.String + } + + ageDatfrozenxidMetric := 0.0 + if ageDatfrozenxid.Valid { + ageDatfrozenxidMetric = ageDatfrozenxid.Float64 + } + ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatfrozenxid, prometheus.GaugeValue, - ageDatfrozenxid, datname, + ageDatfrozenxidMetric, datnameLabel, ) + + ageDatminmxidMetric := 0.0 + if ageDatminmxid.Valid { + ageDatminmxidMetric = ageDatminmxid.Float64 + } ch <- prometheus.MustNewConstMetric( databaseWraparoundAgeDatminmxid, prometheus.GaugeValue, - ageDatminmxid, datname, + ageDatminmxidMetric, datnameLabel, ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_database_wraparound_test.go b/collector/pg_database_wraparound_test.go index d0a74c362..e9ca70335 100644 --- a/collector/pg_database_wraparound_test.go +++ b/collector/pg_database_wraparound_test.go @@ -62,3 +62,44 @@ func TestPGDatabaseWraparoundCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPGDatabaseWraparoundCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "datname", + "age_datfrozenxid", + "age_datminmxid", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(databaseWraparoundQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGDatabaseWraparoundCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGDatabaseWraparoundCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"datname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 8b7ffa6ded951d766c3bb50ad79efcdaea265233 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 10:45:09 -0700 Subject: [PATCH 30/39] index size nullable Signed-off-by: Felix Yuan --- collector/pg_index_size.go | 25 ++++++++++++++++---- collector/pg_index_size_test.go | 41 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 4 deletions(-) diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go index 3c622cd37..27b9620f9 100644 --- a/collector/pg_index_size.go +++ b/collector/pg_index_size.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -67,18 +68,34 @@ func (PGIndexSizeCollector) Update(ctx context.Context, instance *instance, ch c defer rows.Close() for rows.Next() { - var schemaname, relname, indexrelname string - var indexSize float64 + var schemaname, relname, indexrelname sql.NullString + var indexSize sql.NullFloat64 if err := rows.Scan(&schemaname, &relname, &indexrelname, &indexSize); err != nil { return err } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + indexrelnameLabel := "unknown" + if indexrelname.Valid { + indexrelnameLabel = indexrelname.String + } + indexSizeMetric := 0.0 + if indexSize.Valid { + indexSizeMetric = indexSize.Float64 + } ch <- prometheus.MustNewConstMetric( indexSizeDesc, prometheus.GaugeValue, - indexSize, - schemaname, relname, indexrelname, + indexSizeMetric, + schemanameLabel, relnameLabel, indexrelnameLabel, ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_index_size_test.go b/collector/pg_index_size_test.go index b80680260..8adaa4709 100644 --- a/collector/pg_index_size_test.go +++ b/collector/pg_index_size_test.go @@ -62,3 +62,44 @@ func TestPgIndexSizeCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgIndexSizeCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "index_size", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(indexSizeQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGIndexSizeCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGIndexSizeCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 532c04f37bda670ed0343596964d04754e9a4a1d Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 11:02:36 -0700 Subject: [PATCH 31/39] LongRunningTransactionsSUmmary nullable Signed-off-by: Felix Yuan --- .../pg_long_running_transactions_summary.go | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/collector/pg_long_running_transactions_summary.go b/collector/pg_long_running_transactions_summary.go index 1fe024b4e..135a32195 100644 --- a/collector/pg_long_running_transactions_summary.go +++ b/collector/pg_long_running_transactions_summary.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -73,18 +74,31 @@ func (PGLongRunningTransactionsSummaryCollector) Update(ctx context.Context, ins defer rows.Close() for rows.Next() { - var application, endpoint string - var maxAgeInSeconds float64 + var application, endpoint sql.NullString + var maxAgeInSeconds sql.NullFloat64 if err := rows.Scan(&application, &endpoint, &maxAgeInSeconds); err != nil { return err } + applicationLabel := "unknown" + if application.Valid { + applicationLabel = application.String + } + endpointLabel := "unknown" + if endpoint.Valid { + endpointLabel = endpoint.String + } + + maxAgeInSecondsMetric := 0.0 + if maxAgeInSeconds.Valid { + maxAgeInSecondsMetric = maxAgeInSeconds.Float64 + } ch <- prometheus.MustNewConstMetric( longRunningTransactionsSummaryMaxAgeInSeconds, prometheus.GaugeValue, - maxAgeInSeconds, - application, endpoint, + maxAgeInSecondsMetric, + applicationLabel, endpointLabel, ) } if err := rows.Err(); err != nil { From f8397785db2479975814c9f6311b67a0fdab39ba Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 11:40:38 -0700 Subject: [PATCH 32/39] stat user indexes nullable Signed-off-by: Felix Yuan --- collector/pg_stat_user_indexes.go | 44 ++++++++++++++++++++----- collector/pg_stat_user_indexes_test.go | 45 ++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index 6d69e681d..cceef1178 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -14,6 +14,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -75,30 +76,57 @@ func (c *PGStatUserIndexesCollector) Update(ctx context.Context, instance *insta } defer rows.Close() for rows.Next() { - var schemaname, relname, indexrelname string - var idxScan, idxTupRead, idxTupFetch float64 + var schemaname, relname, indexrelname sql.NullString + var idxScan, idxTupRead, idxTupFetch sql.NullFloat64 if err := rows.Scan(&schemaname, &relname, &indexrelname, &idxScan, &idxTupRead, &idxTupFetch); err != nil { return err } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + indexrelnameLabel := "unknown" + if indexrelname.Valid { + indexrelnameLabel = indexrelname.String + } + labels := []string{schemanameLabel, relnameLabel, indexrelnameLabel} + idxScanMetric := 0.0 + if idxScan.Valid { + idxScanMetric = idxScan.Float64 + } ch <- prometheus.MustNewConstMetric( statUserIndexesIdxScan, prometheus.CounterValue, - idxScan, - schemaname, relname, indexrelname, + idxScanMetric, + labels..., ) + + idxTupReadMetric := 0.0 + if idxTupRead.Valid { + idxTupReadMetric = idxTupRead.Float64 + } ch <- prometheus.MustNewConstMetric( statUserIndexesIdxTupRead, prometheus.CounterValue, - idxTupRead, - schemaname, relname, indexrelname, + idxTupReadMetric, + labels..., ) + + idxTupFetchMetric := 0.0 + if idxTupFetch.Valid { + idxTupFetchMetric = idxTupFetch.Float64 + } ch <- prometheus.MustNewConstMetric( statUserIndexesIdxTupFetch, prometheus.CounterValue, - idxTupFetch, - schemaname, relname, indexrelname, + idxTupFetchMetric, + labels..., ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_stat_user_indexes_test.go b/collector/pg_stat_user_indexes_test.go index ce59355b7..b0c7fde19 100644 --- a/collector/pg_stat_user_indexes_test.go +++ b/collector/pg_stat_user_indexes_test.go @@ -66,3 +66,48 @@ func TestPgStatUserIndexesCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgStatUserIndexesCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "idx_scan", + "idx_tup_read", + "idx_tup_fetch", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil, nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(statUserIndexesQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatUserIndexesCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatUserIndexesCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From fddba48c8c4e7fae533bbcc70a9d71f189975b4c Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:27:16 -0700 Subject: [PATCH 33/39] stat walreceiver nil Signed-off-by: Felix Yuan --- collector/pg_stat_walreceiver.go | 105 ++++++++++++++++++-------- collector/pg_stat_walreceiver_test.go | 83 ++++++++++++++++++++ 2 files changed, 156 insertions(+), 32 deletions(-) diff --git a/collector/pg_stat_walreceiver.go b/collector/pg_stat_walreceiver.go index 64e6c4633..deb9aa733 100644 --- a/collector/pg_stat_walreceiver.go +++ b/collector/pg_stat_walreceiver.go @@ -14,6 +14,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -173,18 +174,9 @@ func (c *PGStatWalReceiverCollector) Update(ctx context.Context, instance *insta } defer rows.Close() for rows.Next() { - var upstreamHost string - var slotName string - var status int - var receiveStartLsn int64 - var receiveStartTli int64 - var flushedLsn int64 - var receivedTli int64 - var lastMsgSendTime float64 - var lastMsgReceiptTime float64 - var latestEndLsn int64 - var latestEndTime float64 - var upstreamNode int + var upstreamHost, slotName sql.NullString + var status, receiveStartLsn, receiveStartTli, flushedLsn, receivedTli, latestEndLsn, upstreamNode sql.NullInt64 + var lastMsgSendTime, lastMsgReceiptTime, latestEndTime sql.NullFloat64 if hasFlushedLSN { if err := rows.Scan(&upstreamHost, &slotName, &status, &receiveStartLsn, &receiveStartTli, &flushedLsn, &receivedTli, &lastMsgSendTime, &lastMsgReceiptTime, &latestEndLsn, &latestEndTime, &upstreamNode); err != nil { @@ -195,68 +187,117 @@ func (c *PGStatWalReceiverCollector) Update(ctx context.Context, instance *insta return err } } + upstreamHostLabel := "unknown" + if upstreamHost.Valid { + upstreamHostLabel = upstreamHost.String + } + slotNameLabel := "unknown" + if slotName.Valid { + slotNameLabel = slotName.String + } + labels := []string{upstreamHostLabel, slotNameLabel} + statusMetric := 0.0 + if status.Valid { + statusMetric = float64(status.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverStatus, prometheus.GaugeValue, - float64(status), - upstreamHost, slotName) + statusMetric, + labels...) + receiveStartLsnMetric := 0.0 + if receiveStartLsn.Valid { + receiveStartLsnMetric = float64(receiveStartLsn.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverReceiveStartLsn, prometheus.CounterValue, - float64(receiveStartLsn), - upstreamHost, slotName) + receiveStartLsnMetric, + labels...) + receiveStartTliMetric := 0.0 + if receiveStartTli.Valid { + receiveStartTliMetric = float64(receiveStartTli.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverReceiveStartTli, prometheus.GaugeValue, - float64(receiveStartTli), - upstreamHost, slotName) + receiveStartTliMetric, + labels...) if hasFlushedLSN { + flushedLsnMetric := 0.0 + if flushedLsn.Valid { + flushedLsnMetric = float64(flushedLsn.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverFlushedLSN, prometheus.CounterValue, - float64(flushedLsn), - upstreamHost, slotName) + flushedLsnMetric, + labels...) } + receivedTliMetric := 0.0 + if receivedTli.Valid { + receivedTliMetric = float64(receivedTli.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverReceivedTli, prometheus.GaugeValue, - float64(receivedTli), - upstreamHost, slotName) + receivedTliMetric, + labels...) + lastMsgSendTimeMetric := 0.0 + if lastMsgSendTime.Valid { + lastMsgSendTimeMetric = float64(lastMsgSendTime.Float64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverLastMsgSendTime, prometheus.CounterValue, - float64(lastMsgSendTime), - upstreamHost, slotName) + lastMsgSendTimeMetric, + labels...) + lastMsgReceiptTimeMetric := 0.0 + if lastMsgReceiptTime.Valid { + lastMsgReceiptTimeMetric = float64(lastMsgReceiptTime.Float64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverLastMsgReceiptTime, prometheus.CounterValue, - float64(lastMsgReceiptTime), - upstreamHost, slotName) + lastMsgReceiptTimeMetric, + labels...) + latestEndLsnMetric := 0.0 + if latestEndLsn.Valid { + latestEndLsnMetric = float64(latestEndLsn.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverLatestEndLsn, prometheus.CounterValue, - float64(latestEndLsn), - upstreamHost, slotName) + latestEndLsnMetric, + labels...) + latestEndTimeMetric := 0.0 + if latestEndTime.Valid { + latestEndTimeMetric = float64(latestEndTime.Float64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverLatestEndTime, prometheus.CounterValue, - float64(latestEndTime), - upstreamHost, slotName) + latestEndTimeMetric, + labels...) + upstreamNodeMetric := 0.0 + if upstreamNode.Valid { + upstreamNodeMetric = float64(upstreamNode.Int64) + } ch <- prometheus.MustNewConstMetric( statWalReceiverUpstreamNode, prometheus.GaugeValue, - float64(upstreamNode), - upstreamHost, slotName) + upstreamNodeMetric, + labels...) } if err := rows.Err(); err != nil { return err diff --git a/collector/pg_stat_walreceiver_test.go b/collector/pg_stat_walreceiver_test.go index f9de36c47..6a7dc1cce 100644 --- a/collector/pg_stat_walreceiver_test.go +++ b/collector/pg_stat_walreceiver_test.go @@ -181,3 +181,86 @@ func TestPGStatWalReceiverCollectorWithNoFlushedLSN(t *testing.T) { } } + +func TestPGStatWalReceiverCollectorWithFlushedLSNNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + infoSchemaColumns := []string{ + "column_name", + } + + infoSchemaRows := sqlmock.NewRows(infoSchemaColumns). + AddRow( + "flushed_lsn", + ) + + mock.ExpectQuery(sanitizeQuery(pgStatWalColumnQuery)).WillReturnRows(infoSchemaRows) + + columns := []string{ + "upstream_host", + "slot_name", + "status", + "receive_start_lsn", + "receive_start_tli", + "flushed_lsn", + "received_tli", + "last_msg_send_time", + "last_msg_receipt_time", + "latest_end_lsn", + "latest_end_time", + "upstream_node", + } + rows := sqlmock.NewRows(columns). + AddRow( + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + nil, + ) + mock.ExpectQuery(sanitizeQuery(pgStatWalReceiverQueryWithFlushedLSN)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatWalReceiverCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PgStatWalReceiverCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"upstream_host": "unknown", "slot_name": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } + +} From 6513418aa844b311eb5b8ac0c4859a644c997515 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:32:07 -0700 Subject: [PATCH 34/39] statiouser_indexes nullable Signed-off-by: Felix Yuan --- collector/pg_statio_user_indexes.go | 35 +++++++++++++++---- collector/pg_statio_user_indexes_test.go | 43 ++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/collector/pg_statio_user_indexes.go b/collector/pg_statio_user_indexes.go index df9133ff1..18a29952b 100644 --- a/collector/pg_statio_user_indexes.go +++ b/collector/pg_statio_user_indexes.go @@ -14,6 +14,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -68,24 +69,46 @@ func (c *PGStatioUserIndexesCollector) Update(ctx context.Context, instance *ins } defer rows.Close() for rows.Next() { - var schemaname, relname, indexrelname string - var idxBlksRead, idxBlksHit float64 + var schemaname, relname, indexrelname sql.NullString + var idxBlksRead, idxBlksHit sql.NullFloat64 if err := rows.Scan(&schemaname, &relname, &indexrelname, &idxBlksRead, &idxBlksHit); err != nil { return err } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + indexrelnameLabel := "unknown" + if indexrelname.Valid { + indexrelnameLabel = indexrelname.String + } + labels := []string{schemanameLabel, relnameLabel, indexrelnameLabel} + idxBlksReadMetric := 0.0 + if idxBlksRead.Valid { + idxBlksReadMetric = idxBlksRead.Float64 + } ch <- prometheus.MustNewConstMetric( statioUserIndexesIdxBlksRead, prometheus.CounterValue, - idxBlksRead, - schemaname, relname, indexrelname, + idxBlksReadMetric, + labels..., ) + + idxBlksHitMetric := 0.0 + if idxBlksHit.Valid { + idxBlksHitMetric = idxBlksHit.Float64 + } ch <- prometheus.MustNewConstMetric( statioUserIndexesIdxBlksHit, prometheus.CounterValue, - idxBlksHit, - schemaname, relname, indexrelname, + idxBlksHitMetric, + labels..., ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_statio_user_indexes_test.go b/collector/pg_statio_user_indexes_test.go index 7f599c3e7..174012162 100644 --- a/collector/pg_statio_user_indexes_test.go +++ b/collector/pg_statio_user_indexes_test.go @@ -64,3 +64,46 @@ func TestPgStatioUserIndexesCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgStatioUserIndexesCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "indexrelname", + "idx_blks_read", + "idx_blks_hit", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(statioUserIndexesQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatioUserIndexesCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatioUserIndexesCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + {labels: labelMap{"schemaname": "unknown", "relname": "unknown", "indexrelname": "unknown"}, value: 0, metricType: dto.MetricType_COUNTER}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From aa910f31e81031bd0a0452119ddd8f55d35c6998 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:36:52 -0700 Subject: [PATCH 35/39] total relation size nullable Signed-off-by: Felix Yuan --- collector/pg_total_relation_size.go | 22 ++++++++++--- collector/pg_total_relation_size_test.go | 40 ++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/collector/pg_total_relation_size.go b/collector/pg_total_relation_size.go index 96b000172..b9024833e 100644 --- a/collector/pg_total_relation_size.go +++ b/collector/pg_total_relation_size.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -63,18 +64,31 @@ func (PGTotalRelationSizeCollector) Update(ctx context.Context, instance *instan defer rows.Close() for rows.Next() { - var schemaname, relname string - var bytes float64 + var schemaname, relname sql.NullString + var bytes sql.NullFloat64 if err := rows.Scan(&schemaname, &relname, &bytes); err != nil { return err } + schemanameLabel := "unknown" + if schemaname.Valid { + schemanameLabel = schemaname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + labels := []string{schemanameLabel, relnameLabel} + bytesMetric := 0.0 + if bytes.Valid { + bytesMetric = bytes.Float64 + } ch <- prometheus.MustNewConstMetric( totalRelationSizeBytes, prometheus.GaugeValue, - bytes, - schemaname, relname, + bytesMetric, + labels..., ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_total_relation_size_test.go b/collector/pg_total_relation_size_test.go index 2b835095c..c13bd8045 100644 --- a/collector/pg_total_relation_size_test.go +++ b/collector/pg_total_relation_size_test.go @@ -61,3 +61,43 @@ func TestPgTotalRelationSizeCollector(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPgTotalRelationSizeCollectorNull(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + inst := &instance{db: db} + columns := []string{ + "schemaname", + "relname", + "bytes", + } + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(totalRelationSizeQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGTotalRelationSizeCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGTotalRelationSizeCollector.Update: %s", err) + } + }() + expected := []MetricResult{ + {labels: labelMap{"schemaname": "unknown", "relname": "unknown"}, value: 0, metricType: dto.MetricType_GAUGE}, + } + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 2d33e6139f91d5a73686ef8922df799a8be08054 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:38:01 -0700 Subject: [PATCH 36/39] Remove linebreak Signed-off-by: Felix Yuan --- collector/pg_xlog_location.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/collector/pg_xlog_location.go b/collector/pg_xlog_location.go index 863d4e375..b281d7a51 100644 --- a/collector/pg_xlog_location.go +++ b/collector/pg_xlog_location.go @@ -44,8 +44,7 @@ var ( xlogLocationQuery = ` SELECT CASE - WHEN pg_is_in_recovery() - THEN (pg_last_xlog_replay_location() - '0/0') % (2^52)::bigint + WHEN pg_is_in_recovery() THEN (pg_last_xlog_replay_location() - '0/0') % (2^52)::bigint ELSE (pg_current_xlog_location() - '0/0') % (2^52)::bigint END AS bytes ` From 55034b3fa5039589a732d6608145857bf4d65f3c Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:39:12 -0700 Subject: [PATCH 37/39] Use labels pattern Signed-off-by: Felix Yuan --- collector/pg_index_size.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go index 27b9620f9..b486e4da0 100644 --- a/collector/pg_index_size.go +++ b/collector/pg_index_size.go @@ -86,6 +86,7 @@ func (PGIndexSizeCollector) Update(ctx context.Context, instance *instance, ch c if indexrelname.Valid { indexrelnameLabel = indexrelname.String } + labels := []string{schemanameLabel, relnameLabel, indexrelnameLabel} indexSizeMetric := 0.0 if indexSize.Valid { @@ -95,7 +96,7 @@ func (PGIndexSizeCollector) Update(ctx context.Context, instance *instance, ch c indexSizeDesc, prometheus.GaugeValue, indexSizeMetric, - schemanameLabel, relnameLabel, indexrelnameLabel, + labels..., ) } if err := rows.Err(); err != nil { From 57ea9d84da1615171151a1a967629c92cc9c7640 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Mon, 26 Jun 2023 12:45:06 -0700 Subject: [PATCH 38/39] stat activity summary nullable Signed-off-by: Felix Yuan --- .../pg_long_running_transactions_summary.go | 3 +- .../pg_stat_activity_autovacuum_active.go | 3 +- collector/pg_stat_activity_summary.go | 51 ++++++++++++++++--- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/collector/pg_long_running_transactions_summary.go b/collector/pg_long_running_transactions_summary.go index 135a32195..0fb70c717 100644 --- a/collector/pg_long_running_transactions_summary.go +++ b/collector/pg_long_running_transactions_summary.go @@ -89,6 +89,7 @@ func (PGLongRunningTransactionsSummaryCollector) Update(ctx context.Context, ins if endpoint.Valid { endpointLabel = endpoint.String } + labels := []string{applicationLabel, endpointLabel} maxAgeInSecondsMetric := 0.0 if maxAgeInSeconds.Valid { @@ -98,7 +99,7 @@ func (PGLongRunningTransactionsSummaryCollector) Update(ctx context.Context, ins longRunningTransactionsSummaryMaxAgeInSeconds, prometheus.GaugeValue, maxAgeInSecondsMetric, - applicationLabel, endpointLabel, + labels..., ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_stat_activity_autovacuum_active.go b/collector/pg_stat_activity_autovacuum_active.go index 419ce9259..930663786 100644 --- a/collector/pg_stat_activity_autovacuum_active.go +++ b/collector/pg_stat_activity_autovacuum_active.go @@ -75,12 +75,13 @@ func (PGStatActivityAutovacuumActiveCollector) Update(ctx context.Context, insta if err := rows.Scan(&phase, &mode, &workersCount); err != nil { return err } + labels := []string{phase, mode} ch <- prometheus.MustNewConstMetric( statActivityAutovacuumActiveWorkersCount, prometheus.GaugeValue, workersCount, - phase, mode, + labels..., ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_stat_activity_summary.go b/collector/pg_stat_activity_summary.go index 0b3290501..29c435c56 100644 --- a/collector/pg_stat_activity_summary.go +++ b/collector/pg_stat_activity_summary.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -86,24 +87,62 @@ func (PGStatActivitySummaryCollector) Update(ctx context.Context, instance *inst defer rows.Close() for rows.Next() { - var usename, application, endpoint, command, state, waitEvent, waitEventType string - var count, maxTxAge float64 + var usename, application, endpoint, command, state, waitEvent, waitEventType sql.NullString + var count, maxTxAge sql.NullFloat64 if err := rows.Scan(&usename, &application, &endpoint, &command, &state, &waitEvent, &waitEventType, &count, &maxTxAge); err != nil { return err } + usenameLabel := "unknown" + if usename.Valid { + usenameLabel = usename.String + } + applicationLabel := "unknown" + if application.Valid { + applicationLabel = application.String + } + endpointLabel := "unknown" + if endpoint.Valid { + endpointLabel = endpoint.String + } + commandLabel := "unknown" + if command.Valid { + commandLabel = command.String + } + stateLabel := "unknown" + if state.Valid { + stateLabel = state.String + } + waitEventLabel := "unknown" + if waitEvent.Valid { + waitEventLabel = waitEvent.String + } + waitEventTypeLabel := "unknown" + if waitEventType.Valid { + waitEventTypeLabel = waitEventType.String + } + labels := []string{usenameLabel, applicationLabel, endpointLabel, commandLabel, stateLabel, waitEventLabel, waitEventTypeLabel} + countMetric := 0.0 + if count.Valid { + countMetric = count.Float64 + } ch <- prometheus.MustNewConstMetric( statActivitySummaryActiveCount, prometheus.GaugeValue, - count, - usename, application, endpoint, command, state, waitEvent, waitEventType, + countMetric, + labels..., ) + + maxTxAgeMetric := 0.0 + if maxTxAge.Valid { + maxTxAgeMetric = maxTxAge.Float64 + } ch <- prometheus.MustNewConstMetric( statActivitySummaryMaxTxAgeInSeconds, prometheus.GaugeValue, - maxTxAge, - usename, application, endpoint, command, state, waitEvent, waitEventType, + maxTxAgeMetric, + labels..., ) } if err := rows.Err(); err != nil { From 3718e5279fb6e11c537017c2de8af837956734aa Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Wed, 28 Jun 2023 10:31:41 -0700 Subject: [PATCH 39/39] Redo all change requests --- collector/pg_archiver.go | 2 +- collector/pg_database_wraparound.go | 2 +- collector/pg_index_size.go | 2 +- collector/pg_long_running_transactions.go | 2 +- collector/pg_stat_activity_autovacuum_active.go | 2 +- collector/pg_stat_activity_autovacuum_test.go | 2 +- collector/pg_stat_user_indexes.go | 6 +++--- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/collector/pg_archiver.go b/collector/pg_archiver.go index 74a6de658..9c33a0e44 100644 --- a/collector/pg_archiver.go +++ b/collector/pg_archiver.go @@ -35,7 +35,7 @@ func NewPGArchiverCollector(config collectorConfig) (Collector, error) { var ( pgArchiverPendingWalCount = prometheus.NewDesc( - prometheus.BuildFQName(namespace, archiverSubsystem, "pending_wal_count"), + prometheus.BuildFQName(namespace, archiverSubsystem, "pending_wals"), "Number of WAL files waiting to be archived", []string{}, prometheus.Labels{}, ) diff --git a/collector/pg_database_wraparound.go b/collector/pg_database_wraparound.go index e9aefa884..19f618f83 100644 --- a/collector/pg_database_wraparound.go +++ b/collector/pg_database_wraparound.go @@ -37,7 +37,7 @@ func NewPGDatabaseWraparoundCollector(config collectorConfig) (Collector, error) var ( databaseWraparoundAgeDatfrozenxid = prometheus.NewDesc( - prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datfrozenxid"), + prometheus.BuildFQName(namespace, databaseWraparoundSubsystem, "age_datfrozenxid_seconds"), "Age of the oldest transaction ID that has not been frozen.", []string{"datname"}, prometheus.Labels{}, diff --git a/collector/pg_index_size.go b/collector/pg_index_size.go index b486e4da0..fcc33d2c1 100644 --- a/collector/pg_index_size.go +++ b/collector/pg_index_size.go @@ -37,7 +37,7 @@ func NewPGIndexSizeCollector(config collectorConfig) (Collector, error) { var ( indexSizeDesc = prometheus.NewDesc( - "pg_index_size", + prometheus.BuildFQName(namespace, indexSizeSubsystem, "bytes"), "Size of the index as per pg_table_size function", []string{"schemaname", "relname", "indexrelname"}, prometheus.Labels{}, diff --git a/collector/pg_long_running_transactions.go b/collector/pg_long_running_transactions.go index 417539ab0..baedd7869 100644 --- a/collector/pg_long_running_transactions.go +++ b/collector/pg_long_running_transactions.go @@ -36,7 +36,7 @@ func NewPGLongRunningTransactionsCollector(config collectorConfig) (Collector, e var ( longRunningTransactionsCount = prometheus.NewDesc( - prometheus.BuildFQName(namespace, longRunningTransactionsSubsystem, "count"), + "pg_long_running_transactions", "Current number of long running transactions", []string{}, prometheus.Labels{}, diff --git a/collector/pg_stat_activity_autovacuum_active.go b/collector/pg_stat_activity_autovacuum_active.go index 930663786..1ad960e3c 100644 --- a/collector/pg_stat_activity_autovacuum_active.go +++ b/collector/pg_stat_activity_autovacuum_active.go @@ -36,7 +36,7 @@ func NewPGStatActivityAutovacuumActiveCollector(config collectorConfig) (Collect var ( statActivityAutovacuumActiveWorkersCount = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statActivityAutovacuumActiveSubsystem, "workers_count"), + prometheus.BuildFQName(namespace, statActivityAutovacuumActiveSubsystem, "workers"), "Current number of statActivityAutovacuumActive queries", []string{"phase", "mode"}, prometheus.Labels{}, diff --git a/collector/pg_stat_activity_autovacuum_test.go b/collector/pg_stat_activity_autovacuum_test.go index f79be2d2c..a6fcdbcad 100644 --- a/collector/pg_stat_activity_autovacuum_test.go +++ b/collector/pg_stat_activity_autovacuum_test.go @@ -31,7 +31,7 @@ func TestPGStatActivityAutovacuumCollector(t *testing.T) { inst := &instance{db: db} columns := []string{ "relname", - "age_in_seconds", + "timestamp_seconds", } rows := sqlmock.NewRows(columns). AddRow("test", 3600) diff --git a/collector/pg_stat_user_indexes.go b/collector/pg_stat_user_indexes.go index cceef1178..d7b289cc9 100644 --- a/collector/pg_stat_user_indexes.go +++ b/collector/pg_stat_user_indexes.go @@ -36,19 +36,19 @@ func NewPGStatUserIndexesCollector(config collectorConfig) (Collector, error) { var ( statUserIndexesIdxScan = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_scan"), + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_scans_total"), "Number of index scans initiated on this index", []string{"schemaname", "relname", "indexrelname"}, prometheus.Labels{}, ) statUserIndexesIdxTupRead = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_read"), + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_reads_total"), "Number of index entries returned by scans on this index", []string{"schemaname", "relname", "indexrelname"}, prometheus.Labels{}, ) statUserIndexesIdxTupFetch = prometheus.NewDesc( - prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_fetch"), + prometheus.BuildFQName(namespace, statUserIndexesSubsystem, "idx_tup_fetches_total"), "Number of live table rows fetched by simple index scans using this index", []string{"schemaname", "relname", "indexrelname"}, prometheus.Labels{},