From 5145620988a075c8f4c08ab045145510e3ddf1df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2025 18:36:32 +0100 Subject: [PATCH 01/33] Bump github.com/prometheus/exporter-toolkit from 0.13.1 to 0.13.2 (#1108) Bumps [github.com/prometheus/exporter-toolkit](https://github.com/prometheus/exporter-toolkit) from 0.13.1 to 0.13.2. - [Release notes](https://github.com/prometheus/exporter-toolkit/releases) - [Changelog](https://github.com/prometheus/exporter-toolkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/prometheus/exporter-toolkit/compare/v0.13.1...v0.13.2) --- updated-dependencies: - dependency-name: github.com/prometheus/exporter-toolkit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 40 ++++++++++++++++++++-------------------- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index bc854c3aa..9cdf3cbec 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,8 @@ require ( github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.60.1 - github.com/prometheus/exporter-toolkit v0.13.1 + github.com/prometheus/common v0.61.0 + github.com/prometheus/exporter-toolkit v0.13.2 github.com/smartystreets/goconvey v1.8.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 @@ -36,11 +36,11 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect - golang.org/x/crypto v0.28.0 // indirect - golang.org/x/net v0.29.0 // indirect - golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.26.0 // indirect - golang.org/x/text v0.19.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/net v0.32.0 // indirect + golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect + google.golang.org/protobuf v1.35.2 // indirect ) diff --git a/go.sum b/go.sum index 044805aa2..e20b36f15 100644 --- a/go.sum +++ b/go.sum @@ -54,10 +54,10 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= -github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= -github.com/prometheus/exporter-toolkit v0.13.1 h1:Evsh0gWQo2bdOHlnz9+0Nm7/OFfIwhE2Ws4A2jIlR04= -github.com/prometheus/exporter-toolkit v0.13.1/go.mod h1:ujdv2YIOxtdFxxqtloLpbqmxd5J0Le6IITUvIRSWjj0= +github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= +github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/exporter-toolkit v0.13.2 h1:Z02fYtbqTMy2i/f+xZ+UK5jy/bl1Ex3ndzh06T/Q9DQ= +github.com/prometheus/exporter-toolkit v0.13.2/go.mod h1:tCqnfx21q6qN1KA4U3Bfb8uWzXfijIrJz3/kTIqMV7g= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -69,24 +69,24 @@ github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sS github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= -golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= -golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= +golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= +golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From bea260951948d8fbf5fdd4059cce9f385aaced36 Mon Sep 17 00:00:00 2001 From: Nicolas Rodriguez Date: Wed, 1 Jan 2025 22:03:43 +0100 Subject: [PATCH 02/33] Checkpoint related columns in PG 17 have been moved from pg_stat_bgwriter to pg_stat_checkpointer (#1072) * Checkpoint related columns in PG 17 have been moved from pg_stat_bgwriter to pg_stat_checkpointer Fix https://github.com/prometheus-community/postgres_exporter/issues/1060 See: https://www.dbi-services.com/blog/postgresql-17-new-catalog-view-pg_stat_checkpointer/ Signed-off-by: Nicolas Rodriguez * Add support for pg_stat_checkpointer See: https://www.dbi-services.com/blog/postgresql-17-new-catalog-view-pg_stat_checkpointer/ Signed-off-by: Nicolas Rodriguez * Run integration tests with Postgres 17 Signed-off-by: Nicolas Rodriguez * Update date in file header Signed-off-by: Nicolas Rodriguez --------- Signed-off-by: Nicolas Rodriguez --- .circleci/config.yml | 1 + collector/pg_stat_bgwriter.go | 275 +++++++++++++++---------- collector/pg_stat_bgwriter_test.go | 4 +- collector/pg_stat_checkpointer.go | 221 ++++++++++++++++++++ collector/pg_stat_checkpointer_test.go | 143 +++++++++++++ 5 files changed, 533 insertions(+), 111 deletions(-) create mode 100644 collector/pg_stat_checkpointer.go create mode 100644 collector/pg_stat_checkpointer_test.go diff --git a/.circleci/config.yml b/.circleci/config.yml index d745e30f2..6cfc880d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,7 @@ workflows: - cimg/postgres:14.9 - cimg/postgres:15.4 - cimg/postgres:16.0 + - cimg/postgres:17.0 - prometheus/build: name: build parallelism: 3 diff --git a/collector/pg_stat_bgwriter.go b/collector/pg_stat_bgwriter.go index ec446d58c..6e3bd09cb 100644 --- a/collector/pg_stat_bgwriter.go +++ b/collector/pg_stat_bgwriter.go @@ -17,6 +17,7 @@ import ( "context" "database/sql" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" ) @@ -101,7 +102,7 @@ var ( prometheus.Labels{}, ) - statBGWriterQuery = `SELECT + statBGWriterQueryBefore17 = `SELECT checkpoints_timed ,checkpoints_req ,checkpoint_write_time @@ -114,121 +115,177 @@ var ( ,buffers_alloc ,stats_reset FROM pg_stat_bgwriter;` + + statBGWriterQueryAfter17 = `SELECT + buffers_clean + ,maxwritten_clean + ,buffers_alloc + ,stats_reset + FROM pg_stat_bgwriter;` ) func (PGStatBGWriterCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { - db := instance.getDB() - row := db.QueryRowContext(ctx, - statBGWriterQuery) + if instance.version.GE(semver.MustParse("17.0.0")) { + db := instance.getDB() + row := db.QueryRowContext(ctx, statBGWriterQueryAfter17) - var cpt, cpr, bcp, bc, mwc, bb, bbf, ba sql.NullInt64 - var cpwt, cpst sql.NullFloat64 - var sr sql.NullTime + var bc, mwc, ba sql.NullInt64 + var sr sql.NullTime - err := row.Scan(&cpt, &cpr, &cpwt, &cpst, &bcp, &bc, &mwc, &bb, &bbf, &ba, &sr) - if err != nil { - return err - } + err := row.Scan(&bc, &mwc, &ba, &sr) + if err != nil { + return err + } - cptMetric := 0.0 - if cpt.Valid { - cptMetric = float64(cpt.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterCheckpointsTimedDesc, - prometheus.CounterValue, - cptMetric, - ) - cprMetric := 0.0 - if cpr.Valid { - cprMetric = float64(cpr.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterCheckpointsReqDesc, - prometheus.CounterValue, - cprMetric, - ) - cpwtMetric := 0.0 - if cpwt.Valid { - cpwtMetric = float64(cpwt.Float64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterCheckpointsReqTimeDesc, - prometheus.CounterValue, - cpwtMetric, - ) - cpstMetric := 0.0 - if cpst.Valid { - cpstMetric = float64(cpst.Float64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterCheckpointsSyncTimeDesc, - prometheus.CounterValue, - cpstMetric, - ) - bcpMetric := 0.0 - if bcp.Valid { - bcpMetric = float64(bcp.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterBuffersCheckpointDesc, - prometheus.CounterValue, - bcpMetric, - ) - bcMetric := 0.0 - if bc.Valid { - bcMetric = float64(bc.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterBuffersCleanDesc, - prometheus.CounterValue, - bcMetric, - ) - mwcMetric := 0.0 - if mwc.Valid { - mwcMetric = float64(mwc.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterMaxwrittenCleanDesc, - prometheus.CounterValue, - mwcMetric, - ) - bbMetric := 0.0 - if bb.Valid { - bbMetric = float64(bb.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterBuffersBackendDesc, - prometheus.CounterValue, - bbMetric, - ) - bbfMetric := 0.0 - if bbf.Valid { - bbfMetric = float64(bbf.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterBuffersBackendFsyncDesc, - prometheus.CounterValue, - bbfMetric, - ) - baMetric := 0.0 - if ba.Valid { - baMetric = float64(ba.Int64) - } - ch <- prometheus.MustNewConstMetric( - statBGWriterBuffersAllocDesc, - prometheus.CounterValue, - baMetric, - ) - srMetric := 0.0 - if sr.Valid { - srMetric = float64(sr.Time.Unix()) + bcMetric := 0.0 + if bc.Valid { + bcMetric = float64(bc.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersCleanDesc, + prometheus.CounterValue, + bcMetric, + ) + mwcMetric := 0.0 + if mwc.Valid { + mwcMetric = float64(mwc.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterMaxwrittenCleanDesc, + prometheus.CounterValue, + mwcMetric, + ) + baMetric := 0.0 + if ba.Valid { + baMetric = float64(ba.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersAllocDesc, + prometheus.CounterValue, + baMetric, + ) + srMetric := 0.0 + if sr.Valid { + srMetric = float64(sr.Time.Unix()) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterStatsResetDesc, + prometheus.CounterValue, + srMetric, + ) + } else { + db := instance.getDB() + row := db.QueryRowContext(ctx, statBGWriterQueryBefore17) + + var cpt, cpr, bcp, bc, mwc, bb, bbf, ba sql.NullInt64 + var cpwt, cpst sql.NullFloat64 + var sr sql.NullTime + + err := row.Scan(&cpt, &cpr, &cpwt, &cpst, &bcp, &bc, &mwc, &bb, &bbf, &ba, &sr) + if err != nil { + return err + } + + cptMetric := 0.0 + if cpt.Valid { + cptMetric = float64(cpt.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterCheckpointsTimedDesc, + prometheus.CounterValue, + cptMetric, + ) + cprMetric := 0.0 + if cpr.Valid { + cprMetric = float64(cpr.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterCheckpointsReqDesc, + prometheus.CounterValue, + cprMetric, + ) + cpwtMetric := 0.0 + if cpwt.Valid { + cpwtMetric = float64(cpwt.Float64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterCheckpointsReqTimeDesc, + prometheus.CounterValue, + cpwtMetric, + ) + cpstMetric := 0.0 + if cpst.Valid { + cpstMetric = float64(cpst.Float64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterCheckpointsSyncTimeDesc, + prometheus.CounterValue, + cpstMetric, + ) + bcpMetric := 0.0 + if bcp.Valid { + bcpMetric = float64(bcp.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersCheckpointDesc, + prometheus.CounterValue, + bcpMetric, + ) + bcMetric := 0.0 + if bc.Valid { + bcMetric = float64(bc.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersCleanDesc, + prometheus.CounterValue, + bcMetric, + ) + mwcMetric := 0.0 + if mwc.Valid { + mwcMetric = float64(mwc.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterMaxwrittenCleanDesc, + prometheus.CounterValue, + mwcMetric, + ) + bbMetric := 0.0 + if bb.Valid { + bbMetric = float64(bb.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersBackendDesc, + prometheus.CounterValue, + bbMetric, + ) + bbfMetric := 0.0 + if bbf.Valid { + bbfMetric = float64(bbf.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersBackendFsyncDesc, + prometheus.CounterValue, + bbfMetric, + ) + baMetric := 0.0 + if ba.Valid { + baMetric = float64(ba.Int64) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterBuffersAllocDesc, + prometheus.CounterValue, + baMetric, + ) + srMetric := 0.0 + if sr.Valid { + srMetric = float64(sr.Time.Unix()) + } + ch <- prometheus.MustNewConstMetric( + statBGWriterStatsResetDesc, + prometheus.CounterValue, + srMetric, + ) } - ch <- prometheus.MustNewConstMetric( - statBGWriterStatsResetDesc, - prometheus.CounterValue, - srMetric, - ) return nil } diff --git a/collector/pg_stat_bgwriter_test.go b/collector/pg_stat_bgwriter_test.go index 1c2cf98de..6fde2fb6a 100644 --- a/collector/pg_stat_bgwriter_test.go +++ b/collector/pg_stat_bgwriter_test.go @@ -52,7 +52,7 @@ func TestPGStatBGWriterCollector(t *testing.T) { rows := sqlmock.NewRows(columns). AddRow(354, 4945, 289097744, 1242257, int64(3275602074), 89320867, 450139, 2034563757, 0, int64(2725688749), srT) - mock.ExpectQuery(sanitizeQuery(statBGWriterQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(statBGWriterQueryBefore17)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -113,7 +113,7 @@ func TestPGStatBGWriterCollectorNullValues(t *testing.T) { rows := sqlmock.NewRows(columns). AddRow(nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil) - mock.ExpectQuery(sanitizeQuery(statBGWriterQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(statBGWriterQueryBefore17)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { diff --git a/collector/pg_stat_checkpointer.go b/collector/pg_stat_checkpointer.go new file mode 100644 index 000000000..284cf650c --- /dev/null +++ b/collector/pg_stat_checkpointer.go @@ -0,0 +1,221 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "database/sql" + + "github.com/prometheus/client_golang/prometheus" +) + +const statCheckpointerSubsystem = "stat_checkpointer" + +func init() { + // WARNING: + // Disabled by default because this set of metrics is only available from Postgres 17 + registerCollector(statCheckpointerSubsystem, defaultDisabled, NewPGStatCheckpointerCollector) +} + +type PGStatCheckpointerCollector struct { +} + +func NewPGStatCheckpointerCollector(collectorConfig) (Collector, error) { + return &PGStatCheckpointerCollector{}, nil +} + +var ( + statCheckpointerNumTimedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "num_timed_total"), + "Number of scheduled checkpoints due to timeout", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerNumRequestedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "num_requested_total"), + "Number of requested checkpoints that have been performed", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerRestartpointsTimedDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "restartpoints_timed_total"), + "Number of scheduled restartpoints due to timeout or after a failed attempt to perform it", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerRestartpointsReqDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "restartpoints_req_total"), + "Number of requested restartpoints", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerRestartpointsDoneDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "restartpoints_done_total"), + "Number of restartpoints that have been performed", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerWriteTimeDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "write_time_total"), + "Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are written to disk, in milliseconds", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerSyncTimeDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "sync_time_total"), + "Total amount of time that has been spent in the portion of processing checkpoints and restartpoints where files are synchronized to disk, in milliseconds", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerBuffersWrittenDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "buffers_written_total"), + "Number of buffers written during checkpoints and restartpoints", + []string{}, + prometheus.Labels{}, + ) + statCheckpointerStatsResetDesc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statCheckpointerSubsystem, "stats_reset_total"), + "Time at which these statistics were last reset", + []string{}, + prometheus.Labels{}, + ) + + statCheckpointerQuery = `SELECT + num_timed + ,num_requested + ,restartpoints_timed + ,restartpoints_req + ,restartpoints_done + ,write_time + ,sync_time + ,buffers_written + ,stats_reset + FROM pg_stat_checkpointer;` +) + +func (PGStatCheckpointerCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + row := db.QueryRowContext(ctx, statCheckpointerQuery) + + // num_timed = nt = bigint + // num_requested = nr = bigint + // restartpoints_timed = rpt = bigint + // restartpoints_req = rpr = bigint + // restartpoints_done = rpd = bigint + // write_time = wt = double precision + // sync_time = st = double precision + // buffers_written = bw = bigint + // stats_reset = sr = timestamp + + var nt, nr, rpt, rpr, rpd, bw sql.NullInt64 + var wt, st sql.NullFloat64 + var sr sql.NullTime + + err := row.Scan(&nt, &nr, &rpt, &rpr, &rpd, &wt, &st, &bw, &sr) + if err != nil { + return err + } + + ntMetric := 0.0 + if nt.Valid { + ntMetric = float64(nt.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerNumTimedDesc, + prometheus.CounterValue, + ntMetric, + ) + + nrMetric := 0.0 + if nr.Valid { + nrMetric = float64(nr.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerNumRequestedDesc, + prometheus.CounterValue, + nrMetric, + ) + + rptMetric := 0.0 + if rpt.Valid { + rptMetric = float64(rpt.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerRestartpointsTimedDesc, + prometheus.CounterValue, + rptMetric, + ) + + rprMetric := 0.0 + if rpr.Valid { + rprMetric = float64(rpr.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerRestartpointsReqDesc, + prometheus.CounterValue, + rprMetric, + ) + + rpdMetric := 0.0 + if rpd.Valid { + rpdMetric = float64(rpd.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerRestartpointsDoneDesc, + prometheus.CounterValue, + rpdMetric, + ) + + wtMetric := 0.0 + if wt.Valid { + wtMetric = float64(wt.Float64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerWriteTimeDesc, + prometheus.CounterValue, + wtMetric, + ) + + stMetric := 0.0 + if st.Valid { + stMetric = float64(st.Float64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerSyncTimeDesc, + prometheus.CounterValue, + stMetric, + ) + + bwMetric := 0.0 + if bw.Valid { + bwMetric = float64(bw.Int64) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerBuffersWrittenDesc, + prometheus.CounterValue, + bwMetric, + ) + + srMetric := 0.0 + if sr.Valid { + srMetric = float64(sr.Time.Unix()) + } + ch <- prometheus.MustNewConstMetric( + statCheckpointerStatsResetDesc, + prometheus.CounterValue, + srMetric, + ) + + return nil +} diff --git a/collector/pg_stat_checkpointer_test.go b/collector/pg_stat_checkpointer_test.go new file mode 100644 index 000000000..2f59343bb --- /dev/null +++ b/collector/pg_stat_checkpointer_test.go @@ -0,0 +1,143 @@ +// Copyright 2024 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStatCheckpointerCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + + columns := []string{ + "num_timed", + "num_requested", + "restartpoints_timed", + "restartpoints_req", + "restartpoints_done", + "write_time", + "sync_time", + "buffers_written", + "stats_reset"} + + srT, err := time.Parse("2006-01-02 15:04:05.00000-07", "2023-05-25 17:10:42.81132-07") + if err != nil { + t.Fatalf("Error parsing time: %s", err) + } + + rows := sqlmock.NewRows(columns). + AddRow(354, 4945, 289097744, 1242257, int64(3275602074), 89320867, 450139, 2034563757, srT) + mock.ExpectQuery(sanitizeQuery(statCheckpointerQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatCheckpointerCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatCheckpointerCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 354}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 4945}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 289097744}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 1242257}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 3275602074}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 89320867}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 450139}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 2034563757}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 1685059842}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + +func TestPGStatCheckpointerCollectorNullValues(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + + columns := []string{ + "num_timed", + "num_requested", + "restartpoints_timed", + "restartpoints_req", + "restartpoints_done", + "write_time", + "sync_time", + "buffers_written", + "stats_reset"} + + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil, nil, nil, nil, nil, nil, nil) + mock.ExpectQuery(sanitizeQuery(statCheckpointerQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatCheckpointerCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatCheckpointerCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{}, metricType: dto.MetricType_COUNTER, value: 0}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From ecb5ec5dff0cf2b3eb3668a7fee30d8ad11c82fa Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Tue, 7 Jan 2025 09:28:42 +0100 Subject: [PATCH 03/33] Update common Prometheus files (#1090) Signed-off-by: prombot --- .yamllint | 2 +- Makefile.common | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.yamllint b/.yamllint index 1859cb624..8d09c375f 100644 --- a/.yamllint +++ b/.yamllint @@ -1,7 +1,7 @@ --- extends: default ignore: | - ui/react-app/node_modules + **/node_modules rules: braces: diff --git a/Makefile.common b/Makefile.common index cbb5d8638..09e5bff85 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.60.2 +GOLANGCI_LINT_VERSION ?= v1.61.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) From 9de4f19d43427c24947ba7db7d82127141bd5fa4 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Tue, 7 Jan 2025 21:20:19 +0100 Subject: [PATCH 04/33] Update common Prometheus files (#1110) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 4 ++-- Makefile.common | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 305146993..01b943b9b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0 + uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 with: go-version: 1.23.x - name: Install snmp_exporter/generator dependencies @@ -36,4 +36,4 @@ jobs: uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 with: args: --verbose - version: v1.61.0 + version: v1.62.0 diff --git a/Makefile.common b/Makefile.common index 09e5bff85..fc47bdbb2 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.61.0 +GOLANGCI_LINT_VERSION ?= v1.62.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) From 7d4c278221e95ddbc62108973e5828a3ffaa2eb8 Mon Sep 17 00:00:00 2001 From: Khiem Doan Date: Mon, 13 Jan 2025 09:24:15 +0700 Subject: [PATCH 05/33] Add Postgres 17 for CI test (#1105) Signed-off-by: Khiem Doan --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4c464e210..9494acb24 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Prometheus exporter for PostgreSQL server metrics. -CI Tested PostgreSQL versions: `11`, `12`, `13`, `14`, `15`, `16` +CI Tested PostgreSQL versions: `11`, `12`, `13`, `14`, `15`, `16`, `17`. ## Quick Start This package is available for Docker: From 3acc4793fc63ffb3088668f285ba658429c8806c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Feb 2025 03:35:38 +0100 Subject: [PATCH 06/33] Bump github.com/prometheus/common from 0.61.0 to 0.62.0 (#1118) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.61.0 to 0.62.0. - [Release notes](https://github.com/prometheus/common/releases) - [Changelog](https://github.com/prometheus/common/blob/main/RELEASE.md) - [Commits](https://github.com/prometheus/common/compare/v0.61.0...v0.62.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9cdf3cbec..b2c9500d0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.61.0 + github.com/prometheus/common v0.62.0 github.com/prometheus/exporter-toolkit v0.13.2 github.com/smartystreets/goconvey v1.8.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -37,10 +37,10 @@ require ( github.com/smarty/assertions v1.15.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/crypto v0.31.0 // indirect - golang.org/x/net v0.32.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect - google.golang.org/protobuf v1.35.2 // indirect + google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index e20b36f15..a6dd72407 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+ github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.61.0 h1:3gv/GThfX0cV2lpO7gkTUwZru38mxevy90Bj8YFSRQQ= -github.com/prometheus/common v0.61.0/go.mod h1:zr29OCN/2BsJRaFwG8QOBr41D6kkchKbpeNH7pAjb/s= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/exporter-toolkit v0.13.2 h1:Z02fYtbqTMy2i/f+xZ+UK5jy/bl1Ex3ndzh06T/Q9DQ= github.com/prometheus/exporter-toolkit v0.13.2/go.mod h1:tCqnfx21q6qN1KA4U3Bfb8uWzXfijIrJz3/kTIqMV7g= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= @@ -75,8 +75,8 @@ github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8 github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= -golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= @@ -85,8 +85,8 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= +google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From d85a7710bf084309413986c577ee03d09fb8d2ef Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Fri, 14 Feb 2025 09:36:03 +0100 Subject: [PATCH 07/33] Update common Prometheus files (#1124) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 6 +++--- Makefile.common | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 01b943b9b..def9007ac 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -26,14 +26,14 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0 + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: go-version: 1.23.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1 + uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 with: args: --verbose - version: v1.62.0 + version: v1.63.4 diff --git a/Makefile.common b/Makefile.common index fc47bdbb2..d1576bb31 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.62.0 +GOLANGCI_LINT_VERSION ?= v1.63.4 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) From 072864d179de04327840bc32fa0b1765f250b1f3 Mon Sep 17 00:00:00 2001 From: Nevermind <79126473+NevermindZ4@users.noreply.github.com> Date: Sat, 15 Feb 2025 14:54:12 +0100 Subject: [PATCH 08/33] pg_stat_statements PG17 (#1114) Signed-off-by: Nevermind <79126473+NevermindZ4@users.noreply.github.com> --- collector/pg_stat_statements.go | 30 +++++++++++++++++-- collector/pg_stat_statements_test.go | 43 ++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/collector/pg_stat_statements.go b/collector/pg_stat_statements.go index 7926f533e..cabdc3619 100644 --- a/collector/pg_stat_statements.go +++ b/collector/pg_stat_statements.go @@ -112,12 +112,38 @@ var ( ) ORDER BY seconds_total DESC LIMIT 100;` + + pgStatStatementsQuery_PG17 = `SELECT + pg_get_userbyid(userid) as user, + pg_database.datname, + pg_stat_statements.queryid, + pg_stat_statements.calls as calls_total, + pg_stat_statements.total_exec_time / 1000.0 as seconds_total, + pg_stat_statements.rows as rows_total, + pg_stat_statements.shared_blk_read_time / 1000.0 as block_read_seconds_total, + pg_stat_statements.shared_blk_write_time / 1000.0 as block_write_seconds_total + FROM pg_stat_statements + JOIN pg_database + ON pg_database.oid = pg_stat_statements.dbid + WHERE + total_exec_time > ( + SELECT percentile_cont(0.1) + WITHIN GROUP (ORDER BY total_exec_time) + FROM pg_stat_statements + ) + ORDER BY seconds_total DESC + LIMIT 100;` ) func (PGStatStatementsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { - query := pgStatStatementsQuery - if instance.version.GE(semver.MustParse("13.0.0")) { + var query string + switch { + case instance.version.GE(semver.MustParse("17.0.0")): + query = pgStatStatementsQuery_PG17 + case instance.version.GE(semver.MustParse("13.0.0")): query = pgStatStatementsNewQuery + default: + query = pgStatStatementsQuery } db := instance.getDB() diff --git a/collector/pg_stat_statements_test.go b/collector/pg_stat_statements_test.go index 08aba34c2..ea035f557 100644 --- a/collector/pg_stat_statements_test.go +++ b/collector/pg_stat_statements_test.go @@ -151,3 +151,46 @@ func TestPGStateStatementsCollectorNewPG(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPGStateStatementsCollector_PG17(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("17.0.0")} + + columns := []string{"user", "datname", "queryid", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} + rows := sqlmock.NewRows(columns). + AddRow("postgres", "postgres", 1500, 5, 0.4, 100, 0.1, 0.2) + mock.ExpectQuery(sanitizeQuery(pgStatStatementsQuery_PG17)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatStatementsCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 5}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.4}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 100}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.1}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.2}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From 9e42fc0145a9d780d824d961afb7b70867558424 Mon Sep 17 00:00:00 2001 From: Michael Todorovic Date: Sat, 15 Feb 2025 15:00:48 +0100 Subject: [PATCH 09/33] fix: handle pg_replication_slots on pg<13 (#1098) * fix: handle pg_replication_slots on pg<13 Signed-off-by: Michael Todorovic * fix: tests Signed-off-by: Michael Todorovic --------- Signed-off-by: Michael Todorovic --- collector/pg_replication_slot.go | 38 ++++++++++++++++++++++++--- collector/pg_replication_slot_test.go | 17 ++++++------ 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/collector/pg_replication_slot.go b/collector/pg_replication_slot.go index 27ccddefd..e6c9773eb 100644 --- a/collector/pg_replication_slot.go +++ b/collector/pg_replication_slot.go @@ -18,6 +18,7 @@ import ( "database/sql" "log/slog" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" ) @@ -81,8 +82,18 @@ var ( "availability of WAL files claimed by this slot", []string{"slot_name", "slot_type", "wal_status"}, nil, ) - pgReplicationSlotQuery = `SELECT + slot_name, + slot_type, + CASE WHEN pg_is_in_recovery() THEN + pg_last_wal_receive_lsn() - '0/0' + ELSE + pg_current_wal_lsn() - '0/0' + END AS current_wal_lsn, + COALESCE(confirmed_flush_lsn, '0/0') - '0/0' AS confirmed_flush_lsn, + active + FROM pg_replication_slots;` + pgReplicationSlotNewQuery = `SELECT slot_name, slot_type, CASE WHEN pg_is_in_recovery() THEN @@ -98,9 +109,15 @@ var ( ) func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + query := pgReplicationSlotQuery + abovePG13 := instance.version.GTE(semver.MustParse("13.0.0")) + if abovePG13 { + query = pgReplicationSlotNewQuery + } + db := instance.getDB() rows, err := db.QueryContext(ctx, - pgReplicationSlotQuery) + query) if err != nil { return err } @@ -114,7 +131,22 @@ func (PGReplicationSlotCollector) Update(ctx context.Context, instance *instance var isActive sql.NullBool var safeWalSize sql.NullInt64 var walStatus sql.NullString - if err := rows.Scan(&slotName, &slotType, &walLSN, &flushLSN, &isActive, &safeWalSize, &walStatus); err != nil { + + r := []any{ + &slotName, + &slotType, + &walLSN, + &flushLSN, + &isActive, + } + + if abovePG13 { + r = append(r, &safeWalSize) + r = append(r, &walStatus) + } + + err := rows.Scan(r...) + if err != nil { return err } diff --git a/collector/pg_replication_slot_test.go b/collector/pg_replication_slot_test.go index 174743ac3..981b5db62 100644 --- a/collector/pg_replication_slot_test.go +++ b/collector/pg_replication_slot_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/smartystreets/goconvey/convey" @@ -29,12 +30,12 @@ func TestPgReplicationSlotCollectorActive(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("13.3.7")} columns := []string{"slot_name", "slot_type", "current_wal_lsn", "confirmed_flush_lsn", "active", "safe_wal_size", "wal_status"} rows := sqlmock.NewRows(columns). AddRow("test_slot", "physical", 5, 3, true, 323906992, "reserved") - mock.ExpectQuery(sanitizeQuery(pgReplicationSlotQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(pgReplicationSlotNewQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -72,12 +73,12 @@ func TestPgReplicationSlotCollectorInActive(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("13.3.7")} columns := []string{"slot_name", "slot_type", "current_wal_lsn", "confirmed_flush_lsn", "active", "safe_wal_size", "wal_status"} rows := sqlmock.NewRows(columns). AddRow("test_slot", "physical", 6, 12, false, -4000, "extended") - mock.ExpectQuery(sanitizeQuery(pgReplicationSlotQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(pgReplicationSlotNewQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -115,12 +116,12 @@ func TestPgReplicationSlotCollectorActiveNil(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("13.3.7")} columns := []string{"slot_name", "slot_type", "current_wal_lsn", "confirmed_flush_lsn", "active", "safe_wal_size", "wal_status"} rows := sqlmock.NewRows(columns). AddRow("test_slot", "physical", 6, 12, nil, nil, "lost") - mock.ExpectQuery(sanitizeQuery(pgReplicationSlotQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(pgReplicationSlotNewQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -156,12 +157,12 @@ func TestPgReplicationSlotCollectorTestNilValues(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("13.3.7")} columns := []string{"slot_name", "slot_type", "current_wal_lsn", "confirmed_flush_lsn", "active", "safe_wal_size", "wal_status"} rows := sqlmock.NewRows(columns). AddRow(nil, nil, nil, nil, true, nil, nil) - mock.ExpectQuery(sanitizeQuery(pgReplicationSlotQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(pgReplicationSlotNewQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { From 2ee2a8fa7cbd7b07c58a5e40998d6e1f469dd1af Mon Sep 17 00:00:00 2001 From: Felipe Galindo Sanchez Date: Sat, 15 Feb 2025 06:08:24 -0800 Subject: [PATCH 10/33] feat: add wait/backend to pg_stat_activity (#1106) Signed-off-by: Felipe Galindo Sanchez --- cmd/postgres_exporter/postgres_exporter.go | 3 +++ cmd/postgres_exporter/queries.go | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 90f26beb0..d76e2baed 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -251,6 +251,9 @@ var builtinMetricMaps = map[string]intermediateMetricMap{ "state": {LABEL, "connection state", nil, semver.MustParseRange(">=9.2.0")}, "usename": {LABEL, "connection usename", nil, nil}, "application_name": {LABEL, "connection application_name", nil, nil}, + "backend_type": {LABEL, "connection backend_type", nil, nil}, + "wait_event_type": {LABEL, "connection wait_event_type", nil, nil}, + "wait_event": {LABEL, "connection wait_event", nil, nil}, "count": {GAUGE, "number of connections in this state", nil, nil}, "max_tx_duration": {GAUGE, "max duration in seconds any active transaction has been running", nil, nil}, }, diff --git a/cmd/postgres_exporter/queries.go b/cmd/postgres_exporter/queries.go index 7090606e1..80be72d54 100644 --- a/cmd/postgres_exporter/queries.go +++ b/cmd/postgres_exporter/queries.go @@ -115,6 +115,9 @@ var queryOverrides = map[string][]OverrideQuery{ tmp.state, tmp2.usename, tmp2.application_name, + tmp2.backend_type, + tmp2.wait_event_type, + tmp2.wait_event, COALESCE(count,0) as count, COALESCE(max_tx_duration,0) as max_tx_duration FROM @@ -133,9 +136,13 @@ var queryOverrides = map[string][]OverrideQuery{ state, usename, application_name, + backend_type, + wait_event_type, + wait_event, count(*) AS count, MAX(EXTRACT(EPOCH FROM now() - xact_start))::float AS max_tx_duration - FROM pg_stat_activity GROUP BY datname,state,usename,application_name) AS tmp2 + FROM pg_stat_activity + GROUP BY datname,state,usename,application_name,backend_type,wait_event_type,wait_event) AS tmp2 ON tmp.state = tmp2.state AND pg_database.datname = tmp2.datname `, }, From c3885e840a1dd8eea2a878c044a92088c1820f0e Mon Sep 17 00:00:00 2001 From: Conrad Hoffmann <1226676+bitfehler@users.noreply.github.com> Date: Sat, 15 Feb 2025 15:15:44 +0100 Subject: [PATCH 11/33] Export last replay age in replication collector (#1085) The exported replication lag does not handle all failure modes, and can report 0 for replicas that are out of sync and incapable of recovery. A proper replacement for that metric would require a different approach (see e.g. #1007), but for a lot of folks, simply exporting the age of the last replay can provide a pretty strong signal for something being amiss. I think this solution might be preferable to #977, though the lag metric needs to be fixed or abandoned eventually. Signed-off-by: Conrad Hoffmann --- collector/pg_replication.go | 19 +++++++++++++++++-- collector/pg_replication_test.go | 5 +++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/collector/pg_replication.go b/collector/pg_replication.go index 6067cc9b1..7f8b2fbd7 100644 --- a/collector/pg_replication.go +++ b/collector/pg_replication.go @@ -51,6 +51,15 @@ var ( "Indicates if the server is a replica", []string{}, nil, ) + pgReplicationLastReplay = prometheus.NewDesc( + prometheus.BuildFQName( + namespace, + replicationSubsystem, + "last_replay_seconds", + ), + "Age of last replay in seconds", + []string{}, nil, + ) pgReplicationQuery = `SELECT CASE @@ -61,7 +70,8 @@ var ( CASE WHEN pg_is_in_recovery() THEN 1 ELSE 0 - END as is_replica` + END as is_replica, + GREATEST (0, EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))) as last_replay` ) func (c *PGReplicationCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { @@ -72,7 +82,8 @@ func (c *PGReplicationCollector) Update(ctx context.Context, instance *instance, var lag float64 var isReplica int64 - err := row.Scan(&lag, &isReplica) + var replayAge float64 + err := row.Scan(&lag, &isReplica, &replayAge) if err != nil { return err } @@ -84,5 +95,9 @@ func (c *PGReplicationCollector) Update(ctx context.Context, instance *instance, pgReplicationIsReplica, prometheus.GaugeValue, float64(isReplica), ) + ch <- prometheus.MustNewConstMetric( + pgReplicationLastReplay, + prometheus.GaugeValue, replayAge, + ) return nil } diff --git a/collector/pg_replication_test.go b/collector/pg_replication_test.go index b6df698e3..a48e9fd69 100644 --- a/collector/pg_replication_test.go +++ b/collector/pg_replication_test.go @@ -31,9 +31,9 @@ func TestPgReplicationCollector(t *testing.T) { inst := &instance{db: db} - columns := []string{"lag", "is_replica"} + columns := []string{"lag", "is_replica", "last_replay"} rows := sqlmock.NewRows(columns). - AddRow(1000, 1) + AddRow(1000, 1, 3) mock.ExpectQuery(sanitizeQuery(pgReplicationQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) @@ -49,6 +49,7 @@ func TestPgReplicationCollector(t *testing.T) { expected := []MetricResult{ {labels: labelMap{}, value: 1000, metricType: dto.MetricType_GAUGE}, {labels: labelMap{}, value: 1, metricType: dto.MetricType_GAUGE}, + {labels: labelMap{}, value: 3, metricType: dto.MetricType_GAUGE}, } convey.Convey("Metrics comparison", t, func() { From 99e1b5118caf5094ed69bd7e11cd601e0431bfb5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Feb 2025 10:01:47 -0500 Subject: [PATCH 12/33] Bump github.com/prometheus/exporter-toolkit from 0.13.2 to 0.14.0 (#1126) Bumps [github.com/prometheus/exporter-toolkit](https://github.com/prometheus/exporter-toolkit) from 0.13.2 to 0.14.0. - [Release notes](https://github.com/prometheus/exporter-toolkit/releases) - [Changelog](https://github.com/prometheus/exporter-toolkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/prometheus/exporter-toolkit/compare/v0.13.2...v0.14.0) --- updated-dependencies: - dependency-name: github.com/prometheus/exporter-toolkit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index b2c9500d0..a7b376b4e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.62.0 - github.com/prometheus/exporter-toolkit v0.13.2 + github.com/prometheus/exporter-toolkit v0.14.0 github.com/smartystreets/goconvey v1.8.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 @@ -36,11 +36,11 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index a6dd72407..a57fc83c1 100644 --- a/go.sum +++ b/go.sum @@ -56,8 +56,8 @@ github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/exporter-toolkit v0.13.2 h1:Z02fYtbqTMy2i/f+xZ+UK5jy/bl1Ex3ndzh06T/Q9DQ= -github.com/prometheus/exporter-toolkit v0.13.2/go.mod h1:tCqnfx21q6qN1KA4U3Bfb8uWzXfijIrJz3/kTIqMV7g= +github.com/prometheus/exporter-toolkit v0.14.0 h1:NMlswfibpcZZ+H0sZBiTjrA3/aBFHkNZqE+iCj5EmRg= +github.com/prometheus/exporter-toolkit v0.14.0/go.mod h1:Gu5LnVvt7Nr/oqTBUC23WILZepW0nffNo10XdhQcwWA= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= @@ -73,16 +73,16 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= From 4c170ed56460028f5c02d16a5576250312eecf89 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sat, 15 Feb 2025 10:35:04 -0500 Subject: [PATCH 13/33] Fix missing dsn sanitization for logging (#1104) This log line was not sanitized previously which could result in logging sensitive information. I have scanned the rest of the files and I don't see anywhere else that DSN is used in a log line without this filter. Resolves #1042 Signed-off-by: Joe Adams --- cmd/postgres_exporter/postgres_exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index d76e2baed..a76479611 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -681,7 +681,7 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) { if err := e.scrapeDSN(ch, dsn); err != nil { errorsCount++ - logger.Error("error scraping dsn", "err", err, "dsn", dsn) + logger.Error("error scraping dsn", "err", err, "dsn", loggableDSN(dsn)) if _, ok := err.(*ErrorConnectToServer); ok { connectionErrorsCount++ From 8bb1a41abf0cfc6baf1bf272a7cf8989d1958dde Mon Sep 17 00:00:00 2001 From: Nicolas Rodriguez Date: Thu, 20 Feb 2025 03:49:11 +0100 Subject: [PATCH 14/33] Skip pg_stat_checkpointer collector if pg<17 (#1112) * fix: skip collector if pg<17 Signed-off-by: Michael Todorovic * fix: better condition Signed-off-by: Michael Todorovic * fix: fix PGStatCheckpointerCollector tests Signed-off-by: Nicolas Rodriguez --------- Signed-off-by: Michael Todorovic Signed-off-by: Nicolas Rodriguez Co-authored-by: Michael Todorovic --- collector/pg_stat_checkpointer.go | 16 +++++++++++++--- collector/pg_stat_checkpointer_test.go | 5 +++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/collector/pg_stat_checkpointer.go b/collector/pg_stat_checkpointer.go index 284cf650c..31e9c5d62 100644 --- a/collector/pg_stat_checkpointer.go +++ b/collector/pg_stat_checkpointer.go @@ -16,7 +16,9 @@ package collector import ( "context" "database/sql" + "log/slog" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" ) @@ -29,10 +31,11 @@ func init() { } type PGStatCheckpointerCollector struct { + log *slog.Logger } -func NewPGStatCheckpointerCollector(collectorConfig) (Collector, error) { - return &PGStatCheckpointerCollector{}, nil +func NewPGStatCheckpointerCollector(config collectorConfig) (Collector, error) { + return &PGStatCheckpointerCollector{log: config.logger}, nil } var ( @@ -104,8 +107,15 @@ var ( FROM pg_stat_checkpointer;` ) -func (PGStatCheckpointerCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { +func (c PGStatCheckpointerCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { db := instance.getDB() + + before17 := instance.version.LT(semver.MustParse("17.0.0")) + if before17 { + c.log.Warn("pg_stat_checkpointer collector is not available on PostgreSQL < 17.0.0, skipping") + return nil + } + row := db.QueryRowContext(ctx, statCheckpointerQuery) // num_timed = nt = bigint diff --git a/collector/pg_stat_checkpointer_test.go b/collector/pg_stat_checkpointer_test.go index 2f59343bb..9a8dd7f21 100644 --- a/collector/pg_stat_checkpointer_test.go +++ b/collector/pg_stat_checkpointer_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/DATA-DOG/go-sqlmock" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" "github.com/smartystreets/goconvey/convey" @@ -30,7 +31,7 @@ func TestPGStatCheckpointerCollector(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("17.0.0")} columns := []string{ "num_timed", @@ -92,7 +93,7 @@ func TestPGStatCheckpointerCollectorNullValues(t *testing.T) { } defer db.Close() - inst := &instance{db: db} + inst := &instance{db: db, version: semver.MustParse("17.0.0")} columns := []string{ "num_timed", From 51006aba2f695d318d52b6f5842f18bb52e54b9f Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Fri, 21 Feb 2025 17:38:46 -0500 Subject: [PATCH 15/33] Prep for v0.17 (#1127) Signed-off-by: Joe Adams --- CHANGELOG.md | 22 ++++++++++++++++++++++ VERSION | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 790a109c5..49b6b4101 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,25 @@ +## 0.17.0 / 2025-02-16 + +## What's Changed +* [ENHANCEMENT] Add Postgres 17 for CI test by @khiemdoan in https://github.com/prometheus-community/postgres_exporter/pull/1105 +* [ENHANCEMENT] Add wait/backend to pg_stat_activity by @fgalind1 in https://github.com/prometheus-community/postgres_exporter/pull/1106 +* [ENHANCEMENT] Export last replay age in replication collector by @bitfehler in https://github.com/prometheus-community/postgres_exporter/pull/1085 +* [BUGFIX] Fix pg_long_running_transactions time by @jyothikirant-sayukth in https://github.com/prometheus-community/postgres_exporter/pull/1092 +* [BUGFIX] Fix to replace dashes with underscore in the metric names by @aagarwalla-fx in https://github.com/prometheus-community/postgres_exporter/pull/1103 +* [BIGFIX] Checkpoint related columns in PG 17 have been moved from pg_stat_bgwriter to pg_stat_checkpointer by @n-rodriguez in https://github.com/prometheus-community/postgres_exporter/pull/1072 +* [BUGFIX] Fix pg_stat_statements for PG17 by @NevermindZ4 in https://github.com/prometheus-community/postgres_exporter/pull/1114 +* [BUGFIX] Handle pg_replication_slots on pg<13 by @michael-todorovic in https://github.com/prometheus-community/postgres_exporter/pull/1098 +* [BUGFIX] Fix missing dsn sanitization for logging by @sysadmind in https://github.com/prometheus-community/postgres_exporter/pull/1104 + +## New Contributors +* @jyothikirant-sayukth made their first contribution in https://github.com/prometheus-community/postgres_exporter/pull/1092 +* @aagarwalla-fx made their first contribution in https://github.com/prometheus-community/postgres_exporter/pull/1103 +* @NevermindZ4 made their first contribution in https://github.com/prometheus-community/postgres_exporter/pull/1114 +* @michael-todorovic made their first contribution in https://github.com/prometheus-community/postgres_exporter/pull/1098 +* @fgalind1 made their first contribution in https://github.com/prometheus-community/postgres_exporter/pull/1106 + +**Full Changelog**: https://github.com/prometheus-community/postgres_exporter/compare/v0.16.0...v0.17.0 + ## 0.16.0 / 2024-11-10 BREAKING CHANGES: diff --git a/VERSION b/VERSION index 04a373efe..c5523bd09 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.16.0 +0.17.0 From 2869087f3cec1b2096f33ed9d182b0452f9b8d11 Mon Sep 17 00:00:00 2001 From: vancwo Date: Wed, 26 Feb 2025 05:21:39 -0800 Subject: [PATCH 16/33] Fix: Handle incoming labels with invalid UTF-8 (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's possible that incoming labels will contain invalid UTF-8 characters. This results in a panic. This fix sanitizes the label's string to ensure only valid UTF-8 characters are included, by replacing invalid characters with � (REPLACEMENT CHARACTER) Signed-off-by: Cooper Worobetz --- cmd/postgres_exporter/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/util.go b/cmd/postgres_exporter/util.go index 3baa6f4be..8907e7c5f 100644 --- a/cmd/postgres_exporter/util.go +++ b/cmd/postgres_exporter/util.go @@ -159,7 +159,7 @@ func dbToString(t interface{}) (string, bool) { // Try and convert to string return string(v), true case string: - return v, true + return strings.ToValidUTF8(v, "�"), true case bool: if v { return "true", true From 1e574cf4fd2a75a8a707d424eafcaa0b88cb7af4 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Wed, 26 Feb 2025 14:41:27 +0100 Subject: [PATCH 17/33] Release v0.17.1 (#1132) * [BUGFIX] Fix: Handle incoming labels with invalid UTF-8 #1131 Signed-off-by: SuperQ --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49b6b4101..d01ceb504 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.17.1 / 2025-02-26 + +* [BUGFIX] Fix: Handle incoming labels with invalid UTF-8 #1131 + ## 0.17.0 / 2025-02-16 ## What's Changed diff --git a/VERSION b/VERSION index c5523bd09..7cca7711a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.17.0 +0.17.1 From 457b6fa8cd89b34417a95911d7985d93a1b5c32d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 18:26:29 +0100 Subject: [PATCH 18/33] Bump github.com/prometheus/client_golang from 1.20.5 to 1.21.0 (#1133) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.20.5 to 1.21.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.20.5...v1.21.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index a7b376b4e..422c3241b 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/alecthomas/kingpin/v2 v2.4.0 github.com/blang/semver/v4 v4.0.0 github.com/lib/pq v1.10.9 - github.com/prometheus/client_golang v1.20.5 + github.com/prometheus/client_golang v1.21.0 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.62.0 github.com/prometheus/exporter-toolkit v0.14.0 @@ -25,7 +25,7 @@ require ( github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mdlayher/socket v0.4.1 // indirect diff --git a/go.sum b/go.sum index a57fc83c1..7d4217fa1 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -50,8 +50,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= -github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= +github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= From 602302ffe22485c4e15f3101a705e588a849d1d5 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Sat, 8 Mar 2025 15:00:54 +0100 Subject: [PATCH 19/33] Update common Prometheus files (#1137) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 6 +++--- Makefile.common | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index def9007ac..e36a9f1a4 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -28,12 +28,12 @@ jobs: - name: Install Go uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 with: - go-version: 1.23.x + go-version: 1.24.x - name: Install snmp_exporter/generator dependencies run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@ec5d18412c0aeab7936cb16880d708ba2a64e1ae # v6.2.0 + uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 with: args: --verbose - version: v1.63.4 + version: v1.64.6 diff --git a/Makefile.common b/Makefile.common index d1576bb31..8cb383859 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.63.4 +GOLANGCI_LINT_VERSION ?= v1.64.6 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) From b0e61bf263fffeefc5d5b84fea8b8b0714958862 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Mar 2025 22:34:30 +0100 Subject: [PATCH 20/33] Bump golang.org/x/net from 0.33.0 to 0.36.0 (#1138) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.33.0 to 0.36.0. - [Commits](https://github.com/golang/net/compare/v0.33.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 422c3241b..1d585cd23 100644 --- a/go.mod +++ b/go.mod @@ -36,11 +36,11 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect - golang.org/x/crypto v0.32.0 // indirect - golang.org/x/net v0.33.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect - golang.org/x/sync v0.10.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sync v0.11.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect google.golang.org/protobuf v1.36.1 // indirect ) diff --git a/go.sum b/go.sum index 7d4217fa1..09262cf79 100644 --- a/go.sum +++ b/go.sum @@ -73,18 +73,18 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= -golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= +golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 2ce65c324c58dceedd832583f196987f2c13b011 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Sun, 23 Mar 2025 16:54:21 +0100 Subject: [PATCH 21/33] Update common Prometheus files (#1140) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index e36a9f1a4..b404ce7c7 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -33,7 +33,7 @@ jobs: run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@2226d7cb06a077cd73e56eedd38eecad18e5d837 # v6.5.0 + uses: golangci/golangci-lint-action@55c2c1448f86e01eaae002a5a3a9624417608d84 # v6.5.2 with: args: --verbose version: v1.64.6 From fca2ad84cd79fea6b414a2ce21fe8ee5b8d5a5a4 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Sun, 30 Mar 2025 20:49:06 +0200 Subject: [PATCH 22/33] Update common Prometheus files (#1142) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index b404ce7c7..5342cbe08 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -26,7 +26,7 @@ jobs: - name: Checkout repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - name: Install Go - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5.3.0 + uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0 with: go-version: 1.24.x - name: Install snmp_exporter/generator dependencies From 9e86f1ee380a4c3e4607991cd79e54713c1f3197 Mon Sep 17 00:00:00 2001 From: Ian Bibby <470816+ianbibby@users.noreply.github.com> Date: Thu, 3 Apr 2025 07:45:29 -0700 Subject: [PATCH 23/33] Adds pg_stat_progress_vacuum collector (#1141) Signed-off-by: Ian Bibby Co-authored-by: Ben Kochie --- README.md | 3 + collector/pg_stat_progress_vacuum.go | 222 ++++++++++++++++++++++ collector/pg_stat_progress_vacuum_test.go | 135 +++++++++++++ 3 files changed, 360 insertions(+) create mode 100644 collector/pg_stat_progress_vacuum.go create mode 100644 collector/pg_stat_progress_vacuum_test.go diff --git a/README.md b/README.md index 9494acb24..dedc5aa55 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,9 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra * `[no-]collector.stat_database` Enable the `stat_database` collector (default: enabled). +* `[no-]collector.stat_progress_vacuum` + Enable the `stat_progress_vacuum` collector (default: enabled). + * `[no-]collector.stat_statements` Enable the `stat_statements` collector (default: disabled). diff --git a/collector/pg_stat_progress_vacuum.go b/collector/pg_stat_progress_vacuum.go new file mode 100644 index 000000000..f8083a49f --- /dev/null +++ b/collector/pg_stat_progress_vacuum.go @@ -0,0 +1,222 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "database/sql" + "log/slog" + + "github.com/prometheus/client_golang/prometheus" +) + +const progressVacuumSubsystem = "stat_progress_vacuum" + +func init() { + registerCollector(progressVacuumSubsystem, defaultEnabled, NewPGStatProgressVacuumCollector) +} + +type PGStatProgressVacuumCollector struct { + log *slog.Logger +} + +func NewPGStatProgressVacuumCollector(config collectorConfig) (Collector, error) { + return &PGStatProgressVacuumCollector{log: config.logger}, nil +} + +var vacuumPhases = []string{ + "initializing", + "scanning heap", + "vacuuming indexes", + "vacuuming heap", + "cleaning up indexes", + "truncating heap", + "performing final cleanup", +} + +var ( + statProgressVacuumPhase = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "phase"), + "Current vacuum phase (1 = active, 0 = inactive). Label 'phase' is human-readable.", + []string{"datname", "relname", "phase"}, + nil, + ) + + statProgressVacuumHeapBlksTotal = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "heap_blks"), + "Total number of heap blocks in the table being vacuumed.", + []string{"datname", "relname"}, + nil, + ) + + statProgressVacuumHeapBlksScanned = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "heap_blks_scanned"), + "Number of heap blocks scanned so far.", + []string{"datname", "relname"}, + nil, + ) + + statProgressVacuumHeapBlksVacuumed = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "heap_blks_vacuumed"), + "Number of heap blocks vacuumed so far.", + []string{"datname", "relname"}, + nil, + ) + + statProgressVacuumIndexVacuumCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "index_vacuums"), + "Number of completed index vacuum cycles.", + []string{"datname", "relname"}, + nil, + ) + + statProgressVacuumMaxDeadTuples = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "max_dead_tuples"), + "Maximum number of dead tuples that can be stored before cleanup is performed.", + []string{"datname", "relname"}, + nil, + ) + + statProgressVacuumNumDeadTuples = prometheus.NewDesc( + prometheus.BuildFQName(namespace, progressVacuumSubsystem, "num_dead_tuples"), + "Current number of dead tuples found so far.", + []string{"datname", "relname"}, + nil, + ) + + // This is the view definition of pg_stat_progress_vacuum, albeit without the conversion + // of "phase" to a human-readable string. We will prefer the numeric representation. + statProgressVacuumQuery = `SELECT + d.datname, + s.relid::regclass::text AS relname, + s.param1 AS phase, + s.param2 AS heap_blks_total, + s.param3 AS heap_blks_scanned, + s.param4 AS heap_blks_vacuumed, + s.param5 AS index_vacuum_count, + s.param6 AS max_dead_tuples, + s.param7 AS num_dead_tuples + FROM + pg_stat_get_progress_info('VACUUM'::text) + s(pid, datid, relid, param1, param2, param3, param4, param5, param6, param7, param8, param9, param10, param11, param12, param13, param14, param15, param16, param17, param18, param19, param20) + LEFT JOIN + pg_database d ON s.datid = d.oid` +) + +func (c *PGStatProgressVacuumCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + db := instance.getDB() + rows, err := db.QueryContext(ctx, + statProgressVacuumQuery) + + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + var ( + datname sql.NullString + relname sql.NullString + phase sql.NullInt64 + heapBlksTotal sql.NullInt64 + heapBlksScanned sql.NullInt64 + heapBlksVacuumed sql.NullInt64 + indexVacuumCount sql.NullInt64 + maxDeadTuples sql.NullInt64 + numDeadTuples sql.NullInt64 + ) + + if err := rows.Scan( + &datname, + &relname, + &phase, + &heapBlksTotal, + &heapBlksScanned, + &heapBlksVacuumed, + &indexVacuumCount, + &maxDeadTuples, + &numDeadTuples, + ); err != nil { + return err + } + + datnameLabel := "unknown" + if datname.Valid { + datnameLabel = datname.String + } + relnameLabel := "unknown" + if relname.Valid { + relnameLabel = relname.String + } + + labels := []string{datnameLabel, relnameLabel} + + var phaseMetric *float64 + if phase.Valid { + v := float64(phase.Int64) + phaseMetric = &v + } + + for i, label := range vacuumPhases { + v := 0.0 + // Only the current phase should be 1.0. + if phaseMetric != nil && float64(i) == *phaseMetric { + v = 1.0 + } + labelsCopy := append(labels, label) + ch <- prometheus.MustNewConstMetric(statProgressVacuumPhase, prometheus.GaugeValue, v, labelsCopy...) + } + + heapTotal := 0.0 + if heapBlksTotal.Valid { + heapTotal = float64(heapBlksTotal.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumHeapBlksTotal, prometheus.GaugeValue, heapTotal, labels...) + + heapScanned := 0.0 + if heapBlksScanned.Valid { + heapScanned = float64(heapBlksScanned.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumHeapBlksScanned, prometheus.GaugeValue, heapScanned, labels...) + + heapVacuumed := 0.0 + if heapBlksVacuumed.Valid { + heapVacuumed = float64(heapBlksVacuumed.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumHeapBlksVacuumed, prometheus.GaugeValue, heapVacuumed, labels...) + + indexCount := 0.0 + if indexVacuumCount.Valid { + indexCount = float64(indexVacuumCount.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumIndexVacuumCount, prometheus.GaugeValue, indexCount, labels...) + + maxDead := 0.0 + if maxDeadTuples.Valid { + maxDead = float64(maxDeadTuples.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumMaxDeadTuples, prometheus.GaugeValue, maxDead, labels...) + + numDead := 0.0 + if numDeadTuples.Valid { + numDead = float64(numDeadTuples.Int64) + } + ch <- prometheus.MustNewConstMetric(statProgressVacuumNumDeadTuples, prometheus.GaugeValue, numDead, labels...) + } + + if err := rows.Err(); err != nil { + return err + } + return nil +} diff --git a/collector/pg_stat_progress_vacuum_test.go b/collector/pg_stat_progress_vacuum_test.go new file mode 100644 index 000000000..80572feb8 --- /dev/null +++ b/collector/pg_stat_progress_vacuum_test.go @@ -0,0 +1,135 @@ +// Copyright 2025 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package collector + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/smartystreets/goconvey/convey" +) + +func TestPGStatProgressVacuumCollector(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + + columns := []string{ + "datname", "relname", "phase", "heap_blks_total", "heap_blks_scanned", + "heap_blks_vacuumed", "index_vacuum_count", "max_dead_tuples", "num_dead_tuples", + } + + rows := sqlmock.NewRows(columns).AddRow( + "postgres", "a_table", 3, 3000, 400, 200, 2, 500, 123) + + mock.ExpectQuery(sanitizeQuery(statProgressVacuumQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatProgressVacuumCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatProgressVacuumCollector.Update; %+v", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "initializing"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "scanning heap"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "vacuuming indexes"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "vacuuming heap"}, metricType: dto.MetricType_GAUGE, value: 1}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "cleaning up indexes"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "truncating heap"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table", "phase": "performing final cleanup"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 3000}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 400}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 200}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 2}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 500}, + {labels: labelMap{"datname": "postgres", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 123}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(m, convey.ShouldResemble, expect) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("There were unfulfilled exceptions: %+v", err) + } +} + +func TestPGStatProgressVacuumCollectorNullValues(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db} + + columns := []string{ + "datname", "relname", "phase", "heap_blks_total", "heap_blks_scanned", + "heap_blks_vacuumed", "index_vacuum_count", "max_dead_tuples", "num_dead_tuples", + } + + rows := sqlmock.NewRows(columns).AddRow( + "postgres", nil, nil, nil, nil, nil, nil, nil, nil) + + mock.ExpectQuery(sanitizeQuery(statProgressVacuumQuery)).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatProgressVacuumCollector{} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatProgressVacuumCollector.Update; %+v", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "initializing"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "scanning heap"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "vacuuming indexes"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "vacuuming heap"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "cleaning up indexes"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "truncating heap"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown", "phase": "performing final cleanup"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("There were unfulfilled exceptions: %+v", err) + } +} From 8d5ec4b3ea77033d5d2b50fad1bb8df3f495265e Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Thu, 3 Apr 2025 16:23:40 +0100 Subject: [PATCH 24/33] Update Go (#1147) * Update Go to 1.24. * Update golangci-lint to v2. * Fixup linting issues. Signed-off-by: SuperQ --- .circleci/config.yml | 4 +- .github/workflows/golangci-lint.yml | 4 +- .golangci.yml | 53 +++++++++++++++--------- .promu.yml | 2 +- Makefile.common | 2 +- cmd/postgres_exporter/pg_setting.go | 4 +- cmd/postgres_exporter/pg_setting_test.go | 4 +- collector/collector_test.go | 20 ++++----- config/config.go | 4 +- config/config_test.go | 6 +-- 10 files changed, 58 insertions(+), 45 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6cfc880d2..ee03ff8d2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ executors: # This must match .promu.yml. golang: docker: - - image: cimg/go:1.23 + - image: cimg/go:1.24 jobs: test: @@ -23,7 +23,7 @@ jobs: integration: docker: - - image: cimg/go:1.23 + - image: cimg/go:1.24 - image: << parameters.postgres_image >> environment: POSTGRES_DB: circle_test diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 5342cbe08..3893ef86b 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -33,7 +33,7 @@ jobs: run: sudo apt-get update && sudo apt-get -y install libsnmp-dev if: github.repository == 'prometheus/snmp_exporter' - name: Lint - uses: golangci/golangci-lint-action@55c2c1448f86e01eaae002a5a3a9624417608d84 # v6.5.2 + uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: args: --verbose - version: v1.64.6 + version: v2.0.2 diff --git a/.golangci.yml b/.golangci.yml index 96487c898..4b58b08b6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,23 +1,36 @@ ---- +version: "2" linters: enable: - - misspell - - revive - -issues: - exclude-rules: - - path: _test.go - linters: - - errcheck - -linters-settings: - errcheck: - exclude-functions: - # Never check for logger errors. - - (github.com/go-kit/log.Logger).Log - revive: + - misspell + - revive + settings: + errcheck: + exclude-functions: + - (github.com/go-kit/log.Logger).Log + revive: + rules: + - name: unused-parameter + severity: warning + disabled: true + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling rules: - # https://github.com/mgechev/revive/blob/master/RULES_DESCRIPTIONS.md#unused-parameter - - name: unused-parameter - severity: warning - disabled: true + - linters: + - errcheck + path: _test.go + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.promu.yml b/.promu.yml index c2b361240..5f915289f 100644 --- a/.promu.yml +++ b/.promu.yml @@ -1,6 +1,6 @@ go: # This must match .circle/config.yml. - version: 1.23 + version: 1.24 repository: path: github.com/prometheus-community/postgres_exporter build: diff --git a/Makefile.common b/Makefile.common index 8cb383859..81bad5f42 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.64.6 +GOLANGCI_LINT_VERSION ?= v2.0.2 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) diff --git a/cmd/postgres_exporter/pg_setting.go b/cmd/postgres_exporter/pg_setting.go index 02e65dd2a..5b13e160f 100644 --- a/cmd/postgres_exporter/pg_setting.go +++ b/cmd/postgres_exporter/pg_setting.go @@ -67,7 +67,7 @@ type pgSetting struct { func (s *pgSetting) metric(labels prometheus.Labels) prometheus.Metric { var ( err error - name = strings.Replace(strings.Replace(s.name, ".", "_", -1), "-", "_", -1) + name = strings.ReplaceAll(strings.ReplaceAll(s.name, ".", "_"), "-", "_") unit = s.unit // nolint: ineffassign shortDesc = fmt.Sprintf("Server Parameter: %s", s.name) subsystem = "settings" @@ -131,7 +131,7 @@ func (s *pgSetting) normaliseUnit() (val float64, unit string, err error) { case "B", "kB", "MB", "GB", "TB", "1kB", "2kB", "4kB", "8kB", "16kB", "32kB", "64kB", "16MB", "32MB", "64MB": unit = "bytes" default: - err = fmt.Errorf("Unknown unit for runtime variable: %q", s.unit) + err = fmt.Errorf("unknown unit for runtime variable: %q", s.unit) return } diff --git a/cmd/postgres_exporter/pg_setting_test.go b/cmd/postgres_exporter/pg_setting_test.go index 0e010444d..6923da630 100644 --- a/cmd/postgres_exporter/pg_setting_test.go +++ b/cmd/postgres_exporter/pg_setting_test.go @@ -214,7 +214,7 @@ var fixtures = []fixture{ n: normalised{ val: 10, unit: "", - err: `Unknown unit for runtime variable: "nonexistent"`, + err: `unknown unit for runtime variable: "nonexistent"`, }, }, } @@ -240,7 +240,7 @@ func (s *PgSettingSuite) TestNormaliseUnit(c *C) { func (s *PgSettingSuite) TestMetric(c *C) { defer func() { if r := recover(); r != nil { - if r.(error).Error() != `Unknown unit for runtime variable: "nonexistent"` { + if r.(error).Error() != `unknown unit for runtime variable: "nonexistent"` { panic(r) } } diff --git a/collector/collector_test.go b/collector/collector_test.go index 18101f00e..d3b473b43 100644 --- a/collector/collector_test.go +++ b/collector/collector_test.go @@ -48,15 +48,15 @@ func readMetric(m prometheus.Metric) MetricResult { func sanitizeQuery(q string) string { q = strings.Join(strings.Fields(q), " ") - q = strings.Replace(q, "(", "\\(", -1) - q = strings.Replace(q, "?", "\\?", -1) - q = strings.Replace(q, ")", "\\)", -1) - q = strings.Replace(q, "[", "\\[", -1) - q = strings.Replace(q, "]", "\\]", -1) - q = strings.Replace(q, "{", "\\{", -1) - q = strings.Replace(q, "}", "\\}", -1) - q = strings.Replace(q, "*", "\\*", -1) - q = strings.Replace(q, "^", "\\^", -1) - q = strings.Replace(q, "$", "\\$", -1) + q = strings.ReplaceAll(q, "(", "\\(") + q = strings.ReplaceAll(q, "?", "\\?") + q = strings.ReplaceAll(q, ")", "\\)") + q = strings.ReplaceAll(q, "[", "\\[") + q = strings.ReplaceAll(q, "]", "\\]") + q = strings.ReplaceAll(q, "{", "\\{") + q = strings.ReplaceAll(q, "}", "\\}") + q = strings.ReplaceAll(q, "*", "\\*") + q = strings.ReplaceAll(q, "^", "\\^") + q = strings.ReplaceAll(q, "$", "\\$") return q } diff --git a/config/config.go b/config/config.go index 7cdc08f7b..52c66513a 100644 --- a/config/config.go +++ b/config/config.go @@ -79,14 +79,14 @@ func (ch *Handler) ReloadConfig(f string, logger *slog.Logger) error { yamlReader, err := os.Open(f) if err != nil { - return fmt.Errorf("Error opening config file %q: %s", f, err) + return fmt.Errorf("error opening config file %q: %s", f, err) } defer yamlReader.Close() decoder := yaml.NewDecoder(yamlReader) decoder.KnownFields(true) if err = decoder.Decode(config); err != nil { - return fmt.Errorf("Error parsing config file %q: %s", f, err) + return fmt.Errorf("error parsing config file %q: %s", f, err) } ch.Lock() diff --git a/config/config_test.go b/config/config_test.go index d5d23d3ba..fa59c9b40 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -24,7 +24,7 @@ func TestLoadConfig(t *testing.T) { err := ch.ReloadConfig("testdata/config-good.yaml", nil) if err != nil { - t.Errorf("Error loading config: %s", err) + t.Errorf("error loading config: %s", err) } } @@ -39,11 +39,11 @@ func TestLoadBadConfigs(t *testing.T) { }{ { input: "testdata/config-bad-auth-module.yaml", - want: "Error parsing config file \"testdata/config-bad-auth-module.yaml\": yaml: unmarshal errors:\n line 3: field pretendauth not found in type config.AuthModule", + want: "error parsing config file \"testdata/config-bad-auth-module.yaml\": yaml: unmarshal errors:\n line 3: field pretendauth not found in type config.AuthModule", }, { input: "testdata/config-bad-extra-field.yaml", - want: "Error parsing config file \"testdata/config-bad-extra-field.yaml\": yaml: unmarshal errors:\n line 8: field doesNotExist not found in type config.AuthModule", + want: "error parsing config file \"testdata/config-bad-extra-field.yaml\": yaml: unmarshal errors:\n line 8: field doesNotExist not found in type config.AuthModule", }, } From 43576acc7642f5cb97d3d1f6bbe0f9e514ed9301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:17:55 +0200 Subject: [PATCH 25/33] Bump github.com/prometheus/client_golang from 1.21.0 to 1.21.1 (#1144) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.21.0 to 1.21.1. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.21.0...v1.21.1) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-version: 1.21.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 +++- go.sum | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1d585cd23..889893b5e 100644 --- a/go.mod +++ b/go.mod @@ -2,12 +2,14 @@ module github.com/prometheus-community/postgres_exporter go 1.23.0 +toolchain go1.24.1 + require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/alecthomas/kingpin/v2 v2.4.0 github.com/blang/semver/v4 v4.0.0 github.com/lib/pq v1.10.9 - github.com/prometheus/client_golang v1.21.0 + github.com/prometheus/client_golang v1.21.1 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.62.0 github.com/prometheus/exporter-toolkit v0.14.0 diff --git a/go.sum b/go.sum index 09262cf79..5f7a25e95 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= -github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= +github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= From f8b7139174317e3df6d756a02c99b4d8de649d0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:18:17 +0200 Subject: [PATCH 26/33] Bump github.com/prometheus/common from 0.62.0 to 0.63.0 (#1143) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.62.0 to 0.63.0. - [Release notes](https://github.com/prometheus/common/releases) - [Changelog](https://github.com/prometheus/common/blob/main/RELEASE.md) - [Commits](https://github.com/prometheus/common/compare/v0.62.0...v0.63.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-version: 0.63.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 889893b5e..ac1ceeff4 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/lib/pq v1.10.9 github.com/prometheus/client_golang v1.21.1 github.com/prometheus/client_model v0.6.1 - github.com/prometheus/common v0.62.0 + github.com/prometheus/common v0.63.0 github.com/prometheus/exporter-toolkit v0.14.0 github.com/smartystreets/goconvey v1.8.1 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c @@ -40,9 +40,9 @@ require ( github.com/xhit/go-str2duration/v2 v2.1.0 // indirect golang.org/x/crypto v0.35.0 // indirect golang.org/x/net v0.36.0 // indirect - golang.org/x/oauth2 v0.24.0 // indirect + golang.org/x/oauth2 v0.25.0 // indirect golang.org/x/sync v0.11.0 // indirect golang.org/x/sys v0.30.0 // indirect golang.org/x/text v0.22.0 // indirect - google.golang.org/protobuf v1.36.1 // indirect + google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/go.sum b/go.sum index 5f7a25e95..939e1d083 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= @@ -54,8 +54,8 @@ github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGC github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= github.com/prometheus/exporter-toolkit v0.14.0 h1:NMlswfibpcZZ+H0sZBiTjrA3/aBFHkNZqE+iCj5EmRg= github.com/prometheus/exporter-toolkit v0.14.0/go.mod h1:Gu5LnVvt7Nr/oqTBUC23WILZepW0nffNo10XdhQcwWA= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= @@ -77,16 +77,16 @@ golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= -golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= -golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= +golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk= -google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 5edc331876c3f94bd5ca26e502fee4e4df77e2d1 Mon Sep 17 00:00:00 2001 From: Felix Yuan Date: Fri, 18 Apr 2025 07:14:25 -0700 Subject: [PATCH 27/33] Record table only size bytes as well in addition to the total size bytes (#1149) * Record table only size bytes as well in addition to the total size bytes Signed-off-by: Felix Yuan * Update collector/pg_stat_user_tables.go Co-authored-by: Ben Kochie Signed-off-by: Felix Yuan * Update collector/pg_stat_user_tables.go Co-authored-by: Ben Kochie Signed-off-by: Felix Yuan * Finish renaming elements to index and table size Signed-off-by: Felix Yuan --------- Signed-off-by: Felix Yuan Co-authored-by: Ben Kochie --- collector/pg_stat_user_tables.go | 40 +++++++++++++++++++-------- collector/pg_stat_user_tables_test.go | 14 ++++++++-- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/collector/pg_stat_user_tables.go b/collector/pg_stat_user_tables.go index 254d76a66..ad8bcace7 100644 --- a/collector/pg_stat_user_tables.go +++ b/collector/pg_stat_user_tables.go @@ -150,9 +150,15 @@ var ( []string{"datname", "schemaname", "relname"}, prometheus.Labels{}, ) - statUserTablesTotalSize = prometheus.NewDesc( - prometheus.BuildFQName(namespace, userTableSubsystem, "size_bytes"), - "Total disk space used by this table, in bytes, including all indexes and TOAST data", + statUserIndexSize = prometheus.NewDesc( + prometheus.BuildFQName(namespace, userTableSubsystem, "index_size_bytes"), + "Total disk space used by this index, in bytes", + []string{"datname", "schemaname", "relname"}, + prometheus.Labels{}, + ) + statUserTableSize = prometheus.NewDesc( + prometheus.BuildFQName(namespace, userTableSubsystem, "table_size_bytes"), + "Total disk space used by this table, in bytes", []string{"datname", "schemaname", "relname"}, prometheus.Labels{}, ) @@ -180,7 +186,8 @@ var ( autovacuum_count, analyze_count, autoanalyze_count, - pg_total_relation_size(relid) as total_size + pg_indexes_size(relid) as indexes_size, + pg_table_size(relid) as table_size FROM pg_stat_user_tables` ) @@ -198,10 +205,10 @@ func (c *PGStatUserTablesCollector) Update(ctx context.Context, instance *instan for rows.Next() { var datname, schemaname, relname sql.NullString var seqScan, seqTupRead, idxScan, idxTupFetch, nTupIns, nTupUpd, nTupDel, nTupHotUpd, nLiveTup, nDeadTup, - nModSinceAnalyze, vacuumCount, autovacuumCount, analyzeCount, autoanalyzeCount, totalSize sql.NullInt64 + nModSinceAnalyze, vacuumCount, autovacuumCount, analyzeCount, autoanalyzeCount, indexSize, tableSize sql.NullInt64 var lastVacuum, lastAutovacuum, lastAnalyze, lastAutoanalyze sql.NullTime - if err := rows.Scan(&datname, &schemaname, &relname, &seqScan, &seqTupRead, &idxScan, &idxTupFetch, &nTupIns, &nTupUpd, &nTupDel, &nTupHotUpd, &nLiveTup, &nDeadTup, &nModSinceAnalyze, &lastVacuum, &lastAutovacuum, &lastAnalyze, &lastAutoanalyze, &vacuumCount, &autovacuumCount, &analyzeCount, &autoanalyzeCount, &totalSize); err != nil { + if err := rows.Scan(&datname, &schemaname, &relname, &seqScan, &seqTupRead, &idxScan, &idxTupFetch, &nTupIns, &nTupUpd, &nTupDel, &nTupHotUpd, &nLiveTup, &nDeadTup, &nModSinceAnalyze, &lastVacuum, &lastAutovacuum, &lastAnalyze, &lastAutoanalyze, &vacuumCount, &autovacuumCount, &analyzeCount, &autoanalyzeCount, &indexSize, &tableSize); err != nil { return err } @@ -427,14 +434,25 @@ func (c *PGStatUserTablesCollector) Update(ctx context.Context, instance *instan datnameLabel, schemanameLabel, relnameLabel, ) - totalSizeMetric := 0.0 - if totalSize.Valid { - totalSizeMetric = float64(totalSize.Int64) + indexSizeMetric := 0.0 + if indexSize.Valid { + indexSizeMetric = float64(indexSize.Int64) + } + ch <- prometheus.MustNewConstMetric( + statUserIndexSize, + prometheus.GaugeValue, + indexSizeMetric, + datnameLabel, schemanameLabel, relnameLabel, + ) + + tableSizeMetric := 0.0 + if tableSize.Valid { + tableSizeMetric = float64(tableSize.Int64) } ch <- prometheus.MustNewConstMetric( - statUserTablesTotalSize, + statUserTableSize, prometheus.GaugeValue, - totalSizeMetric, + tableSizeMetric, datnameLabel, schemanameLabel, relnameLabel, ) } diff --git a/collector/pg_stat_user_tables_test.go b/collector/pg_stat_user_tables_test.go index 5e82335c3..4649bdbc5 100644 --- a/collector/pg_stat_user_tables_test.go +++ b/collector/pg_stat_user_tables_test.go @@ -72,7 +72,8 @@ func TestPGStatUserTablesCollector(t *testing.T) { "autovacuum_count", "analyze_count", "autoanalyze_count", - "total_size"} + "index_size", + "table_size"} rows := sqlmock.NewRows(columns). AddRow("postgres", "public", @@ -96,7 +97,8 @@ func TestPGStatUserTablesCollector(t *testing.T) { 12, 13, 14, - 15) + 15, + 16) mock.ExpectQuery(sanitizeQuery(statUserTablesQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -128,6 +130,8 @@ func TestPGStatUserTablesCollector(t *testing.T) { {labels: labelMap{"datname": "postgres", "schemaname": "public", "relname": "a_table"}, metricType: dto.MetricType_COUNTER, value: 12}, {labels: labelMap{"datname": "postgres", "schemaname": "public", "relname": "a_table"}, metricType: dto.MetricType_COUNTER, value: 13}, {labels: labelMap{"datname": "postgres", "schemaname": "public", "relname": "a_table"}, metricType: dto.MetricType_COUNTER, value: 14}, + {labels: labelMap{"datname": "postgres", "schemaname": "public", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 15}, + {labels: labelMap{"datname": "postgres", "schemaname": "public", "relname": "a_table"}, metricType: dto.MetricType_GAUGE, value: 16}, } convey.Convey("Metrics comparison", t, func() { @@ -173,7 +177,8 @@ func TestPGStatUserTablesCollectorNullValues(t *testing.T) { "autovacuum_count", "analyze_count", "autoanalyze_count", - "total_size"} + "index_size", + "table_size"} rows := sqlmock.NewRows(columns). AddRow("postgres", nil, @@ -197,6 +202,7 @@ func TestPGStatUserTablesCollectorNullValues(t *testing.T) { nil, nil, nil, + nil, nil) mock.ExpectQuery(sanitizeQuery(statUserTablesQuery)).WillReturnRows(rows) ch := make(chan prometheus.Metric) @@ -229,6 +235,8 @@ func TestPGStatUserTablesCollectorNullValues(t *testing.T) { {labels: labelMap{"datname": "postgres", "schemaname": "unknown", "relname": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, {labels: labelMap{"datname": "postgres", "schemaname": "unknown", "relname": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, {labels: labelMap{"datname": "postgres", "schemaname": "unknown", "relname": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"datname": "postgres", "schemaname": "unknown", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, + {labels: labelMap{"datname": "postgres", "schemaname": "unknown", "relname": "unknown"}, metricType: dto.MetricType_GAUGE, value: 0}, } convey.Convey("Metrics comparison", t, func() { From 6526065fbcdc2fa780b1db6784b9978602f37d94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 21:26:10 -0400 Subject: [PATCH 28/33] Bump golang.org/x/net from 0.36.0 to 0.38.0 (#1152) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.36.0 to 0.38.0. - [Commits](https://github.com/golang/net/compare/v0.36.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index ac1ceeff4..e3eba7880 100644 --- a/go.mod +++ b/go.mod @@ -38,11 +38,11 @@ require ( github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/smarty/assertions v1.15.0 // indirect github.com/xhit/go-str2duration/v2 v2.1.0 // indirect - golang.org/x/crypto v0.35.0 // indirect - golang.org/x/net v0.36.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/oauth2 v0.25.0 // indirect - golang.org/x/sync v0.11.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect google.golang.org/protobuf v1.36.5 // indirect ) diff --git a/go.sum b/go.sum index 939e1d083..25dd5c07d 100644 --- a/go.sum +++ b/go.sum @@ -73,18 +73,18 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 5ade81cb9d391aef0606c44ee13830efa55bb3f2 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Wed, 7 May 2025 03:54:01 +0200 Subject: [PATCH 29/33] Update common Prometheus files (#1155) Signed-off-by: prombot --- .github/workflows/golangci-lint.yml | 2 +- Makefile.common | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 3893ef86b..672dd424d 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -36,4 +36,4 @@ jobs: uses: golangci/golangci-lint-action@1481404843c368bc19ca9406f87d6e0fc97bdcfd # v7.0.0 with: args: --verbose - version: v2.0.2 + version: v2.1.5 diff --git a/Makefile.common b/Makefile.common index 81bad5f42..d8b798909 100644 --- a/Makefile.common +++ b/Makefile.common @@ -61,7 +61,7 @@ PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v2.0.2 +GOLANGCI_LINT_VERSION ?= v2.1.5 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) From bd8a6135ecc149c02c429a5fa285cac228b244cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 6 May 2025 22:14:44 -0400 Subject: [PATCH 30/33] Bump github.com/prometheus/client_golang from 1.21.1 to 1.22.0 (#1154) Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.21.1 to 1.22.0. - [Release notes](https://github.com/prometheus/client_golang/releases) - [Changelog](https://github.com/prometheus/client_golang/blob/main/CHANGELOG.md) - [Commits](https://github.com/prometheus/client_golang/compare/v1.21.1...v1.22.0) --- updated-dependencies: - dependency-name: github.com/prometheus/client_golang dependency-version: 1.22.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 +-- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index e3eba7880..9acdafc81 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/alecthomas/kingpin/v2 v2.4.0 github.com/blang/semver/v4 v4.0.0 github.com/lib/pq v1.10.9 - github.com/prometheus/client_golang v1.21.1 + github.com/prometheus/client_golang v1.22.0 github.com/prometheus/client_model v0.6.1 github.com/prometheus/common v0.63.0 github.com/prometheus/exporter-toolkit v0.14.0 @@ -27,7 +27,6 @@ require ( github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect - github.com/klauspost/compress v1.17.11 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/mdlayher/socket v0.4.1 // indirect diff --git a/go.sum b/go.sum index 25dd5c07d..cd073f325 100644 --- a/go.sum +++ b/go.sum @@ -26,8 +26,8 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -50,8 +50,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= From de42ef7de8bd14f314a609ef50e4c323617258dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20=C5=A0tiller?= Date: Fri, 16 May 2025 10:50:19 +0200 Subject: [PATCH 31/33] Export query itself together with queryId in stat_statement metrics (#940) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Export query itself together with queryId in stat_statement metrics * The feature must be enabled via flag. * Limit length of selected query. The query is not added to every metrics, but instead of new metric stat_statement_query_id is introduced that contains mapping between queryId and query. Fixes: #813 --------- Signed-off-by: Jakub Štiller Signed-off-by: Jakub Štiller --- README.md | 6 + collector/collector.go | 7 +- collector/pg_stat_statements.go | 89 +++++++++++-- collector/pg_stat_statements_test.go | 185 ++++++++++++++++++++++++++- 4 files changed, 270 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index dedc5aa55..01bd8b30d 100644 --- a/README.md +++ b/README.md @@ -150,6 +150,12 @@ This will build the docker image as `prometheuscommunity/postgres_exporter:${bra * `[no-]collector.stat_statements` Enable the `stat_statements` collector (default: disabled). +* `[no-]collector.stat_statements.include_query` + Enable selecting statement query together with queryId. (default: disabled) + +* `--collector.stat_statements.query_length` + Maximum length of the statement text. Default is 120. + * `[no-]collector.stat_user_tables` Enable the `stat_user_tables` collector (default: enabled). diff --git a/collector/collector.go b/collector/collector.go index f2102d5ef..298bc36ee 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -37,8 +37,9 @@ const ( // Namespace for all metrics. namespace = "pg" - defaultEnabled = true - defaultDisabled = false + collectorFlagPrefix = "collector." + defaultEnabled = true + defaultDisabled = false ) var ( @@ -74,7 +75,7 @@ func registerCollector(name string, isDefaultEnabled bool, createFunc func(colle } // Create flag for this collector - flagName := fmt.Sprintf("collector.%s", name) + flagName := collectorFlagPrefix + name flagHelp := fmt.Sprintf("Enable the %s collector (default: %s).", name, helpDefaultState) defaultValue := fmt.Sprintf("%v", isDefaultEnabled) diff --git a/collector/pg_stat_statements.go b/collector/pg_stat_statements.go index cabdc3619..9160d3c16 100644 --- a/collector/pg_stat_statements.go +++ b/collector/pg_stat_statements.go @@ -16,27 +16,51 @@ package collector import ( "context" "database/sql" + "fmt" "log/slog" + "github.com/alecthomas/kingpin/v2" "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" ) const statStatementsSubsystem = "stat_statements" +var ( + includeQueryFlag *bool = nil + statementLengthFlag *uint = nil +) + func init() { // WARNING: // Disabled by default because this set of metrics can be quite expensive on a busy server // Every unique query will cause a new timeseries to be created registerCollector(statStatementsSubsystem, defaultDisabled, NewPGStatStatementsCollector) + + includeQueryFlag = kingpin.Flag( + fmt.Sprint(collectorFlagPrefix, statStatementsSubsystem, ".include_query"), + "Enable selecting statement query together with queryId. (default: disabled)"). + Default(fmt.Sprintf("%v", defaultDisabled)). + Bool() + statementLengthFlag = kingpin.Flag( + fmt.Sprint(collectorFlagPrefix, statStatementsSubsystem, ".query_length"), + "Maximum length of the statement text."). + Default("120"). + Uint() } type PGStatStatementsCollector struct { - log *slog.Logger + log *slog.Logger + includeQueryStatement bool + statementLength uint } func NewPGStatStatementsCollector(config collectorConfig) (Collector, error) { - return &PGStatStatementsCollector{log: config.logger}, nil + return &PGStatStatementsCollector{ + log: config.logger, + includeQueryStatement: *includeQueryFlag, + statementLength: *statementLengthFlag, + }, nil } var ( @@ -71,10 +95,22 @@ var ( prometheus.Labels{}, ) + statStatementsQuery = prometheus.NewDesc( + prometheus.BuildFQName(namespace, statStatementsSubsystem, "query_id"), + "SQL Query to queryid mapping", + []string{"queryid", "query"}, + prometheus.Labels{}, + ) +) + +const ( + pgStatStatementQuerySelect = `LEFT(pg_stat_statements.query, %d) as query,` + pgStatStatementsQuery = `SELECT pg_get_userbyid(userid) as user, pg_database.datname, pg_stat_statements.queryid, + %s pg_stat_statements.calls as calls_total, pg_stat_statements.total_time / 1000.0 as seconds_total, pg_stat_statements.rows as rows_total, @@ -96,6 +132,7 @@ var ( pg_get_userbyid(userid) as user, pg_database.datname, pg_stat_statements.queryid, + %s pg_stat_statements.calls as calls_total, pg_stat_statements.total_exec_time / 1000.0 as seconds_total, pg_stat_statements.rows as rows_total, @@ -117,6 +154,7 @@ var ( pg_get_userbyid(userid) as user, pg_database.datname, pg_stat_statements.queryid, + %s pg_stat_statements.calls as calls_total, pg_stat_statements.total_exec_time / 1000.0 as seconds_total, pg_stat_statements.rows as rows_total, @@ -135,30 +173,42 @@ var ( LIMIT 100;` ) -func (PGStatStatementsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { - var query string +func (c PGStatStatementsCollector) Update(ctx context.Context, instance *instance, ch chan<- prometheus.Metric) error { + var queryTemplate string switch { case instance.version.GE(semver.MustParse("17.0.0")): - query = pgStatStatementsQuery_PG17 + queryTemplate = pgStatStatementsQuery_PG17 case instance.version.GE(semver.MustParse("13.0.0")): - query = pgStatStatementsNewQuery + queryTemplate = pgStatStatementsNewQuery default: - query = pgStatStatementsQuery + queryTemplate = pgStatStatementsQuery } + var querySelect = "" + if c.includeQueryStatement { + querySelect = fmt.Sprintf(pgStatStatementQuerySelect, c.statementLength) + } + query := fmt.Sprintf(queryTemplate, querySelect) db := instance.getDB() rows, err := db.QueryContext(ctx, query) + var presentQueryIds = make(map[string]struct{}) + if err != nil { return err } defer rows.Close() for rows.Next() { - var user, datname, queryid sql.NullString + var user, datname, queryid, statement sql.NullString var callsTotal, rowsTotal sql.NullInt64 var secondsTotal, blockReadSecondsTotal, blockWriteSecondsTotal sql.NullFloat64 - - if err := rows.Scan(&user, &datname, &queryid, &callsTotal, &secondsTotal, &rowsTotal, &blockReadSecondsTotal, &blockWriteSecondsTotal); err != nil { + var columns []any + if c.includeQueryStatement { + columns = []any{&user, &datname, &queryid, &statement, &callsTotal, &secondsTotal, &rowsTotal, &blockReadSecondsTotal, &blockWriteSecondsTotal} + } else { + columns = []any{&user, &datname, &queryid, &callsTotal, &secondsTotal, &rowsTotal, &blockReadSecondsTotal, &blockWriteSecondsTotal} + } + if err := rows.Scan(columns...); err != nil { return err } @@ -229,6 +279,25 @@ func (PGStatStatementsCollector) Update(ctx context.Context, instance *instance, blockWriteSecondsTotalMetric, userLabel, datnameLabel, queryidLabel, ) + + if c.includeQueryStatement { + _, ok := presentQueryIds[queryidLabel] + if !ok { + presentQueryIds[queryidLabel] = struct{}{} + + queryLabel := "unknown" + if statement.Valid { + queryLabel = statement.String + } + + ch <- prometheus.MustNewConstMetric( + statStatementsQuery, + prometheus.CounterValue, + 1, + queryidLabel, queryLabel, + ) + } + } } if err := rows.Err(); err != nil { return err diff --git a/collector/pg_stat_statements_test.go b/collector/pg_stat_statements_test.go index ea035f557..0497ba380 100644 --- a/collector/pg_stat_statements_test.go +++ b/collector/pg_stat_statements_test.go @@ -14,6 +14,7 @@ package collector import ( "context" + "fmt" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -35,7 +36,7 @@ func TestPGStateStatementsCollector(t *testing.T) { columns := []string{"user", "datname", "queryid", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} rows := sqlmock.NewRows(columns). AddRow("postgres", "postgres", 1500, 5, 0.4, 100, 0.1, 0.2) - mock.ExpectQuery(sanitizeQuery(pgStatStatementsQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsQuery, ""))).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -66,6 +67,50 @@ func TestPGStateStatementsCollector(t *testing.T) { } } +func TestPGStateStatementsCollectorWithStatement(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("12.0.0")} + + columns := []string{"user", "datname", "queryid", "LEFT(pg_stat_statements.query, 100) as query", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} + rows := sqlmock.NewRows(columns). + AddRow("postgres", "postgres", 1500, "select 1 from foo", 5, 0.4, 100, 0.1, 0.2) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsQuery, fmt.Sprintf(pgStatStatementQuerySelect, 100)))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatStatementsCollector{includeQueryStatement: true, statementLength: 100} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 5}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.4}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 100}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.1}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.2}, + {labels: labelMap{"queryid": "1500", "query": "select 1 from foo"}, metricType: dto.MetricType_COUNTER, value: 1}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + func TestPGStateStatementsCollectorNull(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -78,7 +123,7 @@ func TestPGStateStatementsCollectorNull(t *testing.T) { columns := []string{"user", "datname", "queryid", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} rows := sqlmock.NewRows(columns). AddRow(nil, nil, nil, nil, nil, nil, nil, nil) - mock.ExpectQuery(sanitizeQuery(pgStatStatementsNewQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsNewQuery, ""))).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -109,6 +154,50 @@ func TestPGStateStatementsCollectorNull(t *testing.T) { } } +func TestPGStateStatementsCollectorNullWithStatement(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("13.3.7")} + + columns := []string{"user", "datname", "queryid", "LEFT(pg_stat_statements.query, 200) as query", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} + rows := sqlmock.NewRows(columns). + AddRow(nil, nil, nil, nil, nil, nil, nil, nil, nil) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsNewQuery, fmt.Sprintf(pgStatStatementQuerySelect, 200)))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatStatementsCollector{includeQueryStatement: true, statementLength: 200} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"user": "unknown", "datname": "unknown", "queryid": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"user": "unknown", "datname": "unknown", "queryid": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"user": "unknown", "datname": "unknown", "queryid": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"user": "unknown", "datname": "unknown", "queryid": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"user": "unknown", "datname": "unknown", "queryid": "unknown"}, metricType: dto.MetricType_COUNTER, value: 0}, + {labels: labelMap{"queryid": "unknown", "query": "unknown"}, metricType: dto.MetricType_COUNTER, value: 1}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + func TestPGStateStatementsCollectorNewPG(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -121,7 +210,7 @@ func TestPGStateStatementsCollectorNewPG(t *testing.T) { columns := []string{"user", "datname", "queryid", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} rows := sqlmock.NewRows(columns). AddRow("postgres", "postgres", 1500, 5, 0.4, 100, 0.1, 0.2) - mock.ExpectQuery(sanitizeQuery(pgStatStatementsNewQuery)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsNewQuery, ""))).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -152,6 +241,50 @@ func TestPGStateStatementsCollectorNewPG(t *testing.T) { } } +func TestPGStateStatementsCollectorNewPGWithStatement(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("13.3.7")} + + columns := []string{"user", "datname", "queryid", "LEFT(pg_stat_statements.query, 300) as query", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} + rows := sqlmock.NewRows(columns). + AddRow("postgres", "postgres", 1500, "select 1 from foo", 5, 0.4, 100, 0.1, 0.2) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsNewQuery, fmt.Sprintf(pgStatStatementQuerySelect, 300)))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatStatementsCollector{includeQueryStatement: true, statementLength: 300} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 5}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.4}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 100}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.1}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.2}, + {labels: labelMap{"queryid": "1500", "query": "select 1 from foo"}, metricType: dto.MetricType_COUNTER, value: 1}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} + func TestPGStateStatementsCollector_PG17(t *testing.T) { db, mock, err := sqlmock.New() if err != nil { @@ -164,7 +297,7 @@ func TestPGStateStatementsCollector_PG17(t *testing.T) { columns := []string{"user", "datname", "queryid", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} rows := sqlmock.NewRows(columns). AddRow("postgres", "postgres", 1500, 5, 0.4, 100, 0.1, 0.2) - mock.ExpectQuery(sanitizeQuery(pgStatStatementsQuery_PG17)).WillReturnRows(rows) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsQuery_PG17, ""))).WillReturnRows(rows) ch := make(chan prometheus.Metric) go func() { @@ -194,3 +327,47 @@ func TestPGStateStatementsCollector_PG17(t *testing.T) { t.Errorf("there were unfulfilled exceptions: %s", err) } } + +func TestPGStateStatementsCollector_PG17_WithStatement(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("Error opening a stub db connection: %s", err) + } + defer db.Close() + + inst := &instance{db: db, version: semver.MustParse("17.0.0")} + + columns := []string{"user", "datname", "queryid", "LEFT(pg_stat_statements.query, 300) as query", "calls_total", "seconds_total", "rows_total", "block_read_seconds_total", "block_write_seconds_total"} + rows := sqlmock.NewRows(columns). + AddRow("postgres", "postgres", 1500, "select 1 from foo", 5, 0.4, 100, 0.1, 0.2) + mock.ExpectQuery(sanitizeQuery(fmt.Sprintf(pgStatStatementsQuery_PG17, fmt.Sprintf(pgStatStatementQuerySelect, 300)))).WillReturnRows(rows) + + ch := make(chan prometheus.Metric) + go func() { + defer close(ch) + c := PGStatStatementsCollector{includeQueryStatement: true, statementLength: 300} + + if err := c.Update(context.Background(), inst, ch); err != nil { + t.Errorf("Error calling PGStatStatementsCollector.Update: %s", err) + } + }() + + expected := []MetricResult{ + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 5}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.4}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 100}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.1}, + {labels: labelMap{"user": "postgres", "datname": "postgres", "queryid": "1500"}, metricType: dto.MetricType_COUNTER, value: 0.2}, + {labels: labelMap{"queryid": "1500", "query": "select 1 from foo"}, metricType: dto.MetricType_COUNTER, value: 1}, + } + + convey.Convey("Metrics comparison", t, func() { + for _, expect := range expected { + m := readMetric(<-ch) + convey.So(expect, convey.ShouldResemble, m) + } + }) + if err := mock.ExpectationsWereMet(); err != nil { + t.Errorf("there were unfulfilled exceptions: %s", err) + } +} From d1a957f679fd808c02cd88e6da7c11aeb7eae4d8 Mon Sep 17 00:00:00 2001 From: PrometheusBot Date: Sat, 17 May 2025 12:06:43 +0200 Subject: [PATCH 32/33] Update common Prometheus files (#1159) Signed-off-by: prombot --- Makefile.common | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile.common b/Makefile.common index d8b798909..4de21512f 100644 --- a/Makefile.common +++ b/Makefile.common @@ -62,6 +62,7 @@ SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= GOLANGCI_LINT_VERSION ?= v2.1.5 +GOLANGCI_FMT_OPTS ?= # golangci-lint only supports linux, darwin and windows platforms on i386/amd64/arm64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) @@ -156,9 +157,13 @@ $(GOTEST_DIR): @mkdir -p $@ .PHONY: common-format -common-format: +common-format: $(GOLANGCI_LINT) @echo ">> formatting code" $(GO) fmt $(pkgs) +ifdef GOLANGCI_LINT + @echo ">> formatting code with golangci-lint" + $(GOLANGCI_LINT) fmt $(GOLANGCI_FMT_OPTS) +endif .PHONY: common-vet common-vet: @@ -248,8 +253,8 @@ $(PROMU): cp $(PROMU_TMP)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM)/promu $(FIRST_GOPATH)/bin/promu rm -r $(PROMU_TMP) -.PHONY: proto -proto: +.PHONY: common-proto +common-proto: @echo ">> generating code from proto files" @./scripts/genproto.sh From 94e8399935e541262aaa2bf4e1723c232bea6c04 Mon Sep 17 00:00:00 2001 From: Jonathan Bowe Date: Thu, 29 May 2025 07:53:56 -0400 Subject: [PATCH 33/33] Feat: Improve Error Handling for Server.Scrape (#1158) * Update querySettings Error Return in Scrape If there are errors querying namespace mappings, the potential error from querySettings is obscured. Adding an immediate return if there are errors retreiving settings. Signed-off-by: Jonathan Bowe * Improve Verbosity of queryNamespaceMappings Errors Previously if any errors were encountered by queryNamespaceMappings, only a count of those errors was returned - making debugging those errors harder than it needs to be. I'm changing this to immediately return nil if no errors are encountered, and otherwise an error will be formatted with each of the namespaces and what the error was for that namespace. Signed-off-by: Jonathan Bowe * Simplify Error Message Co-authored-by: Ben Kochie Signed-off-by: Jonathan Bowe --------- Signed-off-by: Jonathan Bowe Signed-off-by: Jonathan Bowe Co-authored-by: Ben Kochie --- cmd/postgres_exporter/server.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cmd/postgres_exporter/server.go b/cmd/postgres_exporter/server.go index bd4e76e10..3d2ecde91 100644 --- a/cmd/postgres_exporter/server.go +++ b/cmd/postgres_exporter/server.go @@ -119,12 +119,17 @@ func (s *Server) Scrape(ch chan<- prometheus.Metric, disableSettingsMetrics bool if !disableSettingsMetrics && s.master { if err = querySettings(ch, s); err != nil { err = fmt.Errorf("error retrieving settings: %s", err) + return err } } errMap := queryNamespaceMappings(ch, s) - if len(errMap) > 0 { - err = fmt.Errorf("queryNamespaceMappings returned %d errors", len(errMap)) + if len(errMap) == 0 { + return nil + } + err = fmt.Errorf("queryNamespaceMappings errors encountered") + for namespace, errStr := range errMap { + err = fmt.Errorf("%s, namespace: %s error: %s", err, namespace, errStr) } return err