Merge branch 'master' of https://github.com/wrouesnel/postgres_exporter

Author: disco-stu

.gitignore

@@ -1,8 +1,14 @@
 .build
 postgres_exporter
+postgres_exporter_integration_test
 *.tar.gz
 *.test
 *-stamp
 .idea
 *.iml
 *.rpm
+cover.out
+cover.*.out
+.coverage
+*.prom
+.metrics.*.*.prom

.travis.yml

@@ -3,20 +3,38 @@ services:
 - docker
 language: go
 go:
-- '1.6'
+- '1.8'
+# Make sure we have p2 and the postgres client.
+before_install:
+- go get -v github.com/mattn/goveralls
+- sudo wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 &&
+  sudo chmod +x /usr/local/bin/p2
+- sudo wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 &&
+  sudo chmod +x /usr/local/bin/docker-compose
+- sudo apt-get update && sudo apt-get install postgresql-client-common
+  
 script:
 - make all
 - make docker
 - make test-integration
+- make cover.out
+- $HOME/gopath/bin/goveralls -coverprofile=cover.out -service=travis-ci
 after_success:
-- if [ "$TRAVIS_BRANCH" == "master" ]; then docker login -e $DOCKER_EMAIL -u $DOCKER_USER
-  -p $DOCKER_PASS ; docker push wrouesnel/postgres_exporter ; fi
+- docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
+# Push a tagged build if a tag is found.
+- if [ ! -z "$TRAVIS_TAG" ]; then 
+    docker tag wrouesnel/postgres_exporter:latest wrouesnel/postgres_exporter:$TRAVIS_TAG ;
+    docker push wrouesnel/postgres_exporter:$TRAVIS_TAG ;
+  fi
+# Push a latest version
+- if [ "$TRAVIS_BRANCH" == "master" ]; then docker push wrouesnel/postgres_exporter ; fi
 env:
   global:
   - secure: RfoWQj5tEB/t3XL2tqJW7u7Qscpz1QBOfF9lMFpB4kAUMTtZU0zBbXfMo1JheGoJQQxD/7NLRHhbUWPT2489o3KKpRTQ7RHn3k8n5U7opH01bWX0+l/EPVmhlsKjSDSLGgmxz80j3I6C8ZV3qDUijSx7r90QUNHGbZtV7g+KtoUTpRV0zir/heK6qq9LHWNHbNsJyHK8qHmd6g1UzWIBaZPJ6a/n/rO2jq4uS1JR0VlIJPRF11HOLH8IjFQvVYpN7YbEslxyNsfQJUSP/7CghSLLVWPSATEjMm8a5GJVLc564+nYghm484psEtiMXkZ3n6ie7AT8aJrKfexWrwh2aCc+cK4PiyXrf4euZehZNYogmFCqWzd1LJKcN2uIkpBSuZQDm3e6c4qkkWGpx+RdFWtAMG8IgZLDbcuryxFNzMwHc2CJ009s9Zsa+g7D57csyR5LCZ8YtNGI3g8FmhwpCKvYkfKa9aijUEWyJMyT4Vhd/w7btMTuwYHgUQ85k4ov4Xjz5SNpAGgemig5G5w7PJj4NhGvIBz9weL154x/BFVjHOZZ6Y/bWgJIPoW1KM15x5K8QylWYEBUHtwiyVyXOxHqt6MOX1vYo1L37jMK88IErrfh/VmlxEhtN9wOghk8IudMfFwQtjIwiWlJf218wxMIzUjoyb5/25tU9f2OJrg=
   - secure: WP96T7yshE03XsPVc9ICbwZXZ6nVsQDCQ9NGKnIEQa4T1Swu5uLVzxjGeowHPykKbKphQmT8inMniBxB48OLc3VVqNxVI+00ppLPEf7n79w2wVbwFOEa6TiOdws+0wOphkeSYc0L+o2aSfoMKJHF+rVW9tmM2tszVjofYHhdWjqloc2pqsfOnqbR7icfpmzMWKyezIE20YOIBsiKZJTKXiZ1SaG9ExkNwuZ7L+HRF1yeI0OdAM4VfEzBK1Gwicy2BtrbyHnl4zgcSoIBmuzo+pNuvqyGmBn3C221M6ki7NoDJDfW5brcvDmiMODWGnka7iW0nt5aUbVtURM8BhWZR0uINo30aYjr4j39UBq8y+mqYV0dp/dMEmy2fa1mogr+DuHUNVSg59Au45AZeom8N6FT03nlg+RcG/tV1skvP/mn9n9CKsyfvC4Rf3jp4+LTiJ9JIch74MecRYVwlpKM+i8s6uDftt3mvxeIYdK+NEMcfwKMv8KTwuxRo/3KRhif7z2cOE+oMbT5POWO19bfboRPCs4xiMTcqpx8dJVs41SacY52PPgjFSnyVrKvzAyjn6mePjLhpoPZueHZuJYPNa9QC8JcASMlCI7lf2Eq+2Dmp2JxmndkRs/cIfHgmO4gtiNM7Vb/rlML1D/8LYPWU/Rdp82/yDffC0ugMNovXt0=
   - secure: RRQH4Tr94OblZoqls50BIoyK1OvK9fALs4pAq1Uk5dksY1NWnomheQzOaHzbVfMfXc4zXAzppZIqxUDGn8GiSLbtJL6pnxsxYNGoCGdS8lMjjKWXbCAs8TBJobi3krOOjqgbhOWTpyluTEShnBcg7CjrRQUa/ChS3uE5kl21/4eIl9Be6Q08Eqm3p1yvMAyAgIL6Y6oPAAtBa6zIsi2MSNlryz3RKHJO7AheilppYx3E8B03A+a/oqvTTcw6w/RwBYxB8MYfSLC0jSssZz5pUSX/byUaklGFhQLnKAzJyhrMOvRyMVcO4PHaLgVi1eAKQz6eLQh7uEiIqKh19cuvTbZHDgu8zMpLDTxOW9U95e4kbjOZ5pWZ7E5QTrb24RZIt42JGbmme7PWOvy3zNbWHwfwiOF1qwYwISUcj2KFCpes8mDGt6iv46LfdlU0uoZdZu3MAiTiW0u2SD5hIeFq4XYesPtkS/TKFoAbB5Tu1qbxdmYu5NqmfvmxsmeNEm4inFJ5ap3fRRCVo668Z6qRMuQ1URcEfOz8iEftP9CnwSOXRuiuMo+W9GgckRuDZcPyQMCftq8+PhB+SjK57zrAd4Kxqf6kVHV16tcWqmFjfJJUFqmL+gpjT/VMEVDY2FOnbOARjkeLTjVC4dADBjxfJ6wmlLrfHdUm4GinbaHq0iA=
 deploy:
+  skip_cleanup: true
   provider: releases
   api_key:
     secure: rwlge/Rs3wnWyfKRhD9fd5GviVe0foYUp20DY3AjKdDjhtwScA1EeR9QHOkB3raze52en0+KkpqlLCWbt3q4CRT7+ku1DNKhd6VWALdTZ1RPJYvNlU6CKJdRnWUJsECmSBsShXlbiYR8axqNVedzFPFGKzS9gYlFN6rr7pez/JZhxqucopZ6I+TkRHMELrFXyQK7/Y2bNRCLC4a+rGsjKeLLtYXbRXCmS0G4BSJEBRk7d69fIRzBApCMfrcLftgHzPuPth616yyUusQSCQYvaZ5tlwrPP8/E0wG3SVJVeDCMuDOSBZ9M6vNzR8W8VR/hxQamegn1OQgC5kNOaLZCTcJ5xguRouqb+FNFBqrd/Zi6vESo7RiVLULawzwxkh9sIPa3WZYDb3VK/Z/cpggUeR7wAu0S5ZYEvJHRefIZpqofZEHzDE3Blqp5yErz05e/zmjpd6HHK3f/UHmRRYfbulkvGT3aL/dlq5GcFvuxVC/vTL2VPvg9cGbqtf7PakC5IhoHpDs35tOyLxifOBLHvkwtGSxEfsCohIG8Hz2XFD83EsxgOiKSXVPLNd6yxjdqZj7OeAKFFU3bzGndnRbDIXaf987IN1imgUtP6wegfImoRStqxN4gEwwIMFsZCF86Ug4eLhlajLbWhudriDxDPBM/F9950aVxLwmWh9l5cRI=

Dockerfile

@@ -0,0 +1,7 @@
+FROM scratch
+
+COPY postgres_exporter /postgres_exporter
+
+EXPOSE 9187
+
+ENTRYPOINT [ "/postgres_exporter" ]

Makefile

@@ -1,42 +1,83 @@
 
-GO_SRC := $(shell find -type f -name "*.go")
+COVERDIR = .coverage
+TOOLDIR = tools
+
+GO_SRC := $(shell find . -name '*.go' ! -path '*/vendor/*' ! -path 'tools/*' )
+GO_DIRS := $(shell find . -type d -name '*.go' ! -path '*/vendor/*' ! -path 'tools/*' )
+GO_PKGS := $(shell go list ./... | grep -v '/vendor/')
 
 CONTAINER_NAME ?= wrouesnel/postgres_exporter:latest
+VERSION ?= $(shell git describe --dirty)
+
+CONCURRENT_LINTERS ?= $(shell cat /proc/cpuinfo | grep processor | wc -l)
+LINTER_DEADLINE ?= 30s
+
+export PATH := $(TOOLDIR)/bin:$(PATH)
+SHELL := env PATH=$(PATH) /bin/bash
 
-all: vet test postgres_exporter
+all: style lint test postgres_exporter
+
+# Cross compilation (e.g. if you are on a Mac)
+cross: docker-build docker
 
 # Simple go build
 postgres_exporter: $(GO_SRC)
-	CGO_ENABLED=0 GOOS=linux go build -a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git rev-parse HEAD)" -o postgres_exporter .
+	CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=$(VERSION)" -o postgres_exporter .
+
+postgres_exporter_integration_test: $(GO_SRC)
+	CGO_ENABLED=0 go test -c -tags integration \
+	    -a -ldflags "-extldflags '-static' -X main.Version=$(VERSION)" -o postgres_exporter_integration_test -cover -covermode count .
 
 # Take a go build and turn it into a minimal container
 docker: postgres_exporter
-	tar -cf - postgres_exporter | docker import --change "EXPOSE 9113" \
-			--change 'ENTRYPOINT [ "/postgres_exporter" ]' \
-			- $(CONTAINER_NAME)
+	docker build -t $(CONTAINER_NAME) .
+
+style: tools
+	gometalinter --disable-all --enable=gofmt --vendor
 
-vet:
-	go vet .
+lint: tools
+	@echo Using $(CONCURRENT_LINTERS) processes
+	gometalinter -j $(CONCURRENT_LINTERS) --deadline=$(LINTER_DEADLINE) --disable=gotype --disable=gocyclo $(GO_DIRS)
 
-test:
-	go test -v .
+fmt: tools
+	gofmt -s -w $(GO_SRC)
 
-test-integration:
-	tests/test-smoke
+test: tools
+	@mkdir -p $(COVERDIR)
+	@rm -f $(COVERDIR)/*
+	for pkg in $(GO_PKGS) ; do \
+		go test -v -covermode count -coverprofile=$(COVERDIR)/$$(echo $$pkg | tr '/' '-').out $$pkg ; \
+	done
+	gocovmerge $(shell find $(COVERDIR) -name '*.out') > cover.test.out
 
 rpm: postgres_exporter
 	@echo ">> building rpm"
 	@rpmbuild -bb --define "_sourcedir $(CURDIR)" --define "_rpmdir $(CURDIR)" --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" postgres_exporter.spec
 
-# Do a self-contained docker build - we pull the official upstream container,
-# then template out a dockerfile which builds the real image.
-docker-build: postgres_exporter
+test-integration: postgres_exporter postgres_exporter_integration_test
+	tests/test-smoke "$(shell pwd)/postgres_exporter" "$(shell pwd)/postgres_exporter_integration_test_script $(shell pwd)/postgres_exporter_integration_test $(shell pwd)/cover.integration.out"
+
+cover.out: tools
+	gocovmerge cover.*.out > cover.out
+
+# Do a self-contained docker build - we pull the official upstream container
+# and do a self-contained build.
+docker-build:
 	docker run -v $(shell pwd):/go/src/github.com/wrouesnel/postgres_exporter \
+	    -v $(shell pwd):/real_src \
+	    -e SHELL_UID=$(shell id -u) -e SHELL_GID=$(shell id -g) \
 	    -w /go/src/github.com/wrouesnel/postgres_exporter \
-		golang:1.6-wheezy \
-		/bin/bash -c "make >&2 && tar -cf - ./postgres_exporter" | \
-		docker import --change "EXPOSE 9113" \
-			--change 'ENTRYPOINT [ "/postgres_exporter" ]' \
-			- $(CONTAINER_NAME)
+		golang:1.8-wheezy \
+		/bin/bash -c "make >&2 && chown $$SHELL_UID:$$SHELL_GID ./postgres_exporter"
+	docker build -t $(CONTAINER_NAME) .
+
+push:
+	docker push $(CONTAINER_NAME)
+
+tools:
+	$(MAKE) -C $(TOOLDIR)
+
+clean:
+	rm -f postgres_exporter postgres_exporter_integration_test
 
-.PHONY: docker-build docker test vet
+.PHONY: tools docker-build docker lint fmt test vet push cross clean

README.md

@@ -1,8 +1,10 @@
 [![Build Status](https://travis-ci.org/wrouesnel/postgres_exporter.svg?branch=master)](https://travis-ci.org/wrouesnel/postgres_exporter)
+[![Coverage Status](https://coveralls.io/repos/github/wrouesnel/postgres_exporter/badge.svg?branch=master)](https://coveralls.io/github/wrouesnel/postgres_exporter?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/wrouesnel/postgres_exporter)](https://goreportcard.com/report/github.com/wrouesnel/postgres_exporter)
 
-# PostgresSQL Server Exporter
+# PostgreSQL Server Exporter
 
-Prometheus exporter for PostgresSQL server metrics.
+Prometheus exporter for PostgreSQL server metrics.
 Supported Postgres versions: 9.1 and up.
 
 ## Quick Start
@@ -11,7 +13,7 @@ This package is available for Docker:
 # Start an example database
 docker run --net=host -it --rm -e POSTGRES_PASSWORD=password postgres
 # Connect to it
-docker run -e DATA_SOURCE_NAME="postgresql://postgres:password@localhost:5432/?sslmode=disable" -p 9113:9113 wrouesnel/postgres_exporter
+docker run -e DATA_SOURCE_NAME="postgresql://postgres:password@localhost:5432/?sslmode=disable" -p 9187:9187 wrouesnel/postgres_exporter
 ```
 
 ## Building and running
@@ -34,14 +36,15 @@ Package vendoring is handled with [`govendor`](https://github.com/kardianos/gove
 
 ### Flags
 
-Name               | Description
--------------------|------------
-web.listen-address | Address to listen on for web interface and telemetry.
-web.telemetry-path | Path under which to expose metrics.
+* `web.listen-address` 
+  Address to listen on for web interface and telemetry.
+
+* `web.telemetry-path`
+  Path under which to expose metrics.
 
 ### Setting the Postgres server's data source name
 
-The PostgresSQL server's [data source name](http://en.wikipedia.org/wiki/Data_source_name)
+The PostgreSQL server's [data source name](http://en.wikipedia.org/wiki/Data_source_name)
 must be set via the `DATA_SOURCE_NAME` environment variable.
 
 For running it locally on a default Debian/Ubuntu install, this will work (transpose to init script as appropriate):
@@ -65,6 +68,10 @@ for l in StringIO(x):
 Adjust the value of the resultant prometheus value type appropriately. This helps build
 rich self-documenting metrics for the exporter.
 
+### Adding new metrics via a config file
+
+The -extend.query-path command-line argument specifies a YAML file containing additional queries to run.
+Some examples are provided in [queries.yaml](queries.yaml).
 
 ### Running as non-superuser
 
@@ -74,6 +81,8 @@ To be able to collect metrics from pg_stat_activity and pg_stat_replication as n
 CREATE USER postgres_exporter PASSWORD 'password';
 ALTER USER postgres_exporter SET SEARCH_PATH TO postgres_exporter,pg_catalog;
 
+-- If deploying as non-superuser (for example in AWS RDS)
+-- GRANT postgres_exporter TO :MASTER_USER;
 CREATE SCHEMA postgres_exporter AUTHORIZATION postgres_exporter;
 
 CREATE FUNCTION postgres_exporter.f_select_pg_stat_activity()
@@ -103,3 +112,9 @@ AS
 GRANT SELECT ON postgres_exporter.pg_stat_replication TO postgres_exporter;
 GRANT SELECT ON postgres_exporter.pg_stat_activity TO postgres_exporter;
 ```
+
+> **NOTE**
+> <br />Remember to use `postgres` database name in the connection string:
+> ```
+> DATA_SOURCE_NAME=postgresql://postgres_exporter:password@localhost:5432/postgres?sslmode=disable
+> ```

pg_setting.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"database/sql"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/common/log"
+)
+
+// Query the pg_settings view containing runtime variables
+func querySettings(ch chan<- prometheus.Metric, db *sql.DB) error {
+	log.Debugln("Querying pg_setting view")
+
+	// pg_settings docs: https://www.postgresql.org/docs/current/static/view-pg-settings.html
+	//
+	// NOTE: If you add more vartypes here, you must update the supported
+	// types in normaliseUnit() below
+	query := "SELECT name, setting, COALESCE(unit, ''), short_desc, vartype FROM pg_settings WHERE vartype IN ('bool', 'integer', 'real');"
+
+	rows, err := db.Query(query)
+	if err != nil {
+		return errors.New(fmt.Sprintln("Error running query on database: ", namespace, err))
+	}
+	defer rows.Close() // nolint: errcheck
+
+	for rows.Next() {
+		s := &pgSetting{}
+		err = rows.Scan(&s.name, &s.setting, &s.unit, &s.shortDesc, &s.vartype)
+		if err != nil {
+			return errors.New(fmt.Sprintln("Error retrieving rows:", namespace, err))
+		}
+
+		ch <- s.metric()
+	}
+
+	return nil
+}
+
+// pgSetting is represents a PostgreSQL runtime variable as returned by the
+// pg_settings view.
+type pgSetting struct {
+	name, setting, unit, shortDesc, vartype string
+}
+
+func (s *pgSetting) metric() prometheus.Metric {
+	var (
+		err       error
+		name      = strings.Replace(s.name, ".", "_", -1)
+		unit      = s.unit // nolint: ineffassign
+		shortDesc = s.shortDesc
+		subsystem = "settings"
+		val       float64
+	)
+
+	switch s.vartype {
+	case "bool":
+		if s.setting == "on" {
+			val = 1
+		}
+	case "integer", "real":
+		if val, unit, err = s.normaliseUnit(); err != nil {
+			// Panic, since we should recognise all units
+			// and don't want to silently exlude metrics
+			panic(err)
+		}
+
+		if len(unit) > 0 {
+			name = fmt.Sprintf("%s_%s", name, unit)
+			shortDesc = fmt.Sprintf("%s [Units converted to %s.]", shortDesc, unit)
+		}
+	default:
+		// Panic because we got a type we didn't ask for
+		panic(fmt.Sprintf("Unsupported vartype %q", s.vartype))
+	}
+
+	desc := newDesc(subsystem, name, shortDesc)
+	return prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, val)
+}
+
+func (s *pgSetting) normaliseUnit() (val float64, unit string, err error) {
+	val, err = strconv.ParseFloat(s.setting, 64)
+	if err != nil {
+		return val, unit, fmt.Errorf("Error converting setting %q value %q to float: %s", s.name, s.setting, err)
+	}
+
+	// Units defined in: https://www.postgresql.org/docs/current/static/config-setting.html
+	switch s.unit {
+	case "":
+		return
+	case "ms", "s", "min", "h", "d":
+		unit = "seconds"
+	case "kB", "MB", "GB", "TB", "8kB", "16MB":
+		unit = "bytes"
+	default:
+		err = fmt.Errorf("Unknown unit for runtime variable: %q", s.unit)
+		return
+	}
+
+	// -1 is special, don't modify the value
+	if val == -1 {
+		return
+	}
+
+	switch s.unit {
+	case "ms":
+		val /= 1000
+	case "min":
+		val *= 60
+	case "h":
+		val *= 60 * 60
+	case "d":
+		val *= 60 * 60 * 24
+	case "kB":
+		val *= math.Pow(2, 10)
+	case "MB":
+		val *= math.Pow(2, 20)
+	case "GB":
+		val *= math.Pow(2, 30)
+	case "TB":
+		val *= math.Pow(2, 40)
+	case "8kB":
+		val *= math.Pow(2, 13)
+	case "16MB":
+		val *= math.Pow(2, 24)
+	}
+
+	return
+}