Merge branch 'master' into master

Author: wrouesnel

.gitignore

@@ -6,3 +6,6 @@ postgres_exporter_integration_test
 *-stamp
 .idea
 *.iml
+cover.out
+cover.*.out
+

.travis.yml

@@ -3,9 +3,10 @@ services:
 - docker
 language: go
 go:
-- '1.7'
+- '1.7.5'
 # Make sure we have p2 and the postgres client.
 before_install:
+- go get -v github.com/mattn/goveralls
 - sudo wget -O /usr/local/bin/p2 https://github.com/wrouesnel/p2cli/releases/download/r4/p2 &&
   sudo chmod +x /usr/local/bin/p2
 - sudo wget -O /usr/local/bin/docker-compose https://github.com/docker/compose/releases/download/1.9.0-rc4/docker-compose-Linux-x86_64 &&
@@ -16,6 +17,8 @@ script:
 - make all
 - make docker
 - make test-integration
+- ./concatenate_coverage cover.out cover.test.out cover.integration.out
+- $HOME/gopath/bin/goveralls -coverprofile=cover.out -service=travis-ci
 after_success:
 - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
 # Push a tagged build if a tag is found.

Makefile

@@ -1,34 +1,46 @@
 
-GO_SRC := $(shell find . -type f -name "*.go")
+GO_SRC := $(shell find -type f -name '*.go' ! -path '*/vendor/*')
 
 CONTAINER_NAME ?= wrouesnel/postgres_exporter:latest
+VERSION ?= $(shell git describe --dirty)
 
 all: vet test postgres_exporter
 
+# Cross compilation (e.g. if you are on a Mac)
+cross: docker-build docker
+
 # Simple go build
 postgres_exporter: $(GO_SRC)
-	CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=$(shell git describe --dirty)" -o postgres_exporter .
+	CGO_ENABLED=0 go build -a -ldflags "-extldflags '-static' -X main.Version=$(VERSION)" -o postgres_exporter .
 
 postgres_exporter_integration_test: $(GO_SRC)
 	CGO_ENABLED=0 go test -c -tags integration \
-	    -a -ldflags "-extldflags '-static' -X main.Version=git:$(shell git describe --dirty)" -o postgres_exporter_integration_test .
+	    -a -ldflags "-extldflags '-static' -X main.Version=$(VERSION)" -o postgres_exporter_integration_test -cover -covermode count .
 
 # Take a go build and turn it into a minimal container
 docker: postgres_exporter
 	docker build -t $(CONTAINER_NAME) .
 
 vet:
-	go vet .
+	go vet
+
+# Check code conforms to go fmt
+style:
+	! gofmt -s -l $(GO_SRC) 2>&1 | read 2>/dev/null
+
+# Format the code
+fmt:
+	gofmt -s -w $(GO_SRC)
 
 test:
-	go test -v -cover .
+	go test -v -covermode count -coverprofile=cover.test.out
 
 test-integration: postgres_exporter postgres_exporter_integration_test
-	tests/test-smoke ./postgres_exporter ./postgres_exporter_integration_test
+	tests/test-smoke "$(shell pwd)/postgres_exporter" "$(shell pwd)/postgres_exporter_integration_test_script $(shell pwd)/postgres_exporter_integration_test $(shell pwd)/cover.integration.out"
 
 # Do a self-contained docker build - we pull the official upstream container
 # and do a self-contained build.
-docker-build: postgres_exporter
+docker-build:
 	docker run -v $(shell pwd):/go/src/github.com/wrouesnel/postgres_exporter \
 	    -v $(shell pwd):/real_src \
 	    -e SHELL_UID=$(shell id -u) -e SHELL_GID=$(shell id -g) \
@@ -37,4 +49,7 @@ docker-build: postgres_exporter
 		/bin/bash -c "make >&2 && chown $$SHELL_UID:$$SHELL_GID ./postgres_exporter"
 	docker build -t $(CONTAINER_NAME) .
 
-.PHONY: docker-build docker test vet
+push:
+	docker push $(CONTAINER_NAME)
+
+.PHONY: docker-build docker test vet push cross

README.md

@@ -1,4 +1,6 @@
 [![Build Status](https://travis-ci.org/wrouesnel/postgres_exporter.svg?branch=master)](https://travis-ci.org/wrouesnel/postgres_exporter)
+[![Coverage Status](https://coveralls.io/repos/github/wrouesnel/postgres_exporter/badge.svg?branch=master)](https://coveralls.io/github/wrouesnel/postgres_exporter?branch=master)
+[![Go Report Card](https://goreportcard.com/badge/github.com/wrouesnel/postgres_exporter)](https://goreportcard.com/report/github.com/wrouesnel/postgres_exporter)
 
 # PostgresSQL Server Exporter
 
@@ -108,3 +110,9 @@ AS
 GRANT SELECT ON postgres_exporter.pg_stat_replication TO postgres_exporter;
 GRANT SELECT ON postgres_exporter.pg_stat_activity TO postgres_exporter;
 ```
+
+> **NOTE**
+> <br />Remember to use `postgres` database name in the connection string:
+> ```
+> DATA_SOURCE_NAME=postgresql://postgres_exporter:password@localhost:5432/postgres?sslmode=disable
+> ```

concatenate_coverage

@@ -0,0 +1,12 @@
+#!/bin/bash
+# Concatenate a list of coverage reports
+# Usage: concatenate_coverage <out> [[test] ...]
+
+output_file=$1
+shift
+cat $1 > $output_file
+shift
+
+for f in $@ ; do
+    tail -n +2 $f >> $output_file
+done

postgres_exporter.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"math"
 	"net/http"
+	"net/url"
 	"os"
 	"regexp"
 	"strconv"
@@ -24,6 +25,8 @@ import (
 
 var Version string = "0.0.1"
 
+var db *sql.DB = nil
+
 var (
 	listenAddress = flag.String(
 		"web.listen-address", ":9187",
@@ -146,7 +149,7 @@ type MetricMap struct {
 
 // Metric descriptors for dynamically created metrics.
 var variableMaps = map[string]map[string]ColumnMapping{
-	"pg_runtime_variable": map[string]ColumnMapping{
+	"pg_runtime_variable": {
 		"max_connections":                {GAUGE, "Sets the maximum number of concurrent connections.", nil, nil},
 		"max_files_per_process":          {GAUGE, "Sets the maximum number of simultaneously open files for each server process.", nil, nil},
 		"max_function_args":              {GAUGE, "Shows the maximum number of function arguments.", nil, nil},
@@ -182,7 +185,7 @@ func dumpMaps() {
 }
 
 var metricMaps = map[string]map[string]ColumnMapping{
-	"pg_stat_bgwriter": map[string]ColumnMapping{
+	"pg_stat_bgwriter": {
 		"checkpoints_timed":     {COUNTER, "Number of scheduled checkpoints that have been performed", nil, nil},
 		"checkpoints_req":       {COUNTER, "Number of requested checkpoints that have been performed", nil, nil},
 		"checkpoint_write_time": {COUNTER, "Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds", nil, nil},
@@ -195,7 +198,7 @@ var metricMaps = map[string]map[string]ColumnMapping{
 		"buffers_alloc":         {COUNTER, "Number of buffers allocated", nil, nil},
 		"stats_reset":           {COUNTER, "Time at which these statistics were last reset", nil, nil},
 	},
-	"pg_stat_database": map[string]ColumnMapping{
+	"pg_stat_database": {
 		"datid":          {LABEL, "OID of a database", nil, nil},
 		"datname":        {LABEL, "Name of this database", nil, nil},
 		"numbackends":    {GAUGE, "Number of backends currently connected to this database. This is the only column in this view that returns a value reflecting current state; all other columns return the accumulated values since the last reset.", nil, nil},
@@ -216,7 +219,7 @@ var metricMaps = map[string]map[string]ColumnMapping{
 		"blk_write_time": {COUNTER, "Time spent writing data file blocks by backends in this database, in milliseconds", nil, nil},
 		"stats_reset":    {COUNTER, "Time at which these statistics were last reset", nil, nil},
 	},
-	"pg_stat_database_conflicts": map[string]ColumnMapping{
+	"pg_stat_database_conflicts": {
 		"datid":            {LABEL, "OID of a database", nil, nil},
 		"datname":          {LABEL, "Name of this database", nil, nil},
 		"confl_tablespace": {COUNTER, "Number of queries in this database that have been canceled due to dropped tablespaces", nil, nil},
@@ -225,12 +228,12 @@ var metricMaps = map[string]map[string]ColumnMapping{
 		"confl_bufferpin":  {COUNTER, "Number of queries in this database that have been canceled due to pinned buffers", nil, nil},
 		"confl_deadlock":   {COUNTER, "Number of queries in this database that have been canceled due to deadlocks", nil, nil},
 	},
-	"pg_locks": map[string]ColumnMapping{
+	"pg_locks": {
 		"datname": {LABEL, "Name of this database", nil, nil},
 		"mode":    {LABEL, "Type of Lock", nil, nil},
 		"count":   {GAUGE, "Number of locks", nil, nil},
 	},
-	"pg_stat_replication": map[string]ColumnMapping{
+	"pg_stat_replication": {
 		"procpid":          {DISCARD, "Process ID of a WAL sender process", nil, semver.MustParseRange("<9.2.0")},
 		"pid":              {DISCARD, "Process ID of a WAL sender process", nil, semver.MustParseRange(">=9.2.0")},
 		"usesysid":         {DISCARD, "OID of the user logged into this WAL sender process", nil, nil},
@@ -262,7 +265,7 @@ var metricMaps = map[string]map[string]ColumnMapping{
 		"pg_xlog_location_diff":    {GAUGE, "Lag in bytes between master and slave", nil, semver.MustParseRange(">=9.2.0")},
 		"confirmed_flush_lsn":      {DISCARD, "LSN position a consumer of a slot has confirmed flushing the data received", nil, nil},
 	},
-	"pg_stat_activity": map[string]ColumnMapping{
+	"pg_stat_activity": {
 		"datname":         {LABEL, "Name of this database", nil, nil},
 		"state":           {LABEL, "connection state", nil, semver.MustParseRange(">=9.2.0")},
 		"count":           {GAUGE, "number of connections in this state", nil, nil},
@@ -282,7 +285,7 @@ type OverrideQuery struct {
 // Overriding queries for namespaces above.
 // TODO: validate this is a closed set in tests, and there are no overlaps
 var queryOverrides = map[string][]OverrideQuery{
-	"pg_locks": []OverrideQuery{
+	"pg_locks": {
 		{
 			semver.MustParseRange(">0.0.0"),
 			`SELECT pg_database.datname,tmp.mode,COALESCE(count,0) as count
@@ -306,7 +309,7 @@ var queryOverrides = map[string][]OverrideQuery{
 		},
 	},
 
-	"pg_stat_replication": []OverrideQuery{
+	"pg_stat_replication": {
 		{
 			semver.MustParseRange(">=9.2.0"),
 			`
@@ -326,7 +329,7 @@ var queryOverrides = map[string][]OverrideQuery{
 		},
 	},
 
-	"pg_stat_activity": []OverrideQuery{
+	"pg_stat_activity": {
 		// This query only works
 		{
 			semver.MustParseRange(">=9.2.0"),
@@ -967,6 +970,24 @@ func (e *Exporter) checkMapVersions(ch chan<- prometheus.Metric, db *sql.DB) err
 	return nil
 }
 
+func getDB(conn string) (*sql.DB, error) {
+	if db == nil {
+		d, err := sql.Open("postgres", conn)
+		if err != nil {
+			return nil, err
+		}
+		err = d.Ping()
+		if err != nil {
+			return nil, err
+		}
+		d.SetMaxOpenConns(1)
+		d.SetMaxIdleConns(1)
+		db = d
+	}
+
+	return db, nil
+}
+
 func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
 	defer func(begun time.Time) {
 		e.duration.Set(time.Since(begun).Seconds())
@@ -975,13 +996,17 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) {
 	e.error.Set(0)
 	e.totalScrapes.Inc()
 
-	db, err := sql.Open("postgres", e.dsn)
+	db, err := getDB(e.dsn)
 	if err != nil {
-		log.Infoln("Error opening connection to database:", err)
+		loggableDsn := "could not parse DATA_SOURCE_NAME"
+		if pDsn, pErr := url.Parse(e.dsn); pErr != nil {
+			pDsn.User = url.UserPassword(pDsn.User.Username(), "xxx")
+			loggableDsn = pDsn.String()
+		}
+		log.Infof("Error opening connection to database (%s): %s", loggableDsn, err)
 		e.error.Set(1)
 		return
 	}
-	defer db.Close()
 
 	// Check if map versions need to be updated
 	if err := e.checkMapVersions(ch, db); err != nil {
@@ -1027,4 +1052,7 @@ func main() {
 
 	log.Infof("Starting Server: %s", *listenAddress)
 	log.Fatal(http.ListenAndServe(*listenAddress, nil))
+	if db != nil {
+		defer db.Close()
+	}
 }

postgres_exporter_integration_test.go

@@ -11,16 +11,16 @@ import (
 
 	. "gopkg.in/check.v1"
 
-	"github.com/prometheus/client_golang/prometheus"
 	"database/sql"
-	_ "github.com/lib/pq"
 	"fmt"
+	_ "github.com/lib/pq"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // Hook up gocheck into the "go test" runner.
 func Test(t *testing.T) { TestingT(t) }
 
-type IntegrationSuite struct{
+type IntegrationSuite struct {
 	e *Exporter
 }
 
@@ -43,7 +43,8 @@ func (s *IntegrationSuite) TestAllNamespacesReturnResults(c *C) {
 	// Setup a dummy channel to consume metrics
 	ch := make(chan prometheus.Metric, 100)
 	go func() {
-		for _ = range ch {}
+		for range ch {
+		}
 	}()
 
 	// Open a database connection
@@ -65,7 +66,6 @@ func (s *IntegrationSuite) TestAllNamespacesReturnResults(c *C) {
 		}
 	}
 
-
 	// This should never happen in our test cases.
 	errMap := queryNamespaceMappings(ch, db, s.e.metricMap, s.e.queryOverrides)
 	if !c.Check(len(errMap), Equals, 0) {
@@ -74,4 +74,4 @@ func (s *IntegrationSuite) TestAllNamespacesReturnResults(c *C) {
 			fmt.Println(namespace, ":", err)
 		}
 	}
-}
+}

postgres_exporter_integration_test_script

@@ -0,0 +1,15 @@
+#!/bin/bash
+# This script wraps the integration test binary so it produces concatenated
+# test output.
+
+test_binary=$1
+shift
+output_cov=$1
+shift
+
+echo "mode: count" > $output_cov
+
+test_cov=$(mktemp)
+$test_binary -test.coverprofile=$test_cov $@
+tail -n +2 $test_cov >> $output_cov
+rm -f $test_cov

postgres_exporter_test.go

@@ -3,16 +3,16 @@
 package main
 
 import (
-	"testing"
 	. "gopkg.in/check.v1"
+	"testing"
 
 	"github.com/blang/semver"
 )
 
 // Hook up gocheck into the "go test" runner.
 func Test(t *testing.T) { TestingT(t) }
 
-type FunctionalSuite struct{
+type FunctionalSuite struct {
 	e *Exporter
 }
 
@@ -24,9 +24,9 @@ func (s *FunctionalSuite) SetUpSuite(c *C) {
 
 func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) {
 	testMetricMap := map[string]map[string]ColumnMapping{
-		"test_namespace" : map[string]ColumnMapping{
-			"metric_which_stays" : {COUNTER, "This metric should not be eliminated", nil, nil},
-			"metric_which_discards" : {COUNTER, "This metric should be forced to DISCARD", nil, nil},
+		"test_namespace": {
+			"metric_which_stays":    {COUNTER, "This metric should not be eliminated", nil, nil},
+			"metric_which_discards": {COUNTER, "This metric should be forced to DISCARD", nil, nil},
 		},
 	}
 
@@ -84,4 +84,4 @@ func (s *FunctionalSuite) TestSemanticVersionColumnDiscard(c *C) {
 			false,
 		)
 	}
-}
+}

tests/test-smoke

@@ -81,6 +81,7 @@ smoketest_postgres() {
     wait_for_postgres localhost 55432
 
     DATA_SOURCE_NAME="postgresql://postgres:$POSTGRES_PASSWORD@localhost:55432/?sslmode=disable" $test_binary --log.level=debug || exit $?
+
 #    exporter_pid=$!
 #    trap "docker logs $CONTAINER_NAME ; docker kill $CONTAINER_NAME ; docker rm $CONTAINER_NAME ; kill $exporter_pid; exit 1" EXIT INT TERM
 #    wait_for_exporter