Add Sentry

Author: Fly.io CI

cmd/postgres-operator/main.go

@@ -47,6 +47,7 @@ import (
 	"github.com/percona/percona-postgresql-operator/internal/initialize"
 	"github.com/percona/percona-postgresql-operator/internal/logging"
 	"github.com/percona/percona-postgresql-operator/internal/naming"
+	"github.com/percona/percona-postgresql-operator/internal/sentry"
 	"github.com/percona/percona-postgresql-operator/internal/upgradecheck"
 	perconaController "github.com/percona/percona-postgresql-operator/percona/controller"
 	"github.com/percona/percona-postgresql-operator/percona/controller/pgbackup"
@@ -74,6 +75,13 @@ func main() {
 	assertNoError(err)
 	defer otelFlush()
 
+	// Initialize Sentry if DSN is provided
+	if dsn := os.Getenv("SENTRY_DSN"); dsn != "" {
+		err := sentry.Initialize(dsn)
+		assertNoError(err)
+		defer sentry.Flush()
+	}
+
 	opts := zap.Options{
 		Encoder: getLogEncoder(),
 		Level:   getLogLevel(),

config/manager/default/manager.yaml

@@ -35,6 +35,18 @@ spec:
             value: INFO
           - name: DISABLE_TELEMETRY
             value: "false"
+          - name: SENTRY_DSN
+            valueFrom:
+              secretKeyRef:
+                name: percona-postgresql-operator-sentry
+                key: dsn
+                optional: true
+          - name: SENTRY_ENVIRONMENT
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: SENTRY_DEBUG
+            value: "false"
         ports:
           - containerPort: 8080
             name: metrics

config/manager/default/sentry-secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: percona-postgresql-operator-sentry
+  namespace: default
+type: Opaque
+stringData:
+  dsn: "" # Replace with your Sentry DSN

docs/content/monitoring.md

@@ -0,0 +1,47 @@
+# Error Reporting with Sentry
+
+The Percona PostgreSQL Operator supports error reporting through Sentry. This helps track and monitor errors that occur during the operation of your PostgreSQL clusters.
+
+## Configuration
+
+To enable Sentry error reporting:
+
+1. Create a Sentry project and obtain your DSN (Data Source Name)
+
+2. Create a secret containing your Sentry DSN:
+   ```bash
+   kubectl create secret generic percona-postgresql-operator-sentry \
+     --from-literal=dsn=your-sentry-dsn
+   ```
+
+3. The operator deployment automatically picks up the Sentry configuration from the secret.
+
+## Environment Variables
+
+The following environment variables can be configured:
+
+- `SENTRY_DSN`: The Sentry DSN (configured via secret)
+- `SENTRY_ENVIRONMENT`: The environment name (defaults to namespace name)
+- `SENTRY_DEBUG`: Enable debug mode for Sentry (default: "false")
+
+## Error Tracking
+
+The operator reports various types of errors to Sentry:
+
+- Reconciliation errors
+- Unexpected errors during cluster operations
+- Panics (which are captured and reported before re-panicking)
+
+Each error report includes relevant context such as:
+- Namespace and name of the affected PostgreSQL cluster
+- Operation being performed
+- Controller name
+- Additional error context
+
+## Disabling Error Reporting
+
+To disable Sentry error reporting, simply delete the Sentry secret:
+
+```bash
+kubectl delete secret percona-postgresql-operator-sentry
+```

go.mod

@@ -5,6 +5,7 @@ go 1.22.6
 require (
 	github.com/Percona-Lab/percona-version-service v0.0.0-20230404081016-ea25e30cdcbc
 	github.com/evanphx/json-patch/v5 v5.9.0
+	github.com/getsentry/sentry-go v0.27.0
 	github.com/go-logr/logr v1.4.2
 	github.com/go-openapi/errors v0.22.0
 	github.com/go-openapi/runtime v0.28.0

go.sum

@@ -44,6 +44,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
 github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
 github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
+github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
+github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
+github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
+github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
 github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -136,10 +140,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
-github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
-github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8=
@@ -168,6 +172,8 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
 github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
 github.com/pganalyze/pg_query_go/v5 v5.1.0 h1:MlxQqHZnvA3cbRQYyIrjxEjzo560P6MyTgtlaf3pmXg=
 github.com/pganalyze/pg_query_go/v5 v5.1.0/go.mod h1:FsglvxidZsVN+Ltw3Ai6nTgPVcK2BPukH3jCDEqc1Ug=
+github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=

internal/controller/postgrescluster/controller.go

@@ -39,6 +39,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	sentrygo "github.com/getsentry/sentry-go"
 	"github.com/percona/percona-postgresql-operator/internal/config"
 	"github.com/percona/percona-postgresql-operator/internal/controller/runtime"
 	"github.com/percona/percona-postgresql-operator/internal/initialize"
@@ -84,19 +85,35 @@ func (r *Reconciler) Reconcile(
 	ctx context.Context, request reconcile.Request,
 ) (reconcile.Result, error) {
 	ctx, span := r.Tracer.Start(ctx, "Reconcile")
-	log := logging.FromContext(ctx)
 	defer span.End()
 
+	// Add Sentry context
+	hub := sentrygo.GetHubFromContext(ctx)
+	if hub == nil {
+		hub = sentrygo.CurrentHub().Clone()
+	}
+	hub.ConfigureScope(func(scope *sentrygo.Scope) {
+		scope.SetTag("namespace", request.Namespace)
+		scope.SetTag("name", request.Name)
+	})
+	ctx = sentrygo.SetHubOnContext(ctx, hub)
+	defer func() {
+		if err := recover(); err != nil {
+			hub.Recover(err)
+			panic(err)
+		}
+	}()
+
+	log := logging.FromContext(ctx)
+
 	// get the postgrescluster from the cache
 	cluster := &v1beta1.PostgresCluster{}
-	if err := r.Client.Get(ctx, request.NamespacedName, cluster); err != nil {
-		// NotFound cannot be fixed by requeuing so ignore it. During background
-		// deletion, we receive delete events from cluster's dependents after
-		// cluster is deleted.
-		if err = client.IgnoreNotFound(err); err != nil {
-			log.Error(err, "unable to fetch PostgresCluster")
-			span.RecordError(err)
-		}
+	err := r.Client.Get(ctx, request.NamespacedName, cluster)
+	if err = client.IgnoreNotFound(err); err != nil {
+		log.Error(err, "unable to fetch PostgresCluster")
+		span.RecordError(err)
+		// Record unexpected errors
+		hub.CaptureException(err)
 		return runtime.ErrorWithBackoff(err)
 	}
 
@@ -116,8 +133,7 @@ func (r *Reconciler) Reconcile(
 
 	// Keep a copy of cluster prior to any manipulations.
 	before := cluster.DeepCopy()
-	var err error
-	result := reconcile.Result{}
+	var result reconcile.Result
 	defer func() {
 		if !equality.Semantic.DeepEqual(before.Status, cluster.Status) {
 			statusErr := r.Client.Status().Patch(ctx, cluster, client.MergeFrom(before), r.Owner)
@@ -141,7 +157,7 @@ func (r *Reconciler) Reconcile(
 	if deleteResult, deleteErr := r.handleDelete(ctx, cluster); deleteErr != nil {
 		span.RecordError(deleteErr)
 		log.Error(deleteErr, "deleting")
-		return runtime.ErrorWithBackoff(deleteErr)
+		return reconcile.Result{}, errors.Join(err, deleteErr)
 
 	} else if deleteResult != nil {
 		if log := log.V(1); log.Enabled() {
@@ -162,7 +178,7 @@ func (r *Reconciler) Reconcile(
 		// specifically allow reconciliation if the cluster is shutdown to
 		// facilitate upgrades, otherwise return
 		if !initialize.FromPointer(cluster.Spec.Shutdown) {
-			return runtime.ErrorWithBackoff(err)
+			return reconcile.Result{}, err
 		}
 	}
 
@@ -176,7 +192,7 @@ func (r *Reconciler) Reconcile(
 		path := field.NewPath("spec", "standby")
 		err := field.Invalid(path, cluster.Name, "Standby requires a host or repoName to be enabled")
 		r.Recorder.Event(cluster, corev1.EventTypeWarning, "InvalidStandbyConfiguration", err.Error())
-		return runtime.ErrorWithBackoff(err)
+		return reconcile.Result{}, err
 	}
 
 	var (
@@ -226,7 +242,7 @@ func (r *Reconciler) Reconcile(
 
 			ObservedGeneration: cluster.GetGeneration(),
 		})
-		return runtime.ErrorWithBackoff(patchClusterStatus())
+		return reconcile.Result{}, errors.Join(err, patchClusterStatus())
 	} else {
 		meta.RemoveStatusCondition(&cluster.Status.Conditions, v1beta1.PostgresClusterProgressing)
 	}
@@ -269,7 +285,7 @@ func (r *Reconciler) Reconcile(
 		returnEarly, err := r.reconcileDirMoveJobs(ctx, cluster)
 		if err != nil || returnEarly {
 			log.V(1).Info("waiting for directory move jobs", "cluster", cluster.Name, "error", err)
-			return runtime.ErrorWithBackoff(errors.Join(err, patchClusterStatus()))
+			return reconcile.Result{}, errors.Join(err, patchClusterStatus())
 		}
 	}
 	if err == nil {
@@ -325,7 +341,7 @@ func (r *Reconciler) Reconcile(
 		returnEarly, err := r.reconcileDataSource(ctx, cluster, instances, clusterVolumes, rootCA)
 		if err != nil || returnEarly {
 			log.V(1).Info("waiting for data source initialization", "cluster", cluster.Name, "error", err)
-			return runtime.ErrorWithBackoff(errors.Join(err, patchClusterStatus()))
+			return reconcile.Result{}, errors.Join(err, patchClusterStatus())
 		}
 	}
 	if err == nil {
@@ -431,6 +447,12 @@ func (r *Reconciler) Reconcile(
 		"requeue", result.Requeue,
 		"requeueAfter", result.RequeueAfter)
 
+	// If we get here, reconciliation succeeded.
+	// Record any errors that occurred during reconciliation.
+	if err != nil {
+		hub.CaptureException(err)
+	}
+
 	return result, errors.Join(err, patchClusterStatus())
 }
 

internal/sentry/sentry.go

@@ -0,0 +1,89 @@
+package sentry
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/getsentry/sentry-go"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/percona/percona-postgresql-operator/internal/logging"
+)
+
+const (
+	// DefaultTimeout is the default timeout for flushing events to Sentry
+	DefaultTimeout = 2 * time.Second
+)
+
+// Initialize sets up the Sentry client with the provided DSN
+func Initialize(dsn string) error {
+	if dsn == "" {
+		return nil // Sentry is disabled if no DSN is provided
+	}
+
+	err := sentry.Init(sentry.ClientOptions{
+		Dsn:              dsn,
+		EnableTracing:    true,
+		TracesSampleRate: 1.0,
+		Debug:            os.Getenv("SENTRY_DEBUG") == "true",
+		Environment:      os.Getenv("SENTRY_ENVIRONMENT"),
+	})
+
+	if err != nil {
+		return errors.Wrap(err, "failed to initialize Sentry")
+	}
+
+	return nil
+}
+
+// CaptureError reports an error to Sentry with additional context
+func CaptureError(ctx context.Context, err error, tags map[string]string) {
+	if err == nil {
+		return
+	}
+
+	log := logging.FromContext(ctx)
+
+	hub := sentry.GetHubFromContext(ctx)
+	if hub == nil {
+		hub = sentry.CurrentHub()
+	}
+
+	// Add any context from the logger
+	if len(tags) > 0 {
+		hub.ConfigureScope(func(scope *sentry.Scope) {
+			for k, v := range tags {
+				scope.SetTag(k, v)
+			}
+		})
+	}
+
+	hub.CaptureException(err)
+	log.Error(err, "error captured by Sentry")
+}
+
+// RecoverPanic captures panics and reports them to Sentry
+func RecoverPanic() {
+	if err := recover(); err != nil {
+		sentry.CurrentHub().Recover(err)
+		sentry.Flush(DefaultTimeout)
+		panic(err) // Re-panic after reporting
+	}
+}
+
+// WithContext returns a new context with Sentry hub
+func WithContext(ctx context.Context, namespacedName types.NamespacedName) context.Context {
+	hub := sentry.CurrentHub().Clone()
+	hub.ConfigureScope(func(scope *sentry.Scope) {
+		scope.SetTag("namespace", namespacedName.Namespace)
+		scope.SetTag("name", namespacedName.Name)
+	})
+	return sentry.SetHubOnContext(ctx, hub)
+}
+
+// Flush ensures all queued events are sent to Sentry
+func Flush() {
+	sentry.Flush(DefaultTimeout)
+}