Poll backup status from the repo-host rather than the instance pod.

.github/workflows/docker-build.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          fetch-depth: 0  # Needed for git describe
+          fetch-depth: 0 # Needed for git describe
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -54,4 +54,4 @@ jobs:
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-to: type=gha,mode=max

internal/naming/labels.go

@@ -30,6 +30,9 @@ const (
 	// LabelRepoName is used to specify the name of a pgBackRest repository
 	LabelRepoName = labelPrefix + "name"
 
+	// LabelPgbackrestDedicated is used to select the repo-host pod
+	LabelPgbackrestDedicated = labelPrefix + "pgbackrest-dedicated"
+
 	LabelPatroni = labelPrefix + "patroni"
 	LabelRole    = labelPrefix + "role"
 

internal/naming/names.go

@@ -31,6 +31,8 @@ const (
 
 	// ContainerPGBackRestConfig is the name of a container supporting pgBackRest.
 	ContainerPGBackRestConfig = "pgbackrest-config"
+	// ContainerPGBackRest is the name of a container running pgBackRest.
+	ContainerPGBackRest = "pgbackrest"
 
 	// ContainerPGBouncer is the name of a container running PgBouncer.
 	ContainerPGBouncer = "pgbouncer"

internal/naming/selectors.go

@@ -59,6 +59,17 @@ func ClusterBackupJobs(cluster string) metav1.LabelSelector {
 	}
 }
 
+func ClusterRepoHost(cluster string) metav1.LabelSelector {
+	return metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			LabelCluster: cluster,
+		},
+		MatchExpressions: []metav1.LabelSelectorRequirement{
+			{Key: LabelPgbackrestDedicated, Operator: metav1.LabelSelectorOpExists},
+		},
+	}
+}
+
 // ClusterDataForPostgresAndPGBackRest selects things for PostgreSQL data and
 // things for pgBackRest data.
 func ClusterDataForPostgresAndPGBackRest(cluster string) metav1.LabelSelector {

percona/clientcmd/clientcmd.go

@@ -3,6 +3,7 @@ package clientcmd
 import (
 	"context"
 	"io"
+	"log"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -68,6 +69,7 @@ func (c *Client) Exec(ctx context.Context, pod *corev1.Pod, containerName string
 			TTY:       tty,
 		}, scheme.ParameterCodec)
 
+	log.Println("Execing in pod", pod.Name, containerName, command)
 	exec, err := remotecommand.NewSPDYExecutor(c.restconfig, "POST", req.URL())
 	if err != nil {
 		return errors.Wrap(err, "failed to create executor")

percona/controller/pgbackup/controller.go

@@ -548,9 +548,13 @@ func updatePGBackrestInfo(ctx context.Context, c client.Client, pod *corev1.Pod,
 
 func finishBackup(ctx context.Context, c client.Client, pgBackup *v2.PerconaPGBackup, job *batchv1.Job) (*reconcile.Result, error) {
 	if checkBackupJob(job) == v2.BackupSucceeded {
+		// MARK(AG): Pod for running pgbackrest info.
+		//  Read the repo-host pod instead.
+
+		// readyPod, err := controller.GetReadyRepoHostPod(ctx, c, pgBackup.Spec.PGCluster, pgBackup.Namespace)
 		readyPod, err := controller.GetReadyInstancePod(ctx, c, pgBackup.Spec.PGCluster, pgBackup.Namespace)
 		if err != nil {
-			return nil, errors.Wrap(err, "get ready instance pod")
+			return nil, errors.Wrap(err, "get ready repo-host pod")
 		}
 
 		if err := updatePGBackrestInfo(ctx, c, readyPod, pgBackup); err != nil {

percona/controller/pgcluster/backup.go

@@ -53,7 +53,7 @@ func (r *PGClusterReconciler) cleanupOutdatedBackups(ctx context.Context, cr *v2
 			continue
 		}
 
-		readyPod, err := controller.GetReadyInstancePod(ctx, r.Client, cr.Name, cr.Namespace)
+		readyPod, err := controller.GetReadyRepoHostPod(ctx, r.Client, cr.Name, cr.Namespace)
 		if err != nil {
 			return errors.Wrap(err, "get ready instance pod")
 		}

percona/controller/utils.go

@@ -62,12 +62,32 @@ func (m *CustomManager) Add(r manager.Runnable) error {
 	return nil
 }
 
+func GetReadyRepoHostPod(ctx context.Context, c client.Client, clusterName, namespace string) (*corev1.Pod, error) {
+	pods := &corev1.PodList{}
+	selector, err := naming.AsSelector(naming.ClusterRepoHost(clusterName))
+	if err != nil {
+		return nil, err
+	}
+	if err := c.List(ctx, pods, client.InNamespace(namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil {
+		return nil, errors.Wrap(err, "list pods")
+	}
+
+	for _, pod := range pods.Items {
+		if pod.Status.Phase != corev1.PodRunning {
+			continue
+		}
+		return &pod, nil
+	}
+	return nil, errors.New("no running repo-host found")
+}
+
 func GetReadyInstancePod(ctx context.Context, c client.Client, clusterName, namespace string) (*corev1.Pod, error) {
 	pods := &corev1.PodList{}
 	selector, err := naming.AsSelector(naming.ClusterInstances(clusterName))
 	if err != nil {
 		return nil, err
 	}
+	// Mark (AG): Do something similar for repo-host.
 	if err := c.List(ctx, pods, client.InNamespace(namespace), client.MatchingLabelsSelector{Selector: selector}); err != nil {
 		return nil, errors.Wrap(err, "list pods")
 	}

percona/pgbackrest/pgbackrest.go

@@ -57,7 +57,8 @@ func GetInfo(ctx context.Context, pod *corev1.Pod, repoName string) (InfoOutput,
 		return InfoOutput{}, errors.Wrap(err, "failed to create client")
 	}
 
-	if err := c.Exec(ctx, pod, naming.ContainerDatabase, nil, stdout, stderr, "pgbackrest", "info", "--output=json", "--repo="+strings.TrimPrefix(repoName, "repo")); err != nil {
+	// naming.PGBackRestRepoContainerName
+	if err := c.Exec(ctx, pod, naming.ContainerPGBackRest, nil, stdout, stderr, "pgbackrest", "info", "--output=json", "--repo="+strings.TrimPrefix(repoName, "repo")); err != nil {
 		return InfoOutput{}, errors.Wrapf(err, "exec: %s", stderr.String())
 	}
 
@@ -98,7 +99,7 @@ func SetAnnotationsToBackup(ctx context.Context, pod *corev1.Pod, stanza string,
 	cmd = append(cmd, annotationsOpts...)
 	cmd = append(cmd, "annotate")
 
-	if err := c.Exec(ctx, pod, naming.ContainerDatabase, nil, nil, stderr, cmd...); err != nil {
+	if err := c.Exec(ctx, pod, naming.ContainerPGBackRest, nil, nil, stderr, cmd...); err != nil {
 		return errors.Wrapf(err, "exec: %s", stderr.String())
 	}
 

percona/postgres/common.go

@@ -32,16 +32,37 @@ func GetPrimaryPod(ctx context.Context, cli client.Client, cr *v2.PerconaPGClust
 			"postgres-operator.crunchydata.com/role": role,
 		}),
 	})
+
+	if len(podList.Items) == 0 {
+		return nil, errors.New("no primary pod found")
+	}
+
+	if len(podList.Items) > 1 {
+		return nil, errors.New("multiple primary pods found")
+	}
+
+	return &podList.Items[0], nil
+}
+
+func GetRepoHostPod(ctx context.Context, cli client.Client, cr *v2.PerconaPGCluster) (*corev1.Pod, error) {
+	podList := &corev1.PodList{}
+	err := cli.List(ctx, podList, &client.ListOptions{
+		Namespace: cr.Namespace,
+		LabelSelector: labels.SelectorFromSet(map[string]string{
+			"app.kubernetes.io/instance":                             cr.Name,
+			"postgres-operator.crunchydata.com/pgbackrest-dedicated": "",
+		}),
+	})
 	if err != nil {
 		return nil, err
 	}
 
 	if len(podList.Items) == 0 {
-		return nil, errors.New("no primary pod found")
+		return nil, errors.New("no repo-host pod found")
 	}
 
 	if len(podList.Items) > 1 {
-		return nil, errors.New("multiple primary pods found")
+		return nil, errors.New("multiple repo-host pods found")
 	}
 
 	return &podList.Items[0], nil

percona/watcher/wal.go

@@ -191,11 +191,13 @@ func GetLatestCommitTimestamp(ctx context.Context, cli client.Client, execCli *c
 }
 
 func getBackupStartTimestamp(ctx context.Context, cli client.Client, cr *pgv2.PerconaPGCluster, backup *pgv2.PerconaPGBackup) (time.Time, error) {
-	primary, err := perconaPG.GetPrimaryPod(ctx, cli, cr)
+	// MARK(AG): This might break.
+	primary, err := perconaPG.GetRepoHostPod(ctx, cli, cr)
 	if err != nil {
 		return time.Time{}, PrimaryPodNotFound
 	}
 
+	// MARK(AG): More pgbackest stuff
 	pgbackrestInfo, err := pgbackrest.GetInfo(ctx, primary, backup.Spec.RepoName)
 	if err != nil {
 		return time.Time{}, errors.Wrap(err, "get pgbackrest info")